From a319424e4527cbfce2e4b913dae2aba0fc6fed53 Mon Sep 17 00:00:00 2001 From: Waleed Date: Fri, 24 Jul 2026 12:29:27 -0700 Subject: [PATCH 01/53] feat(realtime): add shared room identity + authorization spine (#5929) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces the foundation for a unified realtime "room" model spanning the Socket.IO presence server (apps/realtime), the durable SSE event log, and the ephemeral pub/sub fanout — all of which today reinvent their own room identity, naming, and authorization. - @sim/realtime-protocol/rooms: RoomRef { type, id }, ROOM_TYPES, and a roomName/parseRoomName codec. WORKFLOW deliberately maps to the bare id so the ~40 existing io.to(workflowId) callsites and presence state keys are unchanged; every other room type is namespaced so id spaces cannot collide. - @sim/platform-authz/rooms: authorizeRoom(userId, room, action) generalizing the exemplary authorizeWorkflowByWorkspacePermission — one resource->workspace resolver per room type, then the shared resolveEffectiveWorkspacePermission + permissionSatisfies gate. Pure foundation, no behavior change: nothing consumes these yet. Prune graph stays at 14/25 (platform-authz already depended transitively on realtime-protocol via apps/realtime). --- bun.lock | 2 + packages/platform-authz/package.json | 5 + packages/platform-authz/src/rooms.ts | 133 +++++++++++++++++++ packages/realtime-protocol/package.json | 9 +- packages/realtime-protocol/src/rooms.test.ts | 85 ++++++++++++ packages/realtime-protocol/src/rooms.ts | 89 +++++++++++++ packages/realtime-protocol/vitest.config.ts | 9 ++ 7 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 packages/platform-authz/src/rooms.ts create mode 100644 packages/realtime-protocol/src/rooms.test.ts create mode 100644 packages/realtime-protocol/src/rooms.ts create mode 100644 packages/realtime-protocol/vitest.config.ts diff --git a/bun.lock b/bun.lock index 849269e0794..033da3635b8 100644 --- a/bun.lock +++ b/bun.lock @@ -446,6 +446,7 @@ "version": "0.1.0", "dependencies": { "@sim/db": "workspace:*", + "@sim/realtime-protocol": "workspace:*", "drizzle-orm": "^0.45.2", }, "devDependencies": { @@ -463,6 +464,7 @@ "devDependencies": { "@sim/tsconfig": "workspace:*", "typescript": "^7.0.2", + "vitest": "^4.1.0", }, }, "packages/runtime-secrets": { diff --git a/packages/platform-authz/package.json b/packages/platform-authz/package.json index fd989954ec1..d8d0bca5b03 100644 --- a/packages/platform-authz/package.json +++ b/packages/platform-authz/package.json @@ -21,6 +21,10 @@ "./workflow": { "types": "./src/workflow.ts", "default": "./src/workflow.ts" + }, + "./rooms": { + "types": "./src/rooms.ts", + "default": "./src/rooms.ts" } }, "scripts": { @@ -32,6 +36,7 @@ }, "dependencies": { "@sim/db": "workspace:*", + "@sim/realtime-protocol": "workspace:*", "drizzle-orm": "^0.45.2" }, "devDependencies": { diff --git a/packages/platform-authz/src/rooms.ts b/packages/platform-authz/src/rooms.ts new file mode 100644 index 00000000000..1fd9949333e --- /dev/null +++ b/packages/platform-authz/src/rooms.ts @@ -0,0 +1,133 @@ +import { db, workspace } from '@sim/db' +import { ROOM_TYPES, type RoomRef, type RoomType } from '@sim/realtime-protocol/rooms' +import { and, eq, isNull } from 'drizzle-orm' +import { getActiveWorkflowContext } from './workflow' +import { + type PermissionType, + permissionSatisfies, + resolveEffectiveWorkspacePermission, +} from './workspace' + +export type { PermissionType, RoomRef, RoomType } + +/** + * The owning workspace of a room, plus the org that owns that workspace — the + * exact inputs {@link resolveEffectiveWorkspacePermission} needs. + */ +export interface RoomWorkspace { + workspaceId: string + workspaceOrganizationId: string | null +} + +/** + * Resolves a room's owning workspace from its {@link RoomRef.id}. Returns `null` + * when the underlying resource is missing/archived (→ a 404 authorization + * result). One resolver per {@link RoomType}; this is the single place a new + * room type declares its resource→workspace lookup. + */ +export type RoomWorkspaceResolver = (roomId: string) => Promise + +async function resolveWorkspaceRoomWorkspace(workspaceId: string): Promise { + const [row] = await db + .select({ id: workspace.id, organizationId: workspace.organizationId }) + .from(workspace) + .where(and(eq(workspace.id, workspaceId), isNull(workspace.archivedAt))) + .limit(1) + + return row ? { workspaceId: row.id, workspaceOrganizationId: row.organizationId } : null +} + +/** + * Single source of truth mapping each room type to its resource→workspace + * lookup. Every realtime room is workspace-scoped and authorizes through the + * same effective-permission resolver, so a room type only has to say *which* + * workspace it belongs to. Adding a room type = adding one entry here. + */ +const ROOM_WORKSPACE_RESOLVERS: Record = { + [ROOM_TYPES.WORKFLOW]: async (workflowId) => { + const context = await getActiveWorkflowContext(workflowId) + if (!context?.workspaceId) return null + return { + workspaceId: context.workspaceId, + workspaceOrganizationId: context.workspaceOrganizationId, + } + }, + // A workspace-files room is addressed directly by its workspace id. + [ROOM_TYPES.WORKSPACE_FILES]: resolveWorkspaceRoomWorkspace, +} + +/** Resolves a room's owning workspace, or `null` if the room resource is gone. */ +export function resolveWorkspaceIdForRoom(room: RoomRef): Promise { + return ROOM_WORKSPACE_RESOLVERS[room.type](room.id) +} + +export interface RoomAuthorizationResult { + allowed: boolean + status: number + message?: string + workspaceId: string | null + workspacePermission: PermissionType | null +} + +/** + * Authorizes a user against a realtime room. Mirrors + * `authorizeWorkflowByWorkspacePermission` (the exemplary workflow authorizer) + * but generalized over room type: resolve the room's workspace, then gate on the + * user's effective workspace permission under the read < write < admin ordering. + * + * Returns a denial (never throws) for unknown room type (400), missing/archived + * resource (404), and insufficient permission (403), so realtime handlers and + * SSE routes can map the `status` to a wire error uniformly. + */ +export async function authorizeRoom(params: { + userId: string + room: RoomRef + action?: PermissionType +}): Promise { + const { userId, room, action = 'read' } = params + + const resolver = ROOM_WORKSPACE_RESOLVERS[room.type] + if (!resolver) { + return { + allowed: false, + status: 400, + message: `Unknown room type: ${room.type}`, + workspaceId: null, + workspacePermission: null, + } + } + + const roomWorkspace = await resolver(room.id) + if (!roomWorkspace) { + return { + allowed: false, + status: 404, + message: 'Room not found', + workspaceId: null, + workspacePermission: null, + } + } + + const workspacePermission = await resolveEffectiveWorkspacePermission( + userId, + roomWorkspace.workspaceId, + roomWorkspace.workspaceOrganizationId + ) + + if (!permissionSatisfies(workspacePermission, action)) { + return { + allowed: false, + status: 403, + message: `Access denied to ${action} this room`, + workspaceId: roomWorkspace.workspaceId, + workspacePermission, + } + } + + return { + allowed: true, + status: 200, + workspaceId: roomWorkspace.workspaceId, + workspacePermission, + } +} diff --git a/packages/realtime-protocol/package.json b/packages/realtime-protocol/package.json index a4248404adc..4f0ea685ebe 100644 --- a/packages/realtime-protocol/package.json +++ b/packages/realtime-protocol/package.json @@ -21,10 +21,16 @@ "./schemas": { "types": "./src/schemas.ts", "default": "./src/schemas.ts" + }, + "./rooms": { + "types": "./src/rooms.ts", + "default": "./src/rooms.ts" } }, "scripts": { "type-check": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest", "lint": "biome check --write --unsafe .", "lint:check": "biome check .", "format": "biome format --write .", @@ -35,6 +41,7 @@ }, "devDependencies": { "@sim/tsconfig": "workspace:*", - "typescript": "^7.0.2" + "typescript": "^7.0.2", + "vitest": "^4.1.0" } } diff --git a/packages/realtime-protocol/src/rooms.test.ts b/packages/realtime-protocol/src/rooms.test.ts new file mode 100644 index 00000000000..6ab07de542e --- /dev/null +++ b/packages/realtime-protocol/src/rooms.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest' +import { + ALL_ROOM_TYPES, + isRoomType, + isSameRoom, + parseRoomName, + ROOM_TYPES, + type RoomRef, + roomName, +} from './rooms' + +describe('roomName', () => { + it('maps a workflow room to its bare id (backward compatibility)', () => { + expect(roomName({ type: ROOM_TYPES.WORKFLOW, id: 'wf-123' })).toBe('wf-123') + }) + + it('namespaces every non-workflow room type', () => { + expect(roomName({ type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-123' })).toBe( + 'workspace-files:ws-123' + ) + }) + + it('never collides a namespaced room with a bare workflow id for real ids', () => { + // Room ids in Sim are opaque tokens without a colon (UUIDs / short ids), so a + // bare workflow id can never look like a `${type}:${id}` namespaced name. + const workflow = roomName({ type: ROOM_TYPES.WORKFLOW, id: 'a1b2c3d4-uuid' }) + const files = roomName({ type: ROOM_TYPES.WORKSPACE_FILES, id: 'a1b2c3d4-uuid' }) + expect(workflow).not.toBe(files) + expect(workflow.includes(':')).toBe(false) + }) +}) + +describe('parseRoomName', () => { + it('round-trips every room type through roomName', () => { + const refs: RoomRef[] = [ + { type: ROOM_TYPES.WORKFLOW, id: 'wf-123' }, + { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-456' }, + ] + for (const ref of refs) { + expect(parseRoomName(roomName(ref))).toEqual(ref) + } + }) + + it('treats an unprefixed name as a workflow room', () => { + expect(parseRoomName('bare-uuid')).toEqual({ type: ROOM_TYPES.WORKFLOW, id: 'bare-uuid' }) + }) + + it('preserves ids that themselves contain colons', () => { + expect(parseRoomName('workspace-files:a:b:c')).toEqual({ + type: ROOM_TYPES.WORKSPACE_FILES, + id: 'a:b:c', + }) + }) + + it('does not treat an unknown prefix as a room type', () => { + expect(parseRoomName('unknown:x')).toEqual({ type: ROOM_TYPES.WORKFLOW, id: 'unknown:x' }) + }) + + it('returns null for the empty string', () => { + expect(parseRoomName('')).toBeNull() + }) +}) + +describe('isRoomType', () => { + it('accepts known types and rejects others', () => { + expect(isRoomType(ROOM_TYPES.WORKFLOW)).toBe(true) + expect(isRoomType(ROOM_TYPES.WORKSPACE_FILES)).toBe(true) + expect(isRoomType('nope')).toBe(false) + }) +}) + +describe('isSameRoom', () => { + it('compares by type and id', () => { + const a: RoomRef = { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-1' } + expect(isSameRoom(a, { ...a })).toBe(true) + expect(isSameRoom(a, { type: ROOM_TYPES.WORKFLOW, id: 'ws-1' })).toBe(false) + expect(isSameRoom(a, { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-2' })).toBe(false) + }) +}) + +describe('ALL_ROOM_TYPES', () => { + it('contains every declared room type', () => { + expect([...ALL_ROOM_TYPES].sort()).toEqual([...Object.values(ROOM_TYPES)].sort()) + }) +}) diff --git a/packages/realtime-protocol/src/rooms.ts b/packages/realtime-protocol/src/rooms.ts new file mode 100644 index 00000000000..994bf6b3cac --- /dev/null +++ b/packages/realtime-protocol/src/rooms.ts @@ -0,0 +1,89 @@ +/** + * Room identity for the realtime layer. + * + * A {@link RoomRef} is the universal address shared by every realtime mechanism + * in Sim — the Socket.IO presence server (`apps/realtime`), the durable SSE + * event log, and the ephemeral pub/sub fanout. Each mechanism encodes a room + * differently on the wire, but they all agree on this `{ type, id }` identity + * and authorize it through the same workspace-permission resolver + * (`@sim/platform-authz/rooms`). + * + * This module is pure (no runtime dependencies) so both `apps/sim` and + * `apps/realtime` can import it. + */ + +/** + * The kinds of realtime room. Each value is a stable wire token — changing one + * is a breaking protocol change (it renames Socket.IO rooms and Redis keys), so + * treat these like enum values that ship to clients. + */ +export const ROOM_TYPES = { + /** The collaborative workflow editor canvas (one room per workflow). */ + WORKFLOW: 'workflow', + /** The workspace file browser (one room per workspace). */ + WORKSPACE_FILES: 'workspace-files', +} as const + +export type RoomType = (typeof ROOM_TYPES)[keyof typeof ROOM_TYPES] + +/** Every known room type, for exhaustive iteration/validation. */ +export const ALL_ROOM_TYPES = Object.values(ROOM_TYPES) as readonly RoomType[] + +/** Universal address of a realtime room. */ +export interface RoomRef { + type: RoomType + id: string +} + +/** Type guard: whether an arbitrary string is a known {@link RoomType}. */ +export function isRoomType(value: string): value is RoomType { + return (ALL_ROOM_TYPES as readonly string[]).includes(value) +} + +/** + * The Socket.IO room name (and default key segment) for a room. + * + * `WORKFLOW` maps to the **bare id** — deliberately. The workflow editor has + * ~40 existing `io.to(workflowId)` / `socket.join(workflowId)` callsites that + * pass the bare workflow id, plus stale-cleanup that cross-references + * `io.in(workflowId).fetchSockets()` against Redis presence state. Preserving + * the bare name keeps every one of those callsites correct with zero diff and + * zero presence-state migration. Every *other* room type is namespaced + * (`${type}:${id}`) so a new id space can never collide with a workflow UUID. + * + * The inverse ({@link parseRoomName}) relies on this: an unprefixed name is a + * workflow, a prefixed name splits on the first `:`. + * + * Precondition: room ids are opaque tokens that never contain `:` — satisfied by + * every id in Sim (`generateId()` UUIDs, `generateShortId()` URL-safe tokens, + * workspace ids). This is what makes a bare workflow id unambiguous against a + * `${type}:${id}` namespace and keeps {@link parseRoomName} lossless. + */ +export function roomName(room: RoomRef): string { + return room.type === ROOM_TYPES.WORKFLOW ? room.id : `${room.type}:${room.id}` +} + +/** + * Inverse of {@link roomName}. A name carrying a known `${type}:` prefix parses + * to that type; any other (unprefixed) name is a {@link ROOM_TYPES.WORKFLOW} + * room whose id is the whole string — see {@link roomName} for why workflow is + * unprefixed. Returns `null` only for the empty string. + */ +export function parseRoomName(name: string): RoomRef | null { + if (!name) return null + + const separatorIndex = name.indexOf(':') + if (separatorIndex > 0) { + const maybeType = name.slice(0, separatorIndex) + if (isRoomType(maybeType) && maybeType !== ROOM_TYPES.WORKFLOW) { + return { type: maybeType, id: name.slice(separatorIndex + 1) } + } + } + + return { type: ROOM_TYPES.WORKFLOW, id: name } +} + +/** Whether two room refs address the same room. */ +export function isSameRoom(a: RoomRef, b: RoomRef): boolean { + return a.type === b.type && a.id === b.id +} diff --git a/packages/realtime-protocol/vitest.config.ts b/packages/realtime-protocol/vitest.config.ts new file mode 100644 index 00000000000..471771e48fe --- /dev/null +++ b/packages/realtime-protocol/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + globals: false, + environment: 'node', + include: ['src/**/*.test.ts'], + }, +}) From d4d2238bc0d8c983d082510229ee0f6409734343 Mon Sep 17 00:00:00 2001 From: Waleed Date: Fri, 24 Jul 2026 12:30:46 -0700 Subject: [PATCH 02/53] refactor(realtime): generalize presence server to multi-room [2/N] (#5930) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(realtime): generalize presence server to multi-room (RoomRef) Generalizes the Socket.IO presence layer from single-workflow-room-per-socket to a domain-neutral, multi-room-per-socket model keyed by RoomRef, so a second domain (workspace files, next PR) can reuse the same membership + presence engine. Behavior-preserving for workflow collaboration. IRoomManager is now domain-neutral (addUserToRoom/removeUserFromRoom/ getRoomForSocket/getRoomUsers/updateUserActivity/... all take a RoomRef). The workflow lifecycle broadcasts (deletion/revert/update/deploy) move out of the manager into WorkflowRoomService, composed over the generic manager. Backward-compat by design (no workflow migration, no regression): - Workflow Socket.IO room name stays the bare workflowId (roomName() maps workflow -> bare id), so the ~40 io.to(workflowId) callsites are untouched. - Workflow Redis presence keys stay workflow:{id}:users/:meta (the type prefix IS "workflow"). Multi-room correctness (from adversarial audit): - socket:{id}:workflow single-value key -> socket:{id}:rooms HASH (type->id). - The SHARED socket:{id}:session key is deleted only when the socket leaves its LAST room (refcount via HLEN) — a leave from one room no longer breaks the other room's handlers. - disconnect enumerates the socket's stored rooms and rebroadcasts presence per room, instead of picking an arbitrary socket.rooms entry. - presence broadcasts use a per-room-type event name (workflow keeps the bare presence-update; others are namespaced). Workflow handlers wrap manager calls with a shared workflowRoom(id) helper; UserPresence.workflowId -> room (the client never reads that field). Tests: existing 112 realtime tests pass unchanged (behavior gate) + 7 new multi-room tests (refcounted session, presence isolation, multi-room disconnect, per-type event names). tsc clean, boundaries + prune (14/25) green. * fix(realtime): harden multi-room disconnect + id-guard room removal Two fixes from an adversarial regression audit of the multi-room refactor: - Disconnect now handles `disconnecting` (where `socket.rooms` is still populated and authoritative) and falls back to the live Socket.IO room set for any room the manager's stored state no longer tracked. This restores reliable presence cleanup + departure broadcast even if the Redis `socket:{id}:rooms` key was evicted or TTL-expired — the one behavioral gap vs the pre-refactor disconnect. - REMOVE_ROOM_SCRIPT now only drops the socket's room mapping (and runs the last-room session cleanup) when the stored id matches the room being removed, matching the memory manager's existing id guard. Prevents a mismatched-room call from wiping a different room's mapping or the shared session. +1 test (id-guarded no-op removal). 120 realtime tests pass, tsc clean. * fix(realtime): only rebroadcast disconnect-fallback rooms whose removal succeeded Greptile 4/5 follow-up: the disconnecting-time fallback ignored removeUserFromRoom's boolean and rebroadcast presence even when the removal reported false. Now it only treats a room as removed (and rebroadcasts) when the manager confirms it — symmetric with removeSocketFromAllRooms, which already only returns rooms it actually removed. * fix(realtime): exclude the disconnecting socket from its farewell broadcast Greptile follow-up (transient-Redis-failure edge): if removeUserFromRoom fails on disconnect, the socket's presence entry can outlive it (room hashes have no TTL) and reappear as a ghost. Disconnect now broadcasts a correction to EVERY room the socket was in (union of the manager's removed rooms and the live Socket.IO membership) and passes the disconnecting socket id as excludeSocketId, so it is never shown as a collaborator regardless of whether the Redis delete succeeded. Any orphaned entry is still reclaimed by the next join's stale-presence sweep. broadcastPresenceUpdate gains an optional excludeSocketId; normal broadcasts are unchanged. +1 test. * fix(realtime): make presence broadcasts liveness-aware (root-cause ghost fix) Presence broadcasts now reconcile the stored list against the live Socket.IO membership (io.in(room).fetchSockets()) before emitting, via a shared filterVisiblePresence helper. This closes the residual behind the earlier disconnect fixes: an entry orphaned by a failed removal (room hashes have no TTL) could reappear in a LATER join's presence snapshot until the 75-min stale sweep. Now such an entry is never emitted, because a non-live socket is filtered out of every broadcast. Combined with excludeSocketId (which handles the disconnecting socket, still momentarily live). Fail-safe: on a fetchSockets throw or an empty result while entries remain, emit the unfiltered list rather than hide live collaborators. Also drops a dead guard in the disconnect union loop (rooms already removed are skipped by the wasInRooms check) and the now-unused isSameRoom import. +1 ghost-guard test. 122 realtime tests pass. --- apps/realtime/src/handlers/connection.ts | 47 ++- apps/realtime/src/handlers/operations.ts | 41 +- apps/realtime/src/handlers/presence.ts | 21 +- apps/realtime/src/handlers/subblocks.ts | 12 +- apps/realtime/src/handlers/variables.ts | 12 +- apps/realtime/src/handlers/workflow.test.ts | 19 +- apps/realtime/src/handlers/workflow.ts | 45 +- apps/realtime/src/index.test.ts | 34 +- apps/realtime/src/rooms/index.ts | 3 +- .../realtime/src/rooms/memory-manager.test.ts | 175 ++++++++ apps/realtime/src/rooms/memory-manager.ts | 279 +++++-------- .../realtime/src/rooms/presence-visibility.ts | 36 ++ apps/realtime/src/rooms/redis-manager.ts | 387 +++++++----------- apps/realtime/src/rooms/types.ts | 137 +++---- .../src/rooms/workflow-room-service.ts | 110 +++++ apps/realtime/src/routes/http.ts | 12 +- packages/realtime-protocol/src/rooms.ts | 10 + 17 files changed, 793 insertions(+), 587 deletions(-) create mode 100644 apps/realtime/src/rooms/memory-manager.test.ts create mode 100644 apps/realtime/src/rooms/presence-visibility.ts create mode 100644 apps/realtime/src/rooms/workflow-room-service.ts diff --git a/apps/realtime/src/handlers/connection.ts b/apps/realtime/src/handlers/connection.ts index 90eddb82464..2f180e7fa53 100644 --- a/apps/realtime/src/handlers/connection.ts +++ b/apps/realtime/src/handlers/connection.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { parseRoomName, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' import { cleanupPendingSubblocksForSocket } from '@/handlers/subblocks' import { cleanupPendingVariablesForSocket } from '@/handlers/variables' import type { AuthenticatedSocket } from '@/middleware/auth' @@ -15,20 +16,50 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager logger.error(`Socket ${socket.id} connection error:`, error) }) - socket.on('disconnect', async (reason) => { + // `disconnecting` (not `disconnect`): here `socket.rooms` is still populated and + // authoritative, so presence is cleaned up even if the Redis room-set key was + // evicted or TTL-expired (which would leave the manager's stored rooms empty). + socket.on('disconnecting', async (reason) => { try { // Clean up pending debounce entries for this socket to prevent memory leaks cleanupPendingSubblocksForSocket(socket.id) cleanupPendingVariablesForSocket(socket.id) - const workflowIdHint = [...socket.rooms].find((roomId) => roomId !== socket.id) - const workflowId = await roomManager.removeUserFromRoom(socket.id, workflowIdHint) + // A socket may occupy multiple rooms (one per type). Remove it from every + // room the manager knows about. + const removedRooms = await roomManager.removeSocketFromAllRooms(socket.id) - if (workflowId) { - await roomManager.broadcastPresenceUpdate(workflowId) - logger.info( - `Socket ${socket.id} disconnected from workflow ${workflowId} (reason: ${reason})` - ) + // Union with the live Socket.IO membership (authoritative here, and it + // survives a Redis eviction/TTL lapse that would leave the manager's tracked + // rooms empty). Attempt removal for any room the manager didn't already + // remove — best-effort, since a transient Redis error can't be recovered here. + const wasInRooms = new Map() + for (const room of removedRooms) wasInRooms.set(roomName(room), room) + for (const name of socket.rooms) { + // `wasInRooms.has(name)` already excludes every room the manager removed + // (same room-name key via the roomName/parseRoomName bijection), so any + // room reaching here was NOT in `removedRooms` and needs a removal attempt. + if (name === socket.id || wasInRooms.has(name)) continue + const ref = parseRoomName(name) + if (!ref) continue + wasInRooms.set(name, ref) + await roomManager.removeUserFromRoom(ref, socket.id) + } + + // Broadcast a correction to every room this socket was in, EXCLUDING this + // socket — so it is never shown as a ghost collaborator even if its presence + // entry outlived a failed removal (transient Redis error; the hashes have no + // TTL). Any orphaned entry is additionally reclaimed by the next join's + // stale-presence sweep. + for (const room of wasInRooms.values()) { + await roomManager.broadcastPresenceUpdate(room, socket.id) + } + + if (wasInRooms.size > 0) { + const rooms = Array.from(wasInRooms.values()) + .map((room) => `${room.type}:${room.id}`) + .join(', ') + logger.info(`Socket ${socket.id} disconnected from [${rooms}] (reason: ${reason})`) } } catch (error) { logger.error(`Error handling disconnect for socket ${socket.id}:`, error) diff --git a/apps/realtime/src/handlers/operations.ts b/apps/realtime/src/handlers/operations.ts index eef51847718..d3d603887db 100644 --- a/apps/realtime/src/handlers/operations.ts +++ b/apps/realtime/src/handlers/operations.ts @@ -9,6 +9,7 @@ import { type VariableOperation, WORKFLOW_OPERATIONS, } from '@sim/realtime-protocol/constants' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { WorkflowOperationSchema } from '@sim/realtime-protocol/schemas' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' @@ -16,7 +17,7 @@ import { ZodError } from 'zod' import { persistWorkflowOperation } from '@/database/operations' import type { AuthenticatedSocket } from '@/middleware/auth' import { checkWorkflowOperationPermission } from '@/middleware/permissions' -import type { IRoomManager, UserSession } from '@/rooms' +import { type IRoomManager, type UserSession, workflowRoom as wf } from '@/rooms' const logger = createLogger('OperationsHandlers') @@ -44,7 +45,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager let session: UserSession | null = null try { - workflowId = await roomManager.getWorkflowIdForSocket(socket.id) + workflowId = (await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW))?.id ?? null session = await roomManager.getUserSession(socket.id) } catch (error) { logger.error('Error loading session for workflow operation:', error) @@ -65,7 +66,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager let hasRoom = false try { - hasRoom = await roomManager.hasWorkflowRoom(workflowId) + hasRoom = await roomManager.hasRoom(wf(workflowId)) } catch (error) { logger.error('Error checking workflow room:', error) emitOperationError( @@ -98,14 +99,16 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager const operationTimestamp = isPositionUpdate ? timestamp : Date.now() // Get user presence for permission checking - const users = await roomManager.getWorkflowUsers(workflowId) + const users = await roomManager.getRoomUsers(wf(workflowId)) const userPresence = users.find((u) => u.socketId === socket.id) // Skip permission checks for non-committed position updates (broadcasts only, no persistence) if (isPositionUpdate && !commitPositionUpdate) { // Update last activity if (userPresence) { - await roomManager.updateUserActivity(workflowId, socket.id, { lastActivity: Date.now() }) + await roomManager.updateUserActivity(wf(workflowId), socket.id, { + lastActivity: Date.now(), + }) } } else { // Check permissions from cached role for all other operations @@ -123,7 +126,9 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager return } - await roomManager.updateUserActivity(workflowId, socket.id, { lastActivity: Date.now() }) + await roomManager.updateUserActivity(wf(workflowId), socket.id, { + lastActivity: Date.now(), + }) // Re-validate the workspace role against the DB (cached per pod for a short // window) so revoked or downgraded collaborators lose write access live. @@ -198,7 +203,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager timestamp: operationTimestamp, userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) if (operationId) { socket.emit('operation-confirmed', { @@ -244,7 +249,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager timestamp: operationTimestamp, userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) if (operationId) { socket.emit('operation-confirmed', { operationId, serverTimestamp: Date.now() }) @@ -277,7 +282,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) const broadcastData = { operation, @@ -317,7 +322,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) const broadcastData = { operation, @@ -354,7 +359,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -386,7 +391,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -415,7 +420,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -447,7 +452,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -479,7 +484,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -511,7 +516,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -540,7 +545,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -569,7 +574,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) const broadcastData = { operation, diff --git a/apps/realtime/src/handlers/presence.ts b/apps/realtime/src/handlers/presence.ts index 13aadc22f34..78b53176e2f 100644 --- a/apps/realtime/src/handlers/presence.ts +++ b/apps/realtime/src/handlers/presence.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -7,16 +8,16 @@ const logger = createLogger('PresenceHandlers') export function setupPresenceHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { socket.on('cursor-update', async ({ cursor }) => { try { - const workflowId = await roomManager.getWorkflowIdForSocket(socket.id) + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) const session = await roomManager.getUserSession(socket.id) - if (!workflowId || !session) return + if (!room || !session) return // Update cursor in room state - await roomManager.updateUserActivity(workflowId, socket.id, { cursor }) + await roomManager.updateUserActivity(room, socket.id, { cursor }) - // Broadcast to other users in the room - socket.to(workflowId).emit('cursor-update', { + // Broadcast to other users in the room (workflow room name is the bare id) + socket.to(room.id).emit('cursor-update', { socketId: socket.id, userId: session.userId, userName: session.userName, @@ -30,16 +31,16 @@ export function setupPresenceHandlers(socket: AuthenticatedSocket, roomManager: socket.on('selection-update', async ({ selection }) => { try { - const workflowId = await roomManager.getWorkflowIdForSocket(socket.id) + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) const session = await roomManager.getUserSession(socket.id) - if (!workflowId || !session) return + if (!room || !session) return // Update selection in room state - await roomManager.updateUserActivity(workflowId, socket.id, { selection }) + await roomManager.updateUserActivity(room, socket.id, { selection }) - // Broadcast to other users in the room - socket.to(workflowId).emit('selection-update', { + // Broadcast to other users in the room (workflow room name is the bare id) + socket.to(room.id).emit('selection-update', { socketId: socket.id, userId: session.userId, userName: session.userName, diff --git a/apps/realtime/src/handlers/subblocks.ts b/apps/realtime/src/handlers/subblocks.ts index b2f94b6fb98..93e7411d148 100644 --- a/apps/realtime/src/handlers/subblocks.ts +++ b/apps/realtime/src/handlers/subblocks.ts @@ -3,12 +3,13 @@ import { workflow, workflowBlocks } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { assertWorkflowMutable, WorkflowLockedError } from '@sim/platform-authz/workflow' import { SUBBLOCK_OPERATIONS } from '@sim/realtime-protocol/constants' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { getErrorMessage } from '@sim/utils/errors' import { isWorkflowBlockProtected } from '@sim/workflow-types/workflow' import { and, eq } from 'drizzle-orm' import type { AuthenticatedSocket } from '@/middleware/auth' import { checkWorkflowOperationPermission } from '@/middleware/permissions' -import type { IRoomManager } from '@/rooms' +import { type IRoomManager, workflowRoom as wf } from '@/rooms' const logger = createLogger('SubblocksHandlers') @@ -69,7 +70,8 @@ export function setupSubblocksHandlers(socket: AuthenticatedSocket, roomManager: } try { - const sessionWorkflowId = await roomManager.getWorkflowIdForSocket(socket.id) + const sessionWorkflowId = + (await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW))?.id ?? null const session = await roomManager.getUserSession(socket.id) if (!sessionWorkflowId || !session) { @@ -106,7 +108,7 @@ export function setupSubblocksHandlers(socket: AuthenticatedSocket, roomManager: return } - const hasRoom = await roomManager.hasWorkflowRoom(workflowId) + const hasRoom = await roomManager.hasRoom(wf(workflowId)) if (!hasRoom) { logger.debug(`Ignoring subblock update: workflow room not found`, { socketId: socket.id, @@ -117,7 +119,7 @@ export function setupSubblocksHandlers(socket: AuthenticatedSocket, roomManager: return } - const users = await roomManager.getWorkflowUsers(workflowId) + const users = await roomManager.getRoomUsers(wf(workflowId)) const userPresence = users.find((user) => user.socketId === socket.id) if (!userPresence) { socket.emit('operation-forbidden', { @@ -182,7 +184,7 @@ export function setupSubblocksHandlers(socket: AuthenticatedSocket, roomManager: } // Update user activity - await roomManager.updateUserActivity(workflowId, socket.id, { lastActivity: Date.now() }) + await roomManager.updateUserActivity(wf(workflowId), socket.id, { lastActivity: Date.now() }) // Server-side debounce/coalesce by workflowId+blockId+subblockId const debouncedKey = `${workflowId}:${blockId}:${subblockId}` diff --git a/apps/realtime/src/handlers/variables.ts b/apps/realtime/src/handlers/variables.ts index 7a4303e70b3..0d6eaac2e3a 100644 --- a/apps/realtime/src/handlers/variables.ts +++ b/apps/realtime/src/handlers/variables.ts @@ -4,11 +4,12 @@ import { workflow } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { assertWorkflowMutable, WorkflowLockedError } from '@sim/platform-authz/workflow' import { VARIABLE_OPERATIONS } from '@sim/realtime-protocol/constants' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { getErrorMessage } from '@sim/utils/errors' import { eq } from 'drizzle-orm' import type { AuthenticatedSocket } from '@/middleware/auth' import { checkWorkflowOperationPermission } from '@/middleware/permissions' -import type { IRoomManager } from '@/rooms' +import { type IRoomManager, workflowRoom as wf } from '@/rooms' const logger = createLogger('VariablesHandlers') @@ -61,7 +62,8 @@ export function setupVariablesHandlers(socket: AuthenticatedSocket, roomManager: } try { - const sessionWorkflowId = await roomManager.getWorkflowIdForSocket(socket.id) + const sessionWorkflowId = + (await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW))?.id ?? null const session = await roomManager.getUserSession(socket.id) if (!sessionWorkflowId || !session) { @@ -98,7 +100,7 @@ export function setupVariablesHandlers(socket: AuthenticatedSocket, roomManager: return } - const hasRoom = await roomManager.hasWorkflowRoom(workflowId) + const hasRoom = await roomManager.hasRoom(wf(workflowId)) if (!hasRoom) { logger.debug(`Ignoring variable update: workflow room not found`, { socketId: socket.id, @@ -109,7 +111,7 @@ export function setupVariablesHandlers(socket: AuthenticatedSocket, roomManager: return } - const users = await roomManager.getWorkflowUsers(workflowId) + const users = await roomManager.getRoomUsers(wf(workflowId)) const userPresence = users.find((user) => user.socketId === socket.id) if (!userPresence) { socket.emit('operation-forbidden', { @@ -174,7 +176,7 @@ export function setupVariablesHandlers(socket: AuthenticatedSocket, roomManager: } // Update user activity - await roomManager.updateUserActivity(workflowId, socket.id, { lastActivity: Date.now() }) + await roomManager.updateUserActivity(wf(workflowId), socket.id, { lastActivity: Date.now() }) const debouncedKey = `${workflowId}:${variableId}:${field}` const existing = pendingVariableUpdates.get(debouncedKey) diff --git a/apps/realtime/src/handlers/workflow.test.ts b/apps/realtime/src/handlers/workflow.test.ts index 9dd82db87c4..89ae5521e2f 100644 --- a/apps/realtime/src/handlers/workflow.test.ts +++ b/apps/realtime/src/handlers/workflow.test.ts @@ -54,21 +54,20 @@ function createSocket(overrides?: Partial>) { function createRoomManager(overrides?: Partial): IRoomManager { return { isReady: vi.fn().mockReturnValue(true), - getWorkflowIdForSocket: vi.fn().mockResolvedValue(null), - removeUserFromRoom: vi.fn().mockResolvedValue(null), + getRoomForSocket: vi.fn().mockResolvedValue(null), + getRoomsForSocket: vi.fn().mockResolvedValue([]), + removeUserFromRoom: vi.fn().mockResolvedValue(false), + removeSocketFromAllRooms: vi.fn().mockResolvedValue([]), broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), - getWorkflowUsers: vi.fn().mockResolvedValue([]), - hasWorkflowRoom: vi.fn().mockResolvedValue(false), + getRoomUsers: vi.fn().mockResolvedValue([]), + hasRoom: vi.fn().mockResolvedValue(false), addUserToRoom: vi.fn().mockResolvedValue(undefined), getUserSession: vi.fn().mockResolvedValue(null), updateUserActivity: vi.fn().mockResolvedValue(undefined), updateRoomLastModified: vi.fn().mockResolvedValue(undefined), - emitToWorkflow: vi.fn(), + emitToRoom: vi.fn(), getUniqueUserCount: vi.fn().mockResolvedValue(1), getTotalActiveConnections: vi.fn().mockResolvedValue(0), - handleWorkflowDeletion: vi.fn().mockResolvedValue(undefined), - handleWorkflowRevert: vi.fn().mockResolvedValue(undefined), - handleWorkflowUpdate: vi.fn().mockResolvedValue(undefined), shutdown: vi.fn().mockResolvedValue(undefined), initialize: vi.fn().mockResolvedValue(undefined), io: { @@ -173,8 +172,8 @@ describe('setupWorkflowHandlers', () => { it('includes workflowId when an unexpected join failure occurs', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager({ - getWorkflowIdForSocket: vi.fn().mockRejectedValue(new Error('boom')), - removeUserFromRoom: vi.fn().mockResolvedValue(null), + getRoomForSocket: vi.fn().mockRejectedValue(new Error('boom')), + removeUserFromRoom: vi.fn().mockResolvedValue(false), }) setupWorkflowHandlers( diff --git a/apps/realtime/src/handlers/workflow.ts b/apps/realtime/src/handlers/workflow.ts index da977fcdb5e..553a92c7b78 100644 --- a/apps/realtime/src/handlers/workflow.ts +++ b/apps/realtime/src/handlers/workflow.ts @@ -1,10 +1,11 @@ import { db, user } from '@sim/db' import { createLogger } from '@sim/logger' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { eq } from 'drizzle-orm' import { getWorkflowState } from '@/database/operations' import type { AuthenticatedSocket } from '@/middleware/auth' import { verifyWorkflowAccess } from '@/middleware/permissions' -import type { IRoomManager, UserPresence } from '@/rooms' +import { type IRoomManager, type UserPresence, workflowRoom as wf } from '@/rooms' const logger = createLogger('WorkflowHandlers') @@ -64,18 +65,18 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: return } - // Leave current room if in one - const currentWorkflowId = await roomManager.getWorkflowIdForSocket(socket.id) - if (currentWorkflowId) { - socket.leave(currentWorkflowId) - await roomManager.removeUserFromRoom(socket.id, currentWorkflowId) - await roomManager.broadcastPresenceUpdate(currentWorkflowId) + // Leave current workflow room if in one + const currentRoom = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) + if (currentRoom) { + socket.leave(currentRoom.id) + await roomManager.removeUserFromRoom(currentRoom, socket.id) + await roomManager.broadcastPresenceUpdate(currentRoom) } // Keep this above Redis socket key TTL (1h) so a normal idle user is not evicted too aggressively. const STALE_THRESHOLD_MS = 75 * 60 * 1000 const now = Date.now() - const existingUsers = await roomManager.getWorkflowUsers(workflowId) + const existingUsers = await roomManager.getRoomUsers(wf(workflowId)) let liveSocketIds = new Set() let canCheckLiveness = false @@ -106,7 +107,7 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: logger.info( `Cleaning up socket ${existingUser.socketId} for user ${existingUser.userId} (same tab)` ) - await roomManager.removeUserFromRoom(existingUser.socketId, workflowId) + await roomManager.removeUserFromRoom(wf(workflowId), existingUser.socketId) await roomManager.io.in(existingUser.socketId).socketsLeave(workflowId) continue } @@ -124,7 +125,7 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: logger.info( `Cleaning up socket ${existingUser.socketId} for user ${existingUser.userId} (stale activity)` ) - await roomManager.removeUserFromRoom(existingUser.socketId, workflowId) + await roomManager.removeUserFromRoom(wf(workflowId), existingUser.socketId) await roomManager.io.in(existingUser.socketId).socketsLeave(workflowId) } catch (error) { logger.warn(`Best-effort cleanup failed for socket ${existingUser.socketId}`, error) @@ -153,7 +154,7 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: // Create presence entry const userPresence: UserPresence = { userId, - workflowId, + room: wf(workflowId), userName, socketId: socket.id, tabSessionId, @@ -164,10 +165,10 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: } // Add user to room - await roomManager.addUserToRoom(workflowId, socket.id, userPresence) + await roomManager.addUserToRoom(wf(workflowId), socket.id, userPresence) // Get current presence list for the join acknowledgment - const presenceUsers = await roomManager.getWorkflowUsers(workflowId) + const presenceUsers = await roomManager.getRoomUsers(wf(workflowId)) // Get workflow state const workflowState = await getWorkflowState(workflowId) @@ -183,9 +184,9 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: socket.emit('workflow-state', workflowState) // Broadcast presence update to all users in the room - await roomManager.broadcastPresenceUpdate(workflowId) + await roomManager.broadcastPresenceUpdate(wf(workflowId)) - const uniqueUserCount = await roomManager.getUniqueUserCount(workflowId) + const uniqueUserCount = await roomManager.getUniqueUserCount(wf(workflowId)) logger.info( `User ${userId} (${userName}) joined workflow ${workflowId}. Room now has ${uniqueUserCount} unique users.` ) @@ -193,7 +194,7 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: logger.error('Error joining workflow:', error) // Undo socket.join and room manager entry if any operation failed socket.leave(workflowId) - await roomManager.removeUserFromRoom(socket.id, workflowId) + await roomManager.removeUserFromRoom(wf(workflowId), socket.id) const isReady = roomManager.isReady() socket.emit('join-workflow-error', { workflowId, @@ -210,15 +211,15 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: return } - const workflowId = await roomManager.getWorkflowIdForSocket(socket.id) + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) const session = await roomManager.getUserSession(socket.id) - if (workflowId && session) { - socket.leave(workflowId) - await roomManager.removeUserFromRoom(socket.id, workflowId) - await roomManager.broadcastPresenceUpdate(workflowId) + if (room && session) { + socket.leave(room.id) + await roomManager.removeUserFromRoom(room, socket.id) + await roomManager.broadcastPresenceUpdate(room) - logger.info(`User ${session.userId} (${session.userName}) left workflow ${workflowId}`) + logger.info(`User ${session.userId} (${session.userName}) left workflow ${room.id}`) } } catch (error) { logger.error('Error leaving workflow:', error) diff --git a/apps/realtime/src/index.test.ts b/apps/realtime/src/index.test.ts index 92ddc101c69..5015b58d9bf 100644 --- a/apps/realtime/src/index.test.ts +++ b/apps/realtime/src/index.test.ts @@ -7,8 +7,9 @@ import { createServer, request as httpRequest } from 'http' import { createMockLogger } from '@sim/testing' import { randomInt } from '@sim/utils/random' import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { createSocketIOServer } from '@/config/socket' -import { MemoryRoomManager } from '@/rooms' +import { MemoryRoomManager, workflowRoom } from '@/rooms' import { createHttpHandler } from '@/routes/http' vi.mock('@/auth', () => ({ @@ -230,9 +231,10 @@ describe('Socket Server Index Integration', () => { const workflowId = 'test-workflow-123' const socketId = 'test-socket-123' + const room = workflowRoom(workflowId) const presence = { userId: 'user-123', - workflowId, + room, userName: 'Test User', socketId, joinedAt: Date.now(), @@ -240,10 +242,10 @@ describe('Socket Server Index Integration', () => { role: 'admin', } - await roomManager.addUserToRoom(workflowId, socketId, presence) + await roomManager.addUserToRoom(room, socketId, presence) - expect(await roomManager.hasWorkflowRoom(workflowId)).toBe(true) - const users = await roomManager.getWorkflowUsers(workflowId) + expect(await roomManager.hasRoom(room)).toBe(true) + const users = await roomManager.getRoomUsers(room) expect(users).toHaveLength(1) expect(users[0].socketId).toBe(socketId) }) @@ -252,9 +254,10 @@ describe('Socket Server Index Integration', () => { const socketId = 'test-socket-123' const workflowId = 'test-workflow-456' + const room = workflowRoom(workflowId) const presence = { userId: 'user-123', - workflowId, + room, userName: 'Test User', socketId, joinedAt: Date.now(), @@ -262,9 +265,9 @@ describe('Socket Server Index Integration', () => { role: 'admin', } - await roomManager.addUserToRoom(workflowId, socketId, presence) + await roomManager.addUserToRoom(room, socketId, presence) - expect(await roomManager.getWorkflowIdForSocket(socketId)).toBe(workflowId) + expect(await roomManager.getRoomForSocket(socketId, ROOM_TYPES.WORKFLOW)).toEqual(room) const session = await roomManager.getUserSession(socketId) expect(session).toBeDefined() expect(session?.userId).toBe('user-123') @@ -274,9 +277,10 @@ describe('Socket Server Index Integration', () => { const workflowId = 'test-workflow-789' const socketId = 'test-socket-789' + const room = workflowRoom(workflowId) const presence = { userId: 'user-789', - workflowId, + room, userName: 'Test User', socketId, joinedAt: Date.now(), @@ -284,16 +288,16 @@ describe('Socket Server Index Integration', () => { role: 'admin', } - await roomManager.addUserToRoom(workflowId, socketId, presence) + await roomManager.addUserToRoom(room, socketId, presence) - expect(await roomManager.hasWorkflowRoom(workflowId)).toBe(true) + expect(await roomManager.hasRoom(room)).toBe(true) // Remove user - await roomManager.removeUserFromRoom(socketId) + await roomManager.removeUserFromRoom(room, socketId) // Room should be cleaned up since it's now empty - expect(await roomManager.hasWorkflowRoom(workflowId)).toBe(false) - expect(await roomManager.getWorkflowIdForSocket(socketId)).toBeNull() + expect(await roomManager.hasRoom(room)).toBe(false) + expect(await roomManager.getRoomForSocket(socketId, ROOM_TYPES.WORKFLOW)).toBeNull() }) }) @@ -324,7 +328,7 @@ describe('Socket Server Index Integration', () => { expect(typeof roomManager.addUserToRoom).toBe('function') expect(typeof roomManager.removeUserFromRoom).toBe('function') - expect(typeof roomManager.handleWorkflowDeletion).toBe('function') + expect(typeof roomManager.removeSocketFromAllRooms).toBe('function') expect(typeof roomManager.broadcastPresenceUpdate).toBe('function') }) }) diff --git a/apps/realtime/src/rooms/index.ts b/apps/realtime/src/rooms/index.ts index 8067fc1b215..29157aa4581 100644 --- a/apps/realtime/src/rooms/index.ts +++ b/apps/realtime/src/rooms/index.ts @@ -1,3 +1,4 @@ export { MemoryRoomManager } from '@/rooms/memory-manager' export { RedisRoomManager } from '@/rooms/redis-manager' -export type { IRoomManager, UserPresence, UserSession, WorkflowRoom } from '@/rooms/types' +export type { IRoomManager, RoomState, UserPresence, UserSession } from '@/rooms/types' +export { WorkflowRoomService, workflowRoom } from '@/rooms/workflow-room-service' diff --git a/apps/realtime/src/rooms/memory-manager.test.ts b/apps/realtime/src/rooms/memory-manager.test.ts new file mode 100644 index 00000000000..d5d2512a2d8 --- /dev/null +++ b/apps/realtime/src/rooms/memory-manager.test.ts @@ -0,0 +1,175 @@ +/** + * Multi-room semantics for the room manager. Exercises the invariants the + * single-room → multi-room migration must preserve: a socket in two rooms, + * refcounted session cleanup, presence isolation, and full-disconnect cleanup. + * + * @vitest-environment node + */ +import { ROOM_TYPES, type RoomRef } from '@sim/realtime-protocol/rooms' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { MemoryRoomManager } from '@/rooms/memory-manager' +import type { UserPresence } from '@/rooms/types' + +function fakeIo(liveSocketIds: string[] = []) { + const emit = vi.fn() + return { + emit, + io: { + to: vi.fn().mockReturnValue({ emit }), + in: vi.fn().mockReturnValue({ + fetchSockets: vi.fn().mockResolvedValue(liveSocketIds.map((id) => ({ id }))), + }), + } as never, + } +} + +function presence(room: RoomRef, socketId: string, userId: string): UserPresence { + return { + userId, + room, + userName: `user-${userId}`, + socketId, + joinedAt: Date.now(), + lastActivity: Date.now(), + role: 'admin', + } +} + +const WORKFLOW: RoomRef = { type: ROOM_TYPES.WORKFLOW, id: 'wf-1' } +const FILES: RoomRef = { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-1' } + +describe('MemoryRoomManager multi-room', () => { + let manager: MemoryRoomManager + + beforeEach(async () => { + manager = new MemoryRoomManager(fakeIo().io) + await manager.initialize() + }) + + it('tracks a single socket in two rooms of different types', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + const rooms = await manager.getRoomsForSocket('socket-1') + expect(rooms).toHaveLength(2) + expect(rooms).toContainEqual(WORKFLOW) + expect(rooms).toContainEqual(FILES) + + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKFLOW)).toEqual(WORKFLOW) + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKSPACE_FILES)).toEqual(FILES) + }) + + it('keeps the shared session alive when leaving one of two rooms (refcount)', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + const removed = await manager.removeUserFromRoom(WORKFLOW, 'socket-1') + expect(removed).toBe(true) + + // The files room and the shared session must survive. + expect(await manager.hasRoom(WORKFLOW)).toBe(false) + expect(await manager.hasRoom(FILES)).toBe(true) + expect(await manager.getUserSession('socket-1')).not.toBeNull() + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKFLOW)).toBeNull() + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKSPACE_FILES)).toEqual(FILES) + }) + + it('drops the shared session only when the last room is left', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + await manager.removeUserFromRoom(WORKFLOW, 'socket-1') + expect(await manager.getUserSession('socket-1')).not.toBeNull() + + await manager.removeUserFromRoom(FILES, 'socket-1') + expect(await manager.getUserSession('socket-1')).toBeNull() + expect(await manager.getRoomsForSocket('socket-1')).toHaveLength(0) + }) + + it('isolates presence between rooms of different types', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + expect(await manager.getRoomUsers(WORKFLOW)).toHaveLength(1) + expect(await manager.getRoomUsers(FILES)).toHaveLength(2) + }) + + it('removes a socket from every room on disconnect and reports them', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + const removed = await manager.removeSocketFromAllRooms('socket-1') + expect(removed).toHaveLength(2) + expect(removed).toContainEqual(WORKFLOW) + expect(removed).toContainEqual(FILES) + + expect(await manager.hasRoom(WORKFLOW)).toBe(false) + // The files room still has socket-2. + expect(await manager.getRoomUsers(FILES)).toHaveLength(1) + expect(await manager.getUserSession('socket-1')).toBeNull() + }) + + it('does not clobber another type when two sockets share a room', async () => { + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + await manager.removeUserFromRoom(FILES, 'socket-1') + expect(await manager.hasRoom(FILES)).toBe(true) + expect(await manager.getUserSession('socket-2')).not.toBeNull() + }) + + it('ignores removal of a room the socket is not in (id-guarded)', async () => { + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + // Removing a workflow room the socket never joined must be a no-op — it must + // not wipe the files mapping or the shared session. + const removed = await manager.removeUserFromRoom(WORKFLOW, 'socket-1') + expect(removed).toBe(false) + expect(await manager.hasRoom(FILES)).toBe(true) + expect(await manager.getUserSession('socket-1')).not.toBeNull() + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKSPACE_FILES)).toEqual(FILES) + }) + + it('broadcasts presence on the room-type-specific event name', async () => { + const { emit, io } = fakeIo() + const m = new MemoryRoomManager(io) + await m.initialize() + await m.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + await m.broadcastPresenceUpdate(FILES) + expect(emit).toHaveBeenCalledWith('workspace-files:presence-update', expect.any(Array)) + + await m.broadcastPresenceUpdate(WORKFLOW) + expect(emit).toHaveBeenCalledWith('presence-update', expect.any(Array)) + }) + + it('omits an excluded socket from the presence broadcast (disconnect ghost guard)', async () => { + const { emit, io } = fakeIo() + const m = new MemoryRoomManager(io) + await m.initialize() + await m.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await m.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + // Broadcast as if socket-1 is disconnecting: even though its presence entry is + // still present, it must not appear in the emitted list. + await m.broadcastPresenceUpdate(FILES, 'socket-1') + const emitted = emit.mock.calls.at(-1)?.[1] as Array<{ socketId: string }> + expect(emitted.map((u) => u.socketId)).toEqual(['socket-2']) + }) + + it('never emits a presence entry whose socket is no longer live (ghost guard)', async () => { + // Only socket-2 is a live Socket.IO member; socket-1 is an orphaned entry that + // outlived a failed removal. + const { emit, io } = fakeIo(['socket-2']) + const m = new MemoryRoomManager(io) + await m.initialize() + await m.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await m.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + await m.broadcastPresenceUpdate(FILES) + const emitted = emit.mock.calls.at(-1)?.[1] as Array<{ socketId: string }> + expect(emitted.map((u) => u.socketId)).toEqual(['socket-2']) + }) +}) diff --git a/apps/realtime/src/rooms/memory-manager.ts b/apps/realtime/src/rooms/memory-manager.ts index a032e785bb5..0e25921a47e 100644 --- a/apps/realtime/src/rooms/memory-manager.ts +++ b/apps/realtime/src/rooms/memory-manager.ts @@ -1,16 +1,30 @@ import { createLogger } from '@sim/logger' +import { + presenceEventName, + type RoomRef, + type RoomType, + roomName, +} from '@sim/realtime-protocol/rooms' import type { Server } from 'socket.io' -import type { IRoomManager, UserPresence, UserSession, WorkflowRoom } from '@/rooms/types' +import { filterVisiblePresence } from '@/rooms/presence-visibility' +import type { IRoomManager, RoomState, UserPresence, UserSession } from '@/rooms/types' const logger = createLogger('MemoryRoomManager') +/** Stable string key for a room in the local maps (distinct from the Socket.IO room name). */ +function roomKey(room: RoomRef): string { + return `${room.type}:${room.id}` +} + /** - * In-memory room manager for single-pod deployments - * Used as fallback when REDIS_URL is not configured + * In-memory room manager for single-pod deployments. Used when REDIS_URL is not + * configured. Domain-neutral: keyed by {@link RoomRef}, supports a socket in + * multiple rooms (one per {@link RoomType}). */ export class MemoryRoomManager implements IRoomManager { - private workflowRooms = new Map() - private socketToWorkflow = new Map() + private rooms = new Map() + /** socketId -> (roomType -> roomId) */ + private socketRooms = new Map>() private userSessions = new Map() private _io: Server @@ -31,228 +45,151 @@ export class MemoryRoomManager implements IRoomManager { } async shutdown(): Promise { - this.workflowRooms.clear() - this.socketToWorkflow.clear() + this.rooms.clear() + this.socketRooms.clear() this.userSessions.clear() logger.info('MemoryRoomManager shutdown complete') } - async addUserToRoom(workflowId: string, socketId: string, presence: UserPresence): Promise { - // Create room if it doesn't exist - if (!this.workflowRooms.has(workflowId)) { - this.workflowRooms.set(workflowId, { - workflowId, - users: new Map(), - lastModified: Date.now(), - activeConnections: 0, - }) + async addUserToRoom(room: RoomRef, socketId: string, presence: UserPresence): Promise { + const key = roomKey(room) + let state = this.rooms.get(key) + if (!state) { + state = { room, users: new Map(), lastModified: Date.now(), activeConnections: 0 } + this.rooms.set(key, state) } - const room = this.workflowRooms.get(workflowId)! - room.users.set(socketId, presence) - room.activeConnections++ - room.lastModified = Date.now() + state.users.set(socketId, presence) + state.activeConnections++ + state.lastModified = Date.now() - // Map socket to workflow - this.socketToWorkflow.set(socketId, workflowId) + let socketRoomMap = this.socketRooms.get(socketId) + if (!socketRoomMap) { + socketRoomMap = new Map() + this.socketRooms.set(socketId, socketRoomMap) + } + socketRoomMap.set(room.type, room.id) - // Store session this.userSessions.set(socketId, { userId: presence.userId, userName: presence.userName, avatarUrl: presence.avatarUrl, }) - logger.debug(`Added user ${presence.userId} to workflow ${workflowId} (socket: ${socketId})`) + logger.debug(`Added user ${presence.userId} to room ${key} (socket: ${socketId})`) } - async removeUserFromRoom(socketId: string, _workflowIdHint?: string): Promise { - const workflowId = this.socketToWorkflow.get(socketId) + async removeUserFromRoom(room: RoomRef, socketId: string): Promise { + const key = roomKey(room) + const state = this.rooms.get(key) + let existed = false - if (!workflowId) { - return null + if (state?.users.has(socketId)) { + existed = true + state.users.delete(socketId) + state.activeConnections = Math.max(0, state.activeConnections - 1) + if (state.users.size === 0) { + this.rooms.delete(key) + logger.info(`Cleaned up empty room: ${key}`) + } } - const room = this.workflowRooms.get(workflowId) - if (room) { - room.users.delete(socketId) - room.activeConnections = Math.max(0, room.activeConnections - 1) - - // Clean up empty rooms - if (room.activeConnections === 0) { - this.workflowRooms.delete(workflowId) - logger.info(`Cleaned up empty workflow room: ${workflowId}`) + const socketRoomMap = this.socketRooms.get(socketId) + if (socketRoomMap && socketRoomMap.get(room.type) === room.id) { + socketRoomMap.delete(room.type) + // Drop the shared session only when the socket has left its last room. + if (socketRoomMap.size === 0) { + this.socketRooms.delete(socketId) + this.userSessions.delete(socketId) } } - this.socketToWorkflow.delete(socketId) + return existed + } + + async removeSocketFromAllRooms(socketId: string): Promise { + const socketRoomMap = this.socketRooms.get(socketId) + if (!socketRoomMap || socketRoomMap.size === 0) { + this.userSessions.delete(socketId) + return [] + } + + const rooms: RoomRef[] = Array.from(socketRoomMap.entries()).map(([type, id]) => ({ type, id })) + for (const room of rooms) { + await this.removeUserFromRoom(room, socketId) + } + // Belt-and-suspenders: ensure session is gone even if the map drifted. + this.socketRooms.delete(socketId) this.userSessions.delete(socketId) + return rooms + } - logger.debug(`Removed socket ${socketId} from workflow ${workflowId}`) - return workflowId + async getRoomsForSocket(socketId: string): Promise { + const socketRoomMap = this.socketRooms.get(socketId) + if (!socketRoomMap) return [] + return Array.from(socketRoomMap.entries()).map(([type, id]) => ({ type, id })) } - async getWorkflowIdForSocket(socketId: string): Promise { - return this.socketToWorkflow.get(socketId) ?? null + async getRoomForSocket(socketId: string, type: RoomType): Promise { + const id = this.socketRooms.get(socketId)?.get(type) + return id ? { type, id } : null } async getUserSession(socketId: string): Promise { return this.userSessions.get(socketId) ?? null } - async getWorkflowUsers(workflowId: string): Promise { - const room = this.workflowRooms.get(workflowId) - if (!room) return [] - return Array.from(room.users.values()) + async getRoomUsers(room: RoomRef): Promise { + const state = this.rooms.get(roomKey(room)) + if (!state) return [] + return Array.from(state.users.values()) } - async hasWorkflowRoom(workflowId: string): Promise { - return this.workflowRooms.has(workflowId) + async hasRoom(room: RoomRef): Promise { + return this.rooms.has(roomKey(room)) } async updateUserActivity( - workflowId: string, + room: RoomRef, socketId: string, updates: Partial> ): Promise { - const room = this.workflowRooms.get(workflowId) - if (!room) return + const presence = this.rooms.get(roomKey(room))?.users.get(socketId) + if (!presence) return - const presence = room.users.get(socketId) - if (presence) { - if (updates.cursor !== undefined) presence.cursor = updates.cursor - if (updates.selection !== undefined) presence.selection = updates.selection - presence.lastActivity = updates.lastActivity ?? Date.now() - } + if (updates.cursor !== undefined) presence.cursor = updates.cursor + if (updates.selection !== undefined) presence.selection = updates.selection + presence.lastActivity = updates.lastActivity ?? Date.now() } - async updateRoomLastModified(workflowId: string): Promise { - const room = this.workflowRooms.get(workflowId) - if (room) { - room.lastModified = Date.now() - } + async updateRoomLastModified(room: RoomRef): Promise { + const state = this.rooms.get(roomKey(room)) + if (state) state.lastModified = Date.now() } - async broadcastPresenceUpdate(workflowId: string): Promise { - const users = await this.getWorkflowUsers(workflowId) - this._io.to(workflowId).emit('presence-update', users) + async broadcastPresenceUpdate(room: RoomRef, excludeSocketId?: string): Promise { + const users = await this.getRoomUsers(room) + const visible = await filterVisiblePresence(this._io, room, users, excludeSocketId) + this._io.to(roomName(room)).emit(presenceEventName(room.type), visible) } - emitToWorkflow(workflowId: string, event: string, payload: T): void { - this._io.to(workflowId).emit(event, payload) + emitToRoom(room: RoomRef, event: string, payload: T): void { + this._io.to(roomName(room)).emit(event, payload) } - async getUniqueUserCount(workflowId: string): Promise { - const room = this.workflowRooms.get(workflowId) - if (!room) return 0 - + async getUniqueUserCount(room: RoomRef): Promise { + const state = this.rooms.get(roomKey(room)) + if (!state) return 0 const uniqueUsers = new Set() - room.users.forEach((presence) => { - uniqueUsers.add(presence.userId) - }) - + state.users.forEach((presence) => uniqueUsers.add(presence.userId)) return uniqueUsers.size } async getTotalActiveConnections(): Promise { let total = 0 - for (const room of this.workflowRooms.values()) { - total += room.activeConnections + for (const state of this.rooms.values()) { + total += state.activeConnections } return total } - - async handleWorkflowDeletion(workflowId: string): Promise { - logger.info(`Handling workflow deletion notification for ${workflowId}`) - - const room = this.workflowRooms.get(workflowId) - if (!room) { - logger.debug(`No active room found for deleted workflow ${workflowId}`) - return - } - - this._io.to(workflowId).emit('workflow-deleted', { - workflowId, - message: 'This workflow has been deleted', - timestamp: Date.now(), - }) - - const socketsToDisconnect: string[] = [] - room.users.forEach((_presence, socketId) => { - socketsToDisconnect.push(socketId) - }) - - for (const socketId of socketsToDisconnect) { - const socket = this._io.sockets.sockets.get(socketId) - if (socket) { - socket.leave(workflowId) - logger.debug(`Disconnected socket ${socketId} from deleted workflow ${workflowId}`) - } - await this.removeUserFromRoom(socketId) - } - - this.workflowRooms.delete(workflowId) - logger.info( - `Cleaned up workflow room ${workflowId} after deletion (${socketsToDisconnect.length} users disconnected)` - ) - } - - async handleWorkflowRevert(workflowId: string, timestamp: number): Promise { - logger.info(`Handling workflow revert notification for ${workflowId}`) - - const room = this.workflowRooms.get(workflowId) - if (!room) { - logger.debug(`No active room found for reverted workflow ${workflowId}`) - return - } - - this._io.to(workflowId).emit('workflow-reverted', { - workflowId, - message: 'Workflow has been reverted to deployed state', - timestamp, - }) - - room.lastModified = timestamp - - logger.info(`Notified ${room.users.size} users about workflow revert: ${workflowId}`) - } - - async handleWorkflowUpdate(workflowId: string): Promise { - logger.info(`Handling workflow update notification for ${workflowId}`) - - const room = this.workflowRooms.get(workflowId) - if (!room) { - logger.debug(`No active room found for updated workflow ${workflowId}`) - return - } - - const timestamp = Date.now() - - this._io.to(workflowId).emit('workflow-updated', { - workflowId, - message: 'Workflow has been updated externally', - timestamp, - }) - - room.lastModified = timestamp - - logger.info(`Notified ${room.users.size} users about workflow update: ${workflowId}`) - } - - async handleWorkflowDeployed(workflowId: string): Promise { - logger.info(`Handling workflow deployed notification for ${workflowId}`) - - const room = this.workflowRooms.get(workflowId) - if (!room) { - logger.debug(`No active room found for deployed workflow ${workflowId}`) - return - } - - this._io.to(workflowId).emit('workflow-deployed', { - workflowId, - timestamp: Date.now(), - }) - - logger.info(`Notified ${room.users.size} users about workflow deployment change: ${workflowId}`) - } } diff --git a/apps/realtime/src/rooms/presence-visibility.ts b/apps/realtime/src/rooms/presence-visibility.ts new file mode 100644 index 00000000000..4680a581964 --- /dev/null +++ b/apps/realtime/src/rooms/presence-visibility.ts @@ -0,0 +1,36 @@ +import { type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import type { Server } from 'socket.io' + +/** + * Filters a room's stored presence down to what should actually be broadcast: + * drops `excludeSocketId` (e.g. a socket mid-disconnect that is still connected), + * then reconciles against the live Socket.IO membership so an entry orphaned by a + * failed removal (the room hashes have no TTL) is never emitted as a ghost. + * + * Fail-safe: if the liveness lookup throws, or returns an empty set while we still + * hold presence entries (a cross-pod `fetchSockets` timeout, not a truly empty + * room), we emit the unfiltered list rather than hide everyone — a transient + * ghost self-corrects on the next broadcast, but hiding live collaborators would + * be a worse, visible failure. + */ +export async function filterVisiblePresence( + io: Server, + room: RoomRef, + users: T[], + excludeSocketId?: string +): Promise { + const candidates = excludeSocketId + ? users.filter((user) => user.socketId !== excludeSocketId) + : users + + try { + const liveSockets = await io.in(roomName(room)).fetchSockets() + if (liveSockets.length === 0) { + return candidates + } + const liveIds = new Set(liveSockets.map((socket) => socket.id)) + return candidates.filter((user) => liveIds.has(user.socketId)) + } catch { + return candidates + } +} diff --git a/apps/realtime/src/rooms/redis-manager.ts b/apps/realtime/src/rooms/redis-manager.ts index 0e6b3eadf2b..2de65bc14c5 100644 --- a/apps/realtime/src/rooms/redis-manager.ts +++ b/apps/realtime/src/rooms/redis-manager.ts @@ -1,86 +1,102 @@ import { createLogger } from '@sim/logger' +import { + presenceEventName, + type RoomRef, + type RoomType, + roomName, +} from '@sim/realtime-protocol/rooms' import { createClient, type RedisClientType } from 'redis' import type { Server } from 'socket.io' +import { filterVisiblePresence } from '@/rooms/presence-visibility' import type { IRoomManager, UserPresence, UserSession } from '@/rooms/types' const logger = createLogger('RedisRoomManager') +/** + * Redis key scheme (all room-scoped keys are prefixed by room type): + * {type}:{id}:users HASH socketId -> UserPresence JSON (room membership) + * {type}:{id}:meta HASH room metadata (lastModified) + * socket:{sid}:rooms HASH roomType -> roomId (the socket's rooms, one per type) + * socket:{sid}:session HASH userId/userName/avatarUrl (shared across the socket's rooms) + * + * Workflow rooms keep their historical `workflow:{id}:users`/`:meta` keys (the + * type prefix IS `workflow`), so no presence-state migration is needed for them. + */ const KEYS = { - workflowUsers: (wfId: string) => `workflow:${wfId}:users`, - workflowMeta: (wfId: string) => `workflow:${wfId}:meta`, - socketWorkflow: (socketId: string) => `socket:${socketId}:workflow`, + roomUsers: (room: RoomRef) => `${room.type}:${room.id}:users`, + roomMeta: (room: RoomRef) => `${room.type}:${room.id}:meta`, + socketRooms: (socketId: string) => `socket:${socketId}:rooms`, socketSession: (socketId: string) => `socket:${socketId}:session`, - socketPresenceWorkflow: (socketId: string) => `socket:${socketId}:presence-workflow`, } as const -const SOCKET_KEY_TTL = 3600 -const SOCKET_PRESENCE_WORKFLOW_KEY_TTL = 24 * 60 * 60 +/** TTL for the socket's room-set. Long enough that an idle-but-connected socket is not evicted. */ +const SOCKET_ROOMS_TTL = 24 * 60 * 60 +/** TTL for the shared session key; refreshed on every activity update. */ +const SESSION_TTL = 60 * 60 /** - * Lua script for atomic user removal from room. - * Returns workflowId if user was removed, null otherwise. - * Handles room cleanup atomically to prevent race conditions. + * Atomic single-room removal. Removes a socket from one room's presence, drops + * the room from the socket's room-set, and — critically — deletes the SHARED + * session key only when the socket has left its LAST room (otherwise a leave from + * one room would break the socket's other rooms). Cleans up empty room state. + * + * KEYS: [socketRooms, socketSession, roomUsers, roomMeta] + * ARGV: [roomType, socketId, roomId] + * Returns 1 if the socket was a member of the room, else 0. */ -const REMOVE_USER_SCRIPT = ` -local socketWorkflowKey = KEYS[1] +const REMOVE_ROOM_SCRIPT = ` +local socketRoomsKey = KEYS[1] local socketSessionKey = KEYS[2] -local socketPresenceWorkflowKey = KEYS[3] -local workflowUsersPrefix = ARGV[1] -local workflowMetaPrefix = ARGV[2] -local socketId = ARGV[3] -local workflowIdHint = ARGV[4] - -local workflowId = redis.call('GET', socketWorkflowKey) -if not workflowId then - workflowId = redis.call('GET', socketPresenceWorkflowKey) -end - -if not workflowId and workflowIdHint ~= '' then - workflowId = workflowIdHint +local roomUsersKey = KEYS[3] +local roomMetaKey = KEYS[4] +local roomType = ARGV[1] +local socketId = ARGV[2] +local roomId = ARGV[3] + +local removed = redis.call('HDEL', roomUsersKey, socketId) + +-- Only drop the socket's mapping for this type if it points at THIS room, so +-- removing a room the socket isn't in can't wipe a different room's mapping or +-- spuriously trigger the last-room session cleanup (mirrors the memory manager). +if redis.call('HGET', socketRoomsKey, roomType) == roomId then + redis.call('HDEL', socketRoomsKey, roomType) + if redis.call('HLEN', socketRoomsKey) == 0 then + redis.call('DEL', socketRoomsKey, socketSessionKey) + end end -if not workflowId then - return nil +if redis.call('HLEN', roomUsersKey) == 0 then + redis.call('DEL', roomUsersKey, roomMetaKey) end -local workflowUsersKey = workflowUsersPrefix .. workflowId .. ':users' -local workflowMetaKey = workflowMetaPrefix .. workflowId .. ':meta' - -redis.call('HDEL', workflowUsersKey, socketId) -redis.call('DEL', socketWorkflowKey, socketSessionKey, socketPresenceWorkflowKey) - -local remaining = redis.call('HLEN', workflowUsersKey) -if remaining == 0 then - redis.call('DEL', workflowUsersKey, workflowMetaKey) -end - -return workflowId +return removed ` /** - * Lua script for atomic user activity update. - * Performs read-modify-write atomically to prevent lost updates. - * Also refreshes TTL on socket keys to prevent expiry during long sessions. + * Atomic presence-activity update (read-modify-write) that also refreshes the + * socket key TTLs to keep a long-lived session alive. + * + * KEYS: [roomUsers, socketRooms, socketSession] + * ARGV: [socketId, cursorJson, selectionJson, lastActivity, roomsTtl, sessionTtl] + * Returns 1 if the socket had presence in the room, else 0. */ const UPDATE_ACTIVITY_SCRIPT = ` -local workflowUsersKey = KEYS[1] -local socketWorkflowKey = KEYS[2] +local roomUsersKey = KEYS[1] +local socketRoomsKey = KEYS[2] local socketSessionKey = KEYS[3] -local socketPresenceWorkflowKey = KEYS[4] local socketId = ARGV[1] local cursorJson = ARGV[2] local selectionJson = ARGV[3] local lastActivity = ARGV[4] -local ttl = tonumber(ARGV[5]) -local presenceWorkflowTtl = tonumber(ARGV[6]) +local roomsTtl = tonumber(ARGV[5]) +local sessionTtl = tonumber(ARGV[6]) -local existingJson = redis.call('HGET', workflowUsersKey, socketId) +local existingJson = redis.call('HGET', roomUsersKey, socketId) if not existingJson then return 0 end local existing = cjson.decode(existingJson) - if cursorJson ~= '' then existing.cursor = cjson.decode(cursorJson) end @@ -89,44 +105,39 @@ if selectionJson ~= '' then end existing.lastActivity = tonumber(lastActivity) -redis.call('HSET', workflowUsersKey, socketId, cjson.encode(existing)) -redis.call('EXPIRE', socketWorkflowKey, ttl) -redis.call('EXPIRE', socketSessionKey, ttl) -redis.call('EXPIRE', socketPresenceWorkflowKey, presenceWorkflowTtl) +redis.call('HSET', roomUsersKey, socketId, cjson.encode(existing)) +redis.call('EXPIRE', socketRoomsKey, roomsTtl) +redis.call('EXPIRE', socketSessionKey, sessionTtl) return 1 ` /** - * Redis-backed room manager for multi-pod deployments. - * Uses Lua scripts for atomic operations to prevent race conditions. + * Redis-backed room manager for multi-pod deployments. Domain-neutral: keyed by + * {@link RoomRef}, supports a socket in multiple rooms (one per {@link RoomType}). + * Uses Lua scripts for atomic multi-key operations. */ export class RedisRoomManager implements IRoomManager { private redis: RedisClientType private _io: Server private isConnected = false - private removeUserScriptSha: string | null = null + private removeRoomScriptSha: string | null = null private updateActivityScriptSha: string | null = null constructor(io: Server, redisUrl: string) { this._io = io - this.redis = createClient({ - url: redisUrl, - }) + this.redis = createClient({ url: redisUrl }) this.redis.on('error', (err) => { logger.error('Redis client error:', err) }) - this.redis.on('reconnecting', () => { logger.warn('Redis client reconnecting...') this.isConnected = false }) - this.redis.on('ready', () => { logger.info('Redis client ready') this.isConnected = true }) - this.redis.on('end', () => { logger.warn('Redis client connection closed') this.isConnected = false @@ -148,8 +159,7 @@ export class RedisRoomManager implements IRoomManager { await this.redis.connect() this.isConnected = true - // Pre-load Lua scripts for better performance - this.removeUserScriptSha = await this.redis.scriptLoad(REMOVE_USER_SCRIPT) + this.removeRoomScriptSha = await this.redis.scriptLoad(REMOVE_ROOM_SCRIPT) this.updateActivityScriptSha = await this.redis.scriptLoad(UPDATE_ACTIVITY_SCRIPT) logger.info('RedisRoomManager connected to Redis and scripts loaded') @@ -161,7 +171,6 @@ export class RedisRoomManager implements IRoomManager { async shutdown(): Promise { if (!this.isConnected) return - try { await this.redis.quit() this.isConnected = false @@ -171,93 +180,102 @@ export class RedisRoomManager implements IRoomManager { } } - async addUserToRoom(workflowId: string, socketId: string, presence: UserPresence): Promise { + async addUserToRoom(room: RoomRef, socketId: string, presence: UserPresence): Promise { try { const pipeline = this.redis.multi() - pipeline.hSet(KEYS.workflowUsers(workflowId), socketId, JSON.stringify(presence)) - pipeline.hSet(KEYS.workflowMeta(workflowId), 'lastModified', Date.now().toString()) - pipeline.set(KEYS.socketWorkflow(socketId), workflowId) - pipeline.expire(KEYS.socketWorkflow(socketId), SOCKET_KEY_TTL) - pipeline.set(KEYS.socketPresenceWorkflow(socketId), workflowId) - pipeline.expire(KEYS.socketPresenceWorkflow(socketId), SOCKET_PRESENCE_WORKFLOW_KEY_TTL) + pipeline.hSet(KEYS.roomUsers(room), socketId, JSON.stringify(presence)) + pipeline.hSet(KEYS.roomMeta(room), 'lastModified', Date.now().toString()) + pipeline.hSet(KEYS.socketRooms(socketId), room.type, room.id) + pipeline.expire(KEYS.socketRooms(socketId), SOCKET_ROOMS_TTL) pipeline.hSet(KEYS.socketSession(socketId), { userId: presence.userId, userName: presence.userName, avatarUrl: presence.avatarUrl || '', }) - pipeline.expire(KEYS.socketSession(socketId), SOCKET_KEY_TTL) + pipeline.expire(KEYS.socketSession(socketId), SESSION_TTL) const results = await pipeline.exec() - // Check if any command failed const failed = results.some((result) => result instanceof Error) if (failed) { - logger.error(`Pipeline partially failed when adding user to room`, { workflowId, socketId }) + logger.error('Pipeline partially failed when adding user to room', { + room, + socketId, + }) throw new Error('Failed to store user session data in Redis') } - logger.debug(`Added user ${presence.userId} to workflow ${workflowId} (socket: ${socketId})`) + logger.debug(`Added user ${presence.userId} to room ${room.type}:${room.id} (${socketId})`) } catch (error) { - logger.error(`Failed to add user to room: ${socketId} -> ${workflowId}`, error) + logger.error(`Failed to add user to room: ${socketId} -> ${room.type}:${room.id}`, error) throw error } } - async removeUserFromRoom( - socketId: string, - workflowIdHint?: string, - retried = false - ): Promise { - if (!this.removeUserScriptSha) { + async removeUserFromRoom(room: RoomRef, socketId: string, retried = false): Promise { + if (!this.removeRoomScriptSha) { logger.error('removeUserFromRoom called before initialize()') - return null + return false } try { - const workflowId = await this.redis.evalSha(this.removeUserScriptSha, { + const removed = await this.redis.evalSha(this.removeRoomScriptSha, { keys: [ - KEYS.socketWorkflow(socketId), + KEYS.socketRooms(socketId), KEYS.socketSession(socketId), - KEYS.socketPresenceWorkflow(socketId), + KEYS.roomUsers(room), + KEYS.roomMeta(room), ], - arguments: ['workflow:', 'workflow:', socketId, workflowIdHint ?? ''], + arguments: [room.type, socketId, room.id], }) - - if (typeof workflowId === 'string' && workflowId.length > 0) { - logger.debug(`Removed socket ${socketId} from workflow ${workflowId}`) - return workflowId - } - - return null + return typeof removed === 'number' ? removed > 0 : Number(removed) > 0 } catch (error) { if ((error as Error).message?.includes('NOSCRIPT') && !retried) { logger.warn('Lua script not found, reloading...') - this.removeUserScriptSha = await this.redis.scriptLoad(REMOVE_USER_SCRIPT) - return this.removeUserFromRoom(socketId, workflowIdHint, true) + this.removeRoomScriptSha = await this.redis.scriptLoad(REMOVE_ROOM_SCRIPT) + return this.removeUserFromRoom(room, socketId, true) } - logger.error(`Failed to remove user from room: ${socketId}`, error) - return null + logger.error(`Failed to remove socket ${socketId} from room ${room.type}:${room.id}`, error) + return false } } - async getWorkflowIdForSocket(socketId: string): Promise { - const workflowId = await this.redis.get(KEYS.socketWorkflow(socketId)) - if (workflowId) { - return workflowId + async removeSocketFromAllRooms(socketId: string): Promise { + const rooms = await this.getRoomsForSocket(socketId) + if (rooms.length === 0) { + // Nothing tracked (already cleaned up or TTL-expired); ensure session is gone. + await this.redis.del(KEYS.socketSession(socketId)).catch(() => {}) + return [] } - return this.redis.get(KEYS.socketPresenceWorkflow(socketId)) + const removed: RoomRef[] = [] + for (const room of rooms) { + const wasMember = await this.removeUserFromRoom(room, socketId) + if (wasMember) removed.push(room) + } + return removed } - async getUserSession(socketId: string): Promise { + async getRoomsForSocket(socketId: string): Promise { try { - const session = await this.redis.hGetAll(KEYS.socketSession(socketId)) + const entries = await this.redis.hGetAll(KEYS.socketRooms(socketId)) + return Object.entries(entries).map(([type, id]) => ({ type: type as RoomType, id })) + } catch (error) { + logger.error(`Failed to get rooms for socket ${socketId}:`, error) + return [] + } + } - if (!session.userId) { - return null - } + async getRoomForSocket(socketId: string, type: RoomType): Promise { + const id = await this.redis.hGet(KEYS.socketRooms(socketId), type) + return id ? { type, id } : null + } + async getUserSession(socketId: string): Promise { + try { + const session = await this.redis.hGetAll(KEYS.socketSession(socketId)) + if (!session.userId) return null return { userId: session.userId, userName: session.userName, @@ -269,9 +287,9 @@ export class RedisRoomManager implements IRoomManager { } } - async getWorkflowUsers(workflowId: string): Promise { + async getRoomUsers(room: RoomRef): Promise { try { - const users = await this.redis.hGetAll(KEYS.workflowUsers(workflowId)) + const users = await this.redis.hGetAll(KEYS.roomUsers(room)) return Object.entries(users) .map(([socketId, json]) => { try { @@ -283,18 +301,18 @@ export class RedisRoomManager implements IRoomManager { }) .filter((u): u is UserPresence => u !== null) } catch (error) { - logger.error(`Failed to get workflow users for ${workflowId}:`, error) + logger.error(`Failed to get room users for ${room.type}:${room.id}:`, error) return [] } } - async hasWorkflowRoom(workflowId: string): Promise { - const exists = await this.redis.exists(KEYS.workflowUsers(workflowId)) + async hasRoom(room: RoomRef): Promise { + const exists = await this.redis.exists(KEYS.roomUsers(room)) return exists > 0 } async updateUserActivity( - workflowId: string, + room: RoomRef, socketId: string, updates: Partial>, retried = false @@ -306,155 +324,48 @@ export class RedisRoomManager implements IRoomManager { try { await this.redis.evalSha(this.updateActivityScriptSha, { - keys: [ - KEYS.workflowUsers(workflowId), - KEYS.socketWorkflow(socketId), - KEYS.socketSession(socketId), - KEYS.socketPresenceWorkflow(socketId), - ], + keys: [KEYS.roomUsers(room), KEYS.socketRooms(socketId), KEYS.socketSession(socketId)], arguments: [ socketId, updates.cursor !== undefined ? JSON.stringify(updates.cursor) : '', updates.selection !== undefined ? JSON.stringify(updates.selection) : '', (updates.lastActivity ?? Date.now()).toString(), - SOCKET_KEY_TTL.toString(), - SOCKET_PRESENCE_WORKFLOW_KEY_TTL.toString(), + SOCKET_ROOMS_TTL.toString(), + SESSION_TTL.toString(), ], }) } catch (error) { if ((error as Error).message?.includes('NOSCRIPT') && !retried) { logger.warn('Lua script not found, reloading...') this.updateActivityScriptSha = await this.redis.scriptLoad(UPDATE_ACTIVITY_SCRIPT) - return this.updateUserActivity(workflowId, socketId, updates, true) + return this.updateUserActivity(room, socketId, updates, true) } logger.error(`Failed to update user activity: ${socketId}`, error) } } - async updateRoomLastModified(workflowId: string): Promise { - await this.redis.hSet(KEYS.workflowMeta(workflowId), 'lastModified', Date.now().toString()) + async updateRoomLastModified(room: RoomRef): Promise { + await this.redis.hSet(KEYS.roomMeta(room), 'lastModified', Date.now().toString()) } - async broadcastPresenceUpdate(workflowId: string): Promise { - const users = await this.getWorkflowUsers(workflowId) - // io.to() with Redis adapter broadcasts to all pods - this._io.to(workflowId).emit('presence-update', users) + async broadcastPresenceUpdate(room: RoomRef, excludeSocketId?: string): Promise { + const users = await this.getRoomUsers(room) + const visible = await filterVisiblePresence(this._io, room, users, excludeSocketId) + // io.to() with the Redis adapter broadcasts to all pods. + this._io.to(roomName(room)).emit(presenceEventName(room.type), visible) } - emitToWorkflow(workflowId: string, event: string, payload: T): void { - this._io.to(workflowId).emit(event, payload) + emitToRoom(room: RoomRef, event: string, payload: T): void { + this._io.to(roomName(room)).emit(event, payload) } - async getUniqueUserCount(workflowId: string): Promise { - const users = await this.getWorkflowUsers(workflowId) - const uniqueUserIds = new Set(users.map((u) => u.userId)) - return uniqueUserIds.size + async getUniqueUserCount(room: RoomRef): Promise { + const users = await this.getRoomUsers(room) + return new Set(users.map((u) => u.userId)).size } async getTotalActiveConnections(): Promise { - // This is more complex with Redis - we'd need to scan all workflow:*:users keys - // For now, just count sockets in this server instance - // The true count would require aggregating across all pods + // Local instance only; the true cross-pod count would require aggregation. return this._io.sockets.sockets.size } - - async handleWorkflowDeletion(workflowId: string): Promise { - logger.info(`Handling workflow deletion notification for ${workflowId}`) - - try { - const users = await this.getWorkflowUsers(workflowId) - if (users.length === 0) { - logger.debug(`No active users found for deleted workflow ${workflowId}`) - return - } - - // Notify all clients across all pods via Redis adapter - this._io.to(workflowId).emit('workflow-deleted', { - workflowId, - message: 'This workflow has been deleted', - timestamp: Date.now(), - }) - - // Use Socket.IO's cross-pod socketsLeave() to remove all sockets from the room - // This works across all pods when using the Redis adapter - await this._io.in(workflowId).socketsLeave(workflowId) - logger.debug(`All sockets left workflow room ${workflowId} via socketsLeave()`) - - // Remove all users from Redis state - for (const user of users) { - await this.removeUserFromRoom(user.socketId, workflowId) - } - - // Clean up room data - await this.redis.del([KEYS.workflowUsers(workflowId), KEYS.workflowMeta(workflowId)]) - - logger.info( - `Cleaned up workflow room ${workflowId} after deletion (${users.length} users disconnected)` - ) - } catch (error) { - logger.error(`Failed to handle workflow deletion for ${workflowId}:`, error) - } - } - - async handleWorkflowRevert(workflowId: string, timestamp: number): Promise { - logger.info(`Handling workflow revert notification for ${workflowId}`) - - const hasRoom = await this.hasWorkflowRoom(workflowId) - if (!hasRoom) { - logger.debug(`No active room found for reverted workflow ${workflowId}`) - return - } - - this._io.to(workflowId).emit('workflow-reverted', { - workflowId, - message: 'Workflow has been reverted to deployed state', - timestamp, - }) - - await this.updateRoomLastModified(workflowId) - - const userCount = await this.getUniqueUserCount(workflowId) - logger.info(`Notified ${userCount} users about workflow revert: ${workflowId}`) - } - - async handleWorkflowUpdate(workflowId: string): Promise { - logger.info(`Handling workflow update notification for ${workflowId}`) - - const hasRoom = await this.hasWorkflowRoom(workflowId) - if (!hasRoom) { - logger.debug(`No active room found for updated workflow ${workflowId}`) - return - } - - const timestamp = Date.now() - - this._io.to(workflowId).emit('workflow-updated', { - workflowId, - message: 'Workflow has been updated externally', - timestamp, - }) - - await this.updateRoomLastModified(workflowId) - - const userCount = await this.getUniqueUserCount(workflowId) - logger.info(`Notified ${userCount} users about workflow update: ${workflowId}`) - } - - async handleWorkflowDeployed(workflowId: string): Promise { - logger.info(`Handling workflow deployed notification for ${workflowId}`) - - const hasRoom = await this.hasWorkflowRoom(workflowId) - if (!hasRoom) { - logger.debug(`No active room found for deployed workflow ${workflowId}`) - return - } - - this._io.to(workflowId).emit('workflow-deployed', { - workflowId, - timestamp: Date.now(), - }) - - const userCount = await this.getUniqueUserCount(workflowId) - logger.info(`Notified ${userCount} users about workflow deployment change: ${workflowId}`) - } } diff --git a/apps/realtime/src/rooms/types.ts b/apps/realtime/src/rooms/types.ts index 9553a427e1e..67cee450832 100644 --- a/apps/realtime/src/rooms/types.ts +++ b/apps/realtime/src/rooms/types.ts @@ -1,11 +1,15 @@ +import type { RoomRef, RoomType } from '@sim/realtime-protocol/rooms' import type { Server } from 'socket.io' /** - * User presence data stored in room state + * User presence data stored in room state. + * + * `room` is the generic room address (see `@sim/realtime-protocol/rooms`). A + * socket may hold presence in more than one room, but only one room per type. */ export interface UserPresence { userId: string - workflowId: string + room: RoomRef userName: string socketId: string tabSessionId?: string @@ -18,7 +22,8 @@ export interface UserPresence { } /** - * User session data (minimal info for quick lookups) + * User session data (minimal info for quick lookups). Shared across all rooms a + * socket is in — keyed by socket, not by room. */ export interface UserSession { userId: string @@ -27,120 +32,94 @@ export interface UserSession { } /** - * Workflow room state + * Room presence state. */ -export interface WorkflowRoom { - workflowId: string +export interface RoomState { + room: RoomRef users: Map lastModified: number activeConnections: number } /** - * Common interface for room managers (in-memory and Redis) - * All methods that access state are async to support Redis operations + * Common interface for room managers (in-memory and Redis). + * + * The manager is domain-neutral: it tracks room membership and presence keyed by + * {@link RoomRef}, and knows nothing about workflows, files, or any specific + * domain. Domain lifecycle concerns (e.g. workflow deletion/deploy broadcasts) + * live in domain services that compose a manager — see `WorkflowRoomService`. + * + * A socket may occupy multiple rooms, at most one per {@link RoomType}. The + * shared session key is dropped only when a socket leaves its last room. + * + * All state-accessing methods are async to support the Redis implementation. */ export interface IRoomManager { readonly io: Server - /** - * Initialize the room manager (connect to Redis, etc.) - */ + /** Initialize the manager (connect to Redis, load scripts, etc.). */ initialize(): Promise - /** - * Whether the room manager is ready to serve requests - */ + /** Whether the manager is ready to serve requests. */ isReady(): boolean - /** - * Clean shutdown - */ + /** Clean shutdown. */ shutdown(): Promise - /** - * Add a user to a workflow room - */ - addUserToRoom(workflowId: string, socketId: string, presence: UserPresence): Promise + /** Add a socket's presence to a room. */ + addUserToRoom(room: RoomRef, socketId: string, presence: UserPresence): Promise /** - * Remove a user from their current room - * Optional workflowIdHint is used when socket mapping keys are missing/expired. - * Returns the workflowId they were in, or null if not in any room. + * Remove a socket from a single room. Returns `true` if it was a member. The + * shared session is dropped only if this was the socket's last room. */ - removeUserFromRoom(socketId: string, workflowIdHint?: string): Promise + removeUserFromRoom(room: RoomRef, socketId: string): Promise /** - * Get the workflow ID for a socket + * Remove a socket from every room it occupies (disconnect). Returns the rooms + * it was in, so the caller can rebroadcast presence per room. */ - getWorkflowIdForSocket(socketId: string): Promise + removeSocketFromAllRooms(socketId: string): Promise - /** - * Get user session data for a socket - */ + /** Every room the socket currently occupies. */ + getRoomsForSocket(socketId: string): Promise + + /** The socket's room of a given type (at most one per type), or `null`. */ + getRoomForSocket(socketId: string, type: RoomType): Promise + + /** Session data for a socket (shared across its rooms). */ getUserSession(socketId: string): Promise - /** - * Get all users in a workflow room - */ - getWorkflowUsers(workflowId: string): Promise + /** All users present in a room. */ + getRoomUsers(room: RoomRef): Promise - /** - * Check if a workflow room exists - */ - hasWorkflowRoom(workflowId: string): Promise + /** Whether a room currently has any presence. */ + hasRoom(room: RoomRef): Promise - /** - * Update user activity (cursor, selection, lastActivity) - */ + /** Update a socket's activity (cursor, selection, lastActivity) within a room. */ updateUserActivity( - workflowId: string, + room: RoomRef, socketId: string, updates: Partial> ): Promise - /** - * Update room's lastModified timestamp - */ - updateRoomLastModified(workflowId: string): Promise + /** Bump a room's lastModified timestamp. */ + updateRoomLastModified(room: RoomRef): Promise /** - * Broadcast presence update to all clients in a workflow room + * Broadcast the room's presence list to all clients in the room. Pass + * `excludeSocketId` (e.g. a disconnecting socket) to omit that socket from the + * broadcast even if its presence entry outlived a failed removal — so it is + * never shown as a ghost collaborator. */ - broadcastPresenceUpdate(workflowId: string): Promise + broadcastPresenceUpdate(room: RoomRef, excludeSocketId?: string): Promise - /** - * Emit an event to all clients in a workflow room - */ - emitToWorkflow(workflowId: string, event: string, payload: T): void + /** Emit an event to all clients in a room. */ + emitToRoom(room: RoomRef, event: string, payload: T): void - /** - * Get the number of unique users in a workflow room - */ - getUniqueUserCount(workflowId: string): Promise + /** Number of unique users in a room. */ + getUniqueUserCount(room: RoomRef): Promise - /** - * Get total active connections across all rooms - */ + /** Total active connections tracked by this instance. */ getTotalActiveConnections(): Promise - - /** - * Handle workflow deletion - notify users and clean up room - */ - handleWorkflowDeletion(workflowId: string): Promise - - /** - * Handle workflow revert - notify users - */ - handleWorkflowRevert(workflowId: string, timestamp: number): Promise - - /** - * Handle workflow update - notify users - */ - handleWorkflowUpdate(workflowId: string): Promise - - /** - * Handle workflow deployment change - notify users to refresh deployment state - */ - handleWorkflowDeployed(workflowId: string): Promise } diff --git a/apps/realtime/src/rooms/workflow-room-service.ts b/apps/realtime/src/rooms/workflow-room-service.ts new file mode 100644 index 00000000000..4623647a7da --- /dev/null +++ b/apps/realtime/src/rooms/workflow-room-service.ts @@ -0,0 +1,110 @@ +import { createLogger } from '@sim/logger' +import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import type { IRoomManager } from '@/rooms/types' + +const logger = createLogger('WorkflowRoomService') + +/** The workflow room ref for a workflow id. Its Socket.IO room name is the bare id. */ +export function workflowRoom(workflowId: string): RoomRef { + return { type: ROOM_TYPES.WORKFLOW, id: workflowId } +} + +/** + * Workflow-domain lifecycle broadcasts, composed over a domain-neutral + * {@link IRoomManager}. Keeps workflow-specific concerns (deletion, revert, + * update, deploy notifications) out of the generic manager, mirroring how the + * workflow socket handlers own workflow semantics. + */ +export class WorkflowRoomService { + constructor(private readonly manager: IRoomManager) {} + + async handleWorkflowDeletion(workflowId: string): Promise { + logger.info(`Handling workflow deletion notification for ${workflowId}`) + const room = workflowRoom(workflowId) + + const users = await this.manager.getRoomUsers(room) + if (users.length === 0) { + logger.debug(`No active users found for deleted workflow ${workflowId}`) + return + } + + this.manager.emitToRoom(room, 'workflow-deleted', { + workflowId, + message: 'This workflow has been deleted', + timestamp: Date.now(), + }) + + // Remove every socket from the Socket.IO room (cross-pod via the Redis adapter). + const name = roomName(room) + await this.manager.io.in(name).socketsLeave(name) + + // Drop presence state for each socket; empty-room cleanup is handled by the manager. + for (const user of users) { + await this.manager.removeUserFromRoom(room, user.socketId) + } + + logger.info( + `Cleaned up workflow room ${workflowId} after deletion (${users.length} users disconnected)` + ) + } + + async handleWorkflowRevert(workflowId: string, timestamp: number): Promise { + logger.info(`Handling workflow revert notification for ${workflowId}`) + const room = workflowRoom(workflowId) + + if (!(await this.manager.hasRoom(room))) { + logger.debug(`No active room found for reverted workflow ${workflowId}`) + return + } + + this.manager.emitToRoom(room, 'workflow-reverted', { + workflowId, + message: 'Workflow has been reverted to deployed state', + timestamp, + }) + + await this.manager.updateRoomLastModified(room) + + const userCount = await this.manager.getUniqueUserCount(room) + logger.info(`Notified ${userCount} users about workflow revert: ${workflowId}`) + } + + async handleWorkflowUpdate(workflowId: string): Promise { + logger.info(`Handling workflow update notification for ${workflowId}`) + const room = workflowRoom(workflowId) + + if (!(await this.manager.hasRoom(room))) { + logger.debug(`No active room found for updated workflow ${workflowId}`) + return + } + + this.manager.emitToRoom(room, 'workflow-updated', { + workflowId, + message: 'Workflow has been updated externally', + timestamp: Date.now(), + }) + + await this.manager.updateRoomLastModified(room) + + const userCount = await this.manager.getUniqueUserCount(room) + logger.info(`Notified ${userCount} users about workflow update: ${workflowId}`) + } + + async handleWorkflowDeployed(workflowId: string): Promise { + logger.info(`Handling workflow deployed notification for ${workflowId}`) + const room = workflowRoom(workflowId) + + if (!(await this.manager.hasRoom(room))) { + logger.debug(`No active room found for deployed workflow ${workflowId}`) + return + } + + this.manager.emitToRoom(room, 'workflow-deployed', { + workflowId, + timestamp: Date.now(), + }) + + const userCount = await this.manager.getUniqueUserCount(room) + logger.info(`Notified ${userCount} users about workflow deployment change: ${workflowId}`) + } +} diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 0f8ed73cc52..233bee4d174 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -1,7 +1,7 @@ import type { IncomingMessage, ServerResponse } from 'http' import { safeCompare } from '@sim/security/compare' import { env } from '@/env' -import type { IRoomManager } from '@/rooms' +import { type IRoomManager, WorkflowRoomService } from '@/rooms' interface Logger { info: (message: string, ...args: unknown[]) => void @@ -58,6 +58,8 @@ function sendError(res: ServerResponse, message: string, status = 500): void { * @returns HTTP request handler function */ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { + const workflowRoomService = new WorkflowRoomService(roomManager) + return async (req: IncomingMessage, res: ServerResponse) => { // Health check doesn't require auth if (req.method === 'GET' && req.url === '/health') { @@ -99,7 +101,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId } = JSON.parse(body) - await roomManager.handleWorkflowDeletion(workflowId) + await workflowRoomService.handleWorkflowDeletion(workflowId) sendSuccess(res) } catch (error) { logger.error('Error handling workflow deletion notification:', error) @@ -113,7 +115,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId } = JSON.parse(body) - await roomManager.handleWorkflowUpdate(workflowId) + await workflowRoomService.handleWorkflowUpdate(workflowId) sendSuccess(res) } catch (error) { logger.error('Error handling workflow update notification:', error) @@ -127,7 +129,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId } = JSON.parse(body) - await roomManager.handleWorkflowDeployed(workflowId) + await workflowRoomService.handleWorkflowDeployed(workflowId) sendSuccess(res) } catch (error) { logger.error('Error handling workflow deployed notification:', error) @@ -141,7 +143,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId, timestamp } = JSON.parse(body) - await roomManager.handleWorkflowRevert(workflowId, timestamp) + await workflowRoomService.handleWorkflowRevert(workflowId, timestamp) sendSuccess(res) } catch (error) { logger.error('Error handling workflow revert notification:', error) diff --git a/packages/realtime-protocol/src/rooms.ts b/packages/realtime-protocol/src/rooms.ts index 994bf6b3cac..c03ad621d45 100644 --- a/packages/realtime-protocol/src/rooms.ts +++ b/packages/realtime-protocol/src/rooms.ts @@ -87,3 +87,13 @@ export function parseRoomName(name: string): RoomRef | null { export function isSameRoom(a: RoomRef, b: RoomRef): boolean { return a.type === b.type && a.id === b.id } + +/** + * The `presence-update` broadcast event name for a room type. `WORKFLOW` keeps + * the historical bare `presence-update` name (client backward-compat); every + * other type is namespaced so a socket joined to more than one room can tell the + * presence streams apart on a single connection. + */ +export function presenceEventName(type: RoomType): string { + return type === ROOM_TYPES.WORKFLOW ? 'presence-update' : `${type}:presence-update` +} From 4a0f241ed3085c12267a2493d2145695f5e88a32 Mon Sep 17 00:00:00 2001 From: Waleed Date: Fri, 24 Jul 2026 12:33:56 -0700 Subject: [PATCH 03/53] feat(files): live presence avatars + live file tree via realtime rooms (#5932) --- apps/realtime/src/handlers/index.ts | 2 + .../src/handlers/workspace-files.test.ts | 169 ++++++++++++++++ apps/realtime/src/handlers/workspace-files.ts | 184 ++++++++++++++++++ apps/realtime/src/rooms/types.ts | 5 + apps/realtime/src/routes/http.ts | 23 +++ .../workspaces/[id]/files/register/route.ts | 3 + .../components/presence/presence-avatars.tsx | 114 +++++++++++ .../workspace/[workspaceId]/files/files.tsx | 8 + .../files/hooks/use-workspace-files-room.ts | 122 ++++++++++++ .../workflow-item/avatars/avatars.tsx | 121 ++---------- .../hooks/queries/workspace-file-folders.ts | 2 +- apps/sim/lib/realtime/notify.ts | 38 ++++ .../orchestration/file-folder-lifecycle.ts | 9 + 13 files changed, 690 insertions(+), 110 deletions(-) create mode 100644 apps/realtime/src/handlers/workspace-files.test.ts create mode 100644 apps/realtime/src/handlers/workspace-files.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts create mode 100644 apps/sim/lib/realtime/notify.ts diff --git a/apps/realtime/src/handlers/index.ts b/apps/realtime/src/handlers/index.ts index 6ded2e54741..91573f1a1fc 100644 --- a/apps/realtime/src/handlers/index.ts +++ b/apps/realtime/src/handlers/index.ts @@ -4,6 +4,7 @@ import { setupPresenceHandlers } from '@/handlers/presence' import { setupSubblocksHandlers } from '@/handlers/subblocks' import { setupVariablesHandlers } from '@/handlers/variables' import { setupWorkflowHandlers } from '@/handlers/workflow' +import { setupWorkspaceFilesHandlers } from '@/handlers/workspace-files' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -13,5 +14,6 @@ export function setupAllHandlers(socket: AuthenticatedSocket, roomManager: IRoom setupSubblocksHandlers(socket, roomManager) setupVariablesHandlers(socket, roomManager) setupPresenceHandlers(socket, roomManager) + setupWorkspaceFilesHandlers(socket, roomManager) setupConnectionHandlers(socket, roomManager) } diff --git a/apps/realtime/src/handlers/workspace-files.test.ts b/apps/realtime/src/handlers/workspace-files.test.ts new file mode 100644 index 00000000000..9ecc943c8bb --- /dev/null +++ b/apps/realtime/src/handlers/workspace-files.test.ts @@ -0,0 +1,169 @@ +/** + * @vitest-environment node + */ +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { IRoomManager } from '@/rooms' + +const { mockAuthorizeRoom } = vi.hoisted(() => ({ + mockAuthorizeRoom: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { select: vi.fn() }, + user: { image: 'image' }, +})) + +vi.mock('@sim/platform-authz/rooms', () => ({ + authorizeRoom: mockAuthorizeRoom, +})) + +import { setupWorkspaceFilesHandlers } from '@/handlers/workspace-files' + +interface JoinPayload { + workspaceId: string + folderId?: string | null + tabSessionId?: string +} + +function createSocket(overrides?: Record) { + const handlers: Record Promise | void> = {} + const socket = { + id: 'socket-1', + userId: 'user-1', + userName: 'Test User', + userImage: 'avatar.png', + on: vi.fn((event: string, handler: (payload: JoinPayload) => Promise | void) => { + handlers[event] = handler + }), + emit: vi.fn(), + join: vi.fn(), + leave: vi.fn(), + to: vi.fn().mockReturnValue({ emit: vi.fn() }), + ...overrides, + } + return { handlers, socket } +} + +function createRoomManager(overrides?: Partial): IRoomManager { + return { + isReady: vi.fn().mockReturnValue(true), + getRoomForSocket: vi.fn().mockResolvedValue(null), + getRoomsForSocket: vi.fn().mockResolvedValue([]), + removeUserFromRoom: vi.fn().mockResolvedValue(false), + removeSocketFromAllRooms: vi.fn().mockResolvedValue([]), + broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), + getRoomUsers: vi.fn().mockResolvedValue([]), + hasRoom: vi.fn().mockResolvedValue(false), + addUserToRoom: vi.fn().mockResolvedValue(undefined), + getUserSession: vi.fn().mockResolvedValue(null), + updateUserActivity: vi.fn().mockResolvedValue(undefined), + updateRoomLastModified: vi.fn().mockResolvedValue(undefined), + emitToRoom: vi.fn(), + getUniqueUserCount: vi.fn().mockResolvedValue(1), + getTotalActiveConnections: vi.fn().mockResolvedValue(0), + shutdown: vi.fn().mockResolvedValue(undefined), + initialize: vi.fn().mockResolvedValue(undefined), + io: { + in: vi.fn().mockReturnValue({ socketsLeave: vi.fn().mockResolvedValue(undefined) }), + }, + ...overrides, + } as unknown as IRoomManager +} + +describe('setupWorkspaceFilesHandlers', () => { + beforeEach(() => { + vi.clearAllMocks() + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', + }) + }) + + it('rejects join when the socket is not authenticated', async () => { + const { socket, handlers } = createSocket({ userId: undefined, userName: undefined }) + setupWorkspaceFilesHandlers( + socket as unknown as Parameters[0], + createRoomManager() + ) + + await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith('join-workspace-files-error', { + workspaceId: 'ws-1', + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + }) + + it('rejects join with a retryable error when realtime is unavailable', async () => { + const { socket, handlers } = createSocket() + setupWorkspaceFilesHandlers( + socket as unknown as Parameters[0], + createRoomManager({ isReady: vi.fn().mockReturnValue(false) }) + ) + + await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + 'join-workspace-files-error', + expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) + ) + }) + + it('rejects join when workspace access is denied', async () => { + mockAuthorizeRoom.mockResolvedValue({ + allowed: false, + status: 403, + workspaceId: 'ws-1', + workspacePermission: null, + }) + const { socket, handlers } = createSocket() + setupWorkspaceFilesHandlers( + socket as unknown as Parameters[0], + createRoomManager() + ) + + await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + 'join-workspace-files-error', + expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) + ) + }) + + it('joins the workspace files room and broadcasts presence on success', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + getRoomUsers: vi.fn().mockResolvedValue([]), + }) + setupWorkspaceFilesHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['join-workspace-files']({ + workspaceId: 'ws-1', + folderId: 'folder-1', + tabSessionId: 'tab-1', + }) + + expect(socket.join).toHaveBeenCalledWith('workspace-files:ws-1') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-1' }, + 'socket-1', + expect.objectContaining({ userId: 'user-1', folderId: 'folder-1', role: 'admin' }) + ) + expect(socket.emit).toHaveBeenCalledWith( + 'join-workspace-files-success', + expect.objectContaining({ workspaceId: 'ws-1', socketId: 'socket-1' }) + ) + expect(roomManager.broadcastPresenceUpdate).toHaveBeenCalledWith({ + type: ROOM_TYPES.WORKSPACE_FILES, + id: 'ws-1', + }) + }) +}) diff --git a/apps/realtime/src/handlers/workspace-files.ts b/apps/realtime/src/handlers/workspace-files.ts new file mode 100644 index 00000000000..c88d403e44e --- /dev/null +++ b/apps/realtime/src/handlers/workspace-files.ts @@ -0,0 +1,184 @@ +import { db, user } from '@sim/db' +import { createLogger } from '@sim/logger' +import { authorizeRoom } from '@sim/platform-authz/rooms' +import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { eq } from 'drizzle-orm' +import type { AuthenticatedSocket } from '@/middleware/auth' +import type { IRoomManager, UserPresence } from '@/rooms' + +const logger = createLogger('WorkspaceFilesHandlers') + +/** The workspace-files room ref for a workspace id. */ +const filesRoom = (workspaceId: string): RoomRef => ({ + type: ROOM_TYPES.WORKSPACE_FILES, + id: workspaceId, +}) + +interface JoinPayload { + workspaceId: string + folderId?: string | null + tabSessionId?: string +} + +async function resolveAvatarUrl( + socket: AuthenticatedSocket, + userId: string +): Promise { + if (socket.userImage) return socket.userImage + try { + const [record] = await db + .select({ image: user.image }) + .from(user) + .where(eq(user.id, userId)) + .limit(1) + return record?.image ?? null + } catch (error) { + logger.warn('Failed to load user avatar for files presence', { userId, error }) + return null + } +} + +/** + * Presence handlers for the workspace file browser. Mirrors the workflow join + * flow but is workspace-scoped (room id = workspaceId) and read-only presence: + * there are no persisted file operations over the socket — file mutations go + * through the HTTP API, which fans out a `workspace-files-changed` event + * separately. The viewer's `folderId` is recorded at join (a future hook for + * folder-scoped presence); there is no cursor channel here yet. + */ +export function setupWorkspaceFilesHandlers( + socket: AuthenticatedSocket, + roomManager: IRoomManager +) { + socket.on( + 'join-workspace-files', + async ({ workspaceId, folderId, tabSessionId }: JoinPayload) => { + try { + const userId = socket.userId + const userName = socket.userName + + if (!userId || !userName) { + socket.emit('join-workspace-files-error', { + workspaceId, + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + return + } + + if (!roomManager.isReady()) { + socket.emit('join-workspace-files-error', { + workspaceId, + error: 'Realtime unavailable', + code: 'ROOM_MANAGER_UNAVAILABLE', + retryable: true, + }) + return + } + + const room = filesRoom(workspaceId) + + let authorized: Awaited> + try { + authorized = await authorizeRoom({ userId, room, action: 'read' }) + } catch (error) { + logger.warn(`Error authorizing files room for ${userId}:`, error) + socket.emit('join-workspace-files-error', { + workspaceId, + error: 'Failed to verify workspace access', + code: 'VERIFY_ACCESS_FAILED', + retryable: true, + }) + return + } + + if (!authorized.allowed) { + socket.emit('join-workspace-files-error', { + workspaceId, + error: authorized.status === 404 ? 'Workspace not found' : 'Access denied to workspace', + code: authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', + retryable: false, + }) + return + } + + // Leave a previously-joined files room if switching workspaces. + const currentRoom = await roomManager.getRoomForSocket( + socket.id, + ROOM_TYPES.WORKSPACE_FILES + ) + if (currentRoom && currentRoom.id !== workspaceId) { + socket.leave(roomName(currentRoom)) + await roomManager.removeUserFromRoom(currentRoom, socket.id) + await roomManager.broadcastPresenceUpdate(currentRoom) + } + + // Clean up the same user's stale socket from the same tab (e.g. a reconnect + // that raced the old socket's disconnect), so presence shows one entry. + if (tabSessionId) { + const existingUsers = await roomManager.getRoomUsers(room) + for (const existing of existingUsers) { + if ( + existing.socketId !== socket.id && + existing.userId === userId && + existing.tabSessionId === tabSessionId + ) { + await roomManager.removeUserFromRoom(room, existing.socketId) + await roomManager.io.in(existing.socketId).socketsLeave(roomName(room)) + } + } + } + + socket.join(roomName(room)) + + const presence: UserPresence = { + userId, + room, + userName, + socketId: socket.id, + tabSessionId, + joinedAt: Date.now(), + lastActivity: Date.now(), + role: authorized.workspacePermission ?? 'read', + folderId: folderId ?? null, + avatarUrl: await resolveAvatarUrl(socket, userId), + } + + await roomManager.addUserToRoom(room, socket.id, presence) + + const presenceUsers = await roomManager.getRoomUsers(room) + socket.emit('join-workspace-files-success', { + workspaceId, + socketId: socket.id, + presenceUsers, + }) + + await roomManager.broadcastPresenceUpdate(room) + + logger.info(`User ${userId} (${userName}) joined files room for workspace ${workspaceId}`) + } catch (error) { + logger.error('Error joining workspace files room:', error) + socket.emit('join-workspace-files-error', { + workspaceId, + error: 'Failed to join workspace files', + code: 'JOIN_FAILED', + retryable: true, + }) + } + } + ) + + socket.on('leave-workspace-files', async () => { + try { + if (!roomManager.isReady()) return + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKSPACE_FILES) + if (!room) return + socket.leave(roomName(room)) + await roomManager.removeUserFromRoom(room, socket.id) + await roomManager.broadcastPresenceUpdate(room) + } catch (error) { + logger.error('Error leaving workspace files room:', error) + } + }) +} diff --git a/apps/realtime/src/rooms/types.ts b/apps/realtime/src/rooms/types.ts index 67cee450832..80525195ef8 100644 --- a/apps/realtime/src/rooms/types.ts +++ b/apps/realtime/src/rooms/types.ts @@ -19,6 +19,11 @@ export interface UserPresence { cursor?: { x: number; y: number } selection?: { type: 'block' | 'edge' | 'none'; id?: string } avatarUrl?: string | null + /** + * The subfolder the user is viewing, recorded at join for room types that track + * a per-viewer location (e.g. the workspace file browser). `null` is the root. + */ + folderId?: string | null } /** diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 233bee4d174..d8db9473cb6 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -1,4 +1,5 @@ import type { IncomingMessage, ServerResponse } from 'http' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { safeCompare } from '@sim/security/compare' import { env } from '@/env' import { type IRoomManager, WorkflowRoomService } from '@/rooms' @@ -152,6 +153,28 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } + // Fan out a file-tree change to everyone viewing a workspace's files, so their + // browser refetches. File mutations happen over the HTTP API (not the socket); + // this is the lossy liveness signal — a missed one only means stale-until-refetch. + if (req.method === 'POST' && req.url === '/api/workspace-files-changed') { + try { + const body = await readRequestBody(req) + const { workspaceId } = JSON.parse(body) + if (typeof workspaceId === 'string' && workspaceId.length > 0) { + roomManager.emitToRoom( + { type: ROOM_TYPES.WORKSPACE_FILES, id: workspaceId }, + 'workspace-files-changed', + { workspaceId, timestamp: Date.now() } + ) + } + sendSuccess(res) + } catch (error) { + logger.error('Error handling workspace files changed notification:', error) + sendError(res, 'Failed to process files change notification') + } + return + } + res.writeHead(404, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ error: 'Not found' })) } diff --git a/apps/sim/app/api/workspaces/[id]/files/register/route.ts b/apps/sim/app/api/workspaces/[id]/files/register/route.ts index fd29a6c9dfb..4ed0b90c285 100644 --- a/apps/sim/app/api/workspaces/[id]/files/register/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/register/route.ts @@ -7,6 +7,7 @@ import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' +import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { FileConflictError, parseWorkspaceFileKey, @@ -63,6 +64,8 @@ export const POST = withRouteHandler( if (created) { logger.info(`Registered direct upload ${name} -> ${key}`) + await notifyWorkspaceFilesChanged(workspaceId) + captureServerEvent( userId, 'file_uploaded', diff --git a/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx new file mode 100644 index 00000000000..3698578f8af --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx @@ -0,0 +1,114 @@ +'use client' + +import { useMemo } from 'react' +import { Avatar, AvatarFallback, AvatarImage, Tooltip } from '@sim/emcn' +import { getUserColor } from '@/lib/workspaces/colors' + +/** Minimal presence shape the avatar stack renders — shared by workflow and files. */ +export interface PresenceAvatarUser { + socketId: string + userId: string + userName?: string + avatarUrl?: string | null +} + +interface UserAvatarProps { + user: PresenceAvatarUser + index: number +} + +/** + * A single collaborator avatar: their image, falling back to a colored circle + * with their initial. Wrapped in a name tooltip when the name is known. + */ +function UserAvatar({ user, index }: UserAvatarProps) { + const color = getUserColor(user.userId) + const initials = user.userName ? user.userName.charAt(0).toUpperCase() : '?' + + const avatarElement = ( + + {user.avatarUrl && ( + + )} + + {initials} + + + ) + + if (user.userName) { + return ( + + {avatarElement} + + {user.userName} + + + ) + } + + return avatarElement +} + +interface PresenceAvatarsProps { + /** Collaborators to show — already filtered to exclude the current socket. */ + users: PresenceAvatarUser[] + /** Max avatars before collapsing the remainder into a "+N" chip. */ + maxVisible?: number +} + +const DEFAULT_MAX_VISIBLE = 5 + +/** + * Overlapping stack of collaborator avatars for presence. Presentational only — + * the caller owns fetching/filtering presence (workflow sidebar item, files + * header, etc.), so the stack looks identical everywhere it appears. + */ +export function PresenceAvatars({ users, maxVisible = DEFAULT_MAX_VISIBLE }: PresenceAvatarsProps) { + const { visibleUsers, overflowCount } = useMemo(() => { + if (users.length === 0) { + return { visibleUsers: [] as PresenceAvatarUser[], overflowCount: 0 } + } + const visible = users.slice(0, maxVisible) + const overflow = Math.max(0, users.length - maxVisible) + // Reverse so the rightmost avatar stays stable as new ones reveal on the left. + return { visibleUsers: [...visible].reverse(), overflowCount: overflow } + }, [users, maxVisible]) + + if (visibleUsers.length === 0) { + return null + } + + return ( +
+ {overflowCount > 0 && ( + + + + + +{overflowCount} + + + + + {overflowCount} more user{overflowCount > 1 ? 's' : ''} + + + )} + {visibleUsers.map((user, index) => ( + + ))} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index c6bcecde52d..9128bebf31d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -61,6 +61,7 @@ import { Resource, timeCell, } from '@/app/workspace/[workspaceId]/components' +import { PresenceAvatars } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' import { FilesActionBar } from '@/app/workspace/[workspaceId]/files/components/action-bar' import { DeleteConfirmModal } from '@/app/workspace/[workspaceId]/files/components/delete-confirm-modal' import { FileRowContextMenu } from '@/app/workspace/[workspaceId]/files/components/file-row-context-menu' @@ -74,6 +75,7 @@ import { } from '@/app/workspace/[workspaceId]/files/components/file-viewer' import { FilesListContextMenu } from '@/app/workspace/[workspaceId]/files/components/files-list-context-menu' import { ShareModal } from '@/app/workspace/[workspaceId]/files/components/share-modal' +import { useWorkspaceFilesRoom } from '@/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room' import type { MoveOptionNode } from '@/app/workspace/[workspaceId]/files/move-options' import { filesFilterParsers, @@ -203,6 +205,11 @@ export function Files() { const canEdit = userPermissions.canEdit === true const { config: permissionConfig } = usePermissionConfig() + const { otherUsers: filesPresenceUsers } = useWorkspaceFilesRoom( + workspaceId, + currentFolderId ?? null + ) + useEffect(() => { if (permissionConfig.hideFilesTab) { router.replace(`/workspace/${workspaceId}`) @@ -1959,6 +1966,7 @@ export function Files() { title='Files' breadcrumbs={listBreadcrumbs} actions={headerActionsConfig} + aside={} /> ([]) + + const tabSessionIdRef = useRef('') + if (!tabSessionIdRef.current) tabSessionIdRef.current = generateShortId() + const folderIdRef = useRef(folderId) + + useEffect(() => { + folderIdRef.current = folderId + }, [folderId]) + + useEffect(() => { + if (!socket || !workspaceId) return + + let retries = 0 + let retryTimer: ReturnType | null = null + + const join = () => { + socket.emit('join-workspace-files', { + workspaceId, + folderId: folderIdRef.current, + tabSessionId: tabSessionIdRef.current, + }) + } + + const handleJoinSuccess = (data: { + workspaceId: string + presenceUsers: PresenceUpdatePayload[] + }) => { + if (data.workspaceId !== workspaceId) return + retries = 0 + setPresenceUsers(data.presenceUsers ?? []) + } + const handleJoinError = (data: JoinErrorPayload) => { + if (data.workspaceId !== workspaceId) return + logger.warn('Failed to join workspace files room', { code: data.code, error: data.error }) + if (data.retryable && retries < MAX_JOIN_RETRIES) { + retries += 1 + retryTimer = setTimeout(join, JOIN_RETRY_BASE_MS * retries) + } + } + const handlePresence = (users: PresenceUpdatePayload[]) => setPresenceUsers(users ?? []) + const handleChanged = (data: { workspaceId: string }) => { + if (data.workspaceId === workspaceId) + invalidateWorkspaceFileBrowsers(queryClient, workspaceId) + } + + // Join now if the socket is already connected; `connect` covers (re)connects. + if (socket.connected) join() + socket.on('connect', join) + socket.on('join-workspace-files-success', handleJoinSuccess) + socket.on('join-workspace-files-error', handleJoinError) + socket.on('workspace-files:presence-update', handlePresence) + socket.on('workspace-files-changed', handleChanged) + + return () => { + if (retryTimer) clearTimeout(retryTimer) + socket.emit('leave-workspace-files') + socket.off('connect', join) + socket.off('join-workspace-files-success', handleJoinSuccess) + socket.off('join-workspace-files-error', handleJoinError) + socket.off('workspace-files:presence-update', handlePresence) + socket.off('workspace-files-changed', handleChanged) + setPresenceUsers([]) + } + }, [socket, workspaceId, queryClient]) + + const otherUsers = useMemo( + () => presenceUsers.filter((user) => user.socketId !== currentSocketId), + [presenceUsers, currentSocketId] + ) + + return { otherUsers } +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/workflow-item/avatars/avatars.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/workflow-item/avatars/avatars.tsx index 8626149a174..19b29832c61 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/workflow-item/avatars/avatars.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/workflow-item/avatars/avatars.tsx @@ -1,8 +1,7 @@ 'use client' -import { type CSSProperties, useMemo } from 'react' -import { Avatar, AvatarFallback, AvatarImage, Tooltip } from '@sim/emcn' -import { getUserColor } from '@/lib/workspaces/colors' +import { useMemo } from 'react' +import { PresenceAvatars } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' import { useSocket } from '@/app/workspace/providers/socket-provider' import { SIDEBAR_WIDTH } from '@/stores/constants' import { usePresenceStore } from '@/stores/presence/store' @@ -21,64 +20,11 @@ interface AvatarsProps { workflowId: string } -interface PresenceUser { - socketId: string - userId: string - userName?: string - avatarUrl?: string | null -} - -interface UserAvatarProps { - user: PresenceUser - index: number -} - /** - * Individual user avatar using emcn Avatar component. - * Falls back to colored circle with initials if image fails to load. - */ -function UserAvatar({ user, index }: UserAvatarProps) { - const color = getUserColor(user.userId) - const initials = user.userName ? user.userName.charAt(0).toUpperCase() : '?' - - const avatarElement = ( - - {user.avatarUrl && ( - - )} - - {initials} - - - ) - - if (user.userName) { - return ( - - {avatarElement} - - {user.userName} - - - ) - } - - return avatarElement -} - -/** - * Displays user avatars for presence in a workflow item. - * Only shows avatars for the currently active workflow. - * - * @param props - Component props - * @returns Avatar stack for workflow presence + * Presence avatars for a workflow sidebar item. Owns the workflow-specific + * concerns — the presence source, the current-workflow filter, and the + * sidebar-width-driven visible count — and delegates the stack rendering to the + * shared {@link PresenceAvatars}, so it looks identical to every other surface. */ export function Avatars({ workflowId }: AvatarsProps) { const { currentWorkflowId, currentSocketId } = useSocket() @@ -86,8 +32,8 @@ export function Avatars({ workflowId }: AvatarsProps) { const sidebarWidth = useSidebarStore((state) => state.sidebarWidth) /** - * Calculate max visible avatars based on sidebar width. - * Scales between MIN_COUNT and MAX_COUNT as sidebar expands. + * Scale the max visible avatars between MIN_COUNT and MAX_COUNT as the sidebar + * widens. */ const maxVisible = useMemo(() => { const widthDelta = sidebarWidth - SIDEBAR_WIDTH.MIN @@ -97,56 +43,13 @@ export function Avatars({ workflowId }: AvatarsProps) { }, [sidebarWidth]) /** - * Only show presence for the currently active workflow. - * Filter out the current socket connection (allows same user's other tabs to appear). + * Only show presence for the currently active workflow, excluding the current + * socket (so the user's own other tabs still appear). */ const workflowUsers = useMemo(() => { - if (currentWorkflowId !== workflowId) { - return [] - } + if (currentWorkflowId !== workflowId) return [] return presenceUsers.filter((user) => user.socketId !== currentSocketId) }, [presenceUsers, currentWorkflowId, workflowId, currentSocketId]) - /** - * Calculate visible users and overflow count. - * Shows up to maxVisible avatars, with overflow indicator for any remaining. - * Users are reversed so new avatars appear on the left (keeping right side stable). - */ - const { visibleUsers, overflowCount } = useMemo(() => { - if (workflowUsers.length === 0) { - return { visibleUsers: [], overflowCount: 0 } - } - - const visible = workflowUsers.slice(0, maxVisible) - const overflow = Math.max(0, workflowUsers.length - maxVisible) - - // Reverse so rightmost avatars stay stable as new ones are revealed on the left - return { visibleUsers: [...visible].reverse(), overflowCount: overflow } - }, [workflowUsers, maxVisible]) - - if (visibleUsers.length === 0) { - return null - } - - return ( -
- {overflowCount > 0 && ( - - - - - +{overflowCount} - - - - - {overflowCount} more user{overflowCount > 1 ? 's' : ''} - - - )} - {visibleUsers.map((user, index) => ( - - ))} -
- ) + return } diff --git a/apps/sim/hooks/queries/workspace-file-folders.ts b/apps/sim/hooks/queries/workspace-file-folders.ts index bad1c52c0ee..357a6b8b3af 100644 --- a/apps/sim/hooks/queries/workspace-file-folders.ts +++ b/apps/sim/hooks/queries/workspace-file-folders.ts @@ -40,7 +40,7 @@ async function fetchWorkspaceFileFolders( return data.folders } -function invalidateWorkspaceFileBrowsers( +export function invalidateWorkspaceFileBrowsers( queryClient: ReturnType, workspaceId: string ) { diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts new file mode 100644 index 00000000000..e738130b6de --- /dev/null +++ b/apps/sim/lib/realtime/notify.ts @@ -0,0 +1,38 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { env } from '@/lib/core/config/env' +import { getSocketServerUrl } from '@/lib/core/utils/urls' + +const logger = createLogger('RealtimeNotify') + +/** Bound the wait on the realtime server so a slow/hung socket pod can't stall a file mutation. */ +const NOTIFY_TIMEOUT_MS = 2000 + +/** + * Best-effort fan-out to the realtime server that a workspace's file tree changed, + * so every browser currently viewing that workspace's files refetches. File + * mutations happen over the HTTP API (not the socket); this is the lossy liveness + * signal — a dropped notification only degrades to stale-until-refetch, so it must + * never throw or block the originating mutation. + */ +export async function notifyWorkspaceFilesChanged(workspaceId: string): Promise { + try { + const response = await fetch(`${getSocketServerUrl()}/api/workspace-files-changed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ workspaceId }), + signal: AbortSignal.timeout(NOTIFY_TIMEOUT_MS), + }) + if (!response.ok) { + logger.warn('workspace-files-changed notify failed', { + workspaceId, + status: response.status, + }) + } + } catch (error) { + logger.warn('workspace-files-changed notify error', { + workspaceId, + error: getErrorMessage(error), + }) + } +} diff --git a/apps/sim/lib/workspace-files/orchestration/file-folder-lifecycle.ts b/apps/sim/lib/workspace-files/orchestration/file-folder-lifecycle.ts index ce11157ccb7..4abfd980bcf 100644 --- a/apps/sim/lib/workspace-files/orchestration/file-folder-lifecycle.ts +++ b/apps/sim/lib/workspace-files/orchestration/file-folder-lifecycle.ts @@ -1,6 +1,7 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getPostgresErrorCode, toError } from '@sim/utils/errors' +import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { bulkArchiveWorkspaceFileItems, createWorkspaceFileFolder, @@ -194,6 +195,7 @@ export async function performDeleteWorkspaceFileItems( }) } + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, deletedItems } } catch (error) { logger.error('Failed to delete workspace file items', { error }) @@ -254,6 +256,7 @@ export async function performMoveWorkspaceFileItems( }) } + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, movedItems } } catch (error) { logger.error('Failed to move workspace file items', { error }) @@ -298,6 +301,7 @@ export async function performRenameWorkspaceFile( description: `Renamed file to "${file.name}"`, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, file } } catch (error) { logger.error('Failed to rename workspace file', { error }) @@ -361,6 +365,7 @@ export async function performMoveRenameWorkspaceFile( }) } + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, file } } catch (error) { logger.error('Failed to move/rename workspace file', { error }) @@ -394,6 +399,7 @@ export async function performRestoreWorkspaceFile( description: `Restored workspace file ${fileId}`, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true } } catch (error) { logger.error('Failed to restore workspace file', { error }) @@ -424,6 +430,7 @@ export async function performCreateWorkspaceFileFolder( description: `Created file folder "${folder.name}"`, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, folder } } catch (error) { logger.error('Failed to create workspace file folder', { error }) @@ -463,6 +470,7 @@ export async function performUpdateWorkspaceFileFolder( description: `Updated file folder "${folder.name}"`, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, folder } } catch (error) { logger.error('Failed to update workspace file folder', { error }) @@ -509,6 +517,7 @@ export async function performRestoreWorkspaceFileFolder( }, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, folder, restoredItems } } catch (error) { logger.error('Failed to restore workspace file folder', { error }) From a639af358d4237f40c228ab44b72a20278df96cf Mon Sep 17 00:00:00 2001 From: Waleed Date: Fri, 24 Jul 2026 12:34:24 -0700 Subject: [PATCH 04/53] refactor(tables): adopt shared durable event-log core (#5934) --- .../table/[tableId]/events/stream/route.ts | 152 +-------- apps/sim/lib/realtime/event-log.test.ts | 86 +++++ apps/sim/lib/realtime/event-log.ts | 280 +++++++++++++++++ apps/sim/lib/realtime/event-stream-route.ts | 160 ++++++++++ apps/sim/lib/table/events.ts | 293 +++--------------- 5 files changed, 579 insertions(+), 392 deletions(-) create mode 100644 apps/sim/lib/realtime/event-log.test.ts create mode 100644 apps/sim/lib/realtime/event-log.ts create mode 100644 apps/sim/lib/realtime/event-stream-route.ts diff --git a/apps/sim/app/api/table/[tableId]/events/stream/route.ts b/apps/sim/app/api/table/[tableId]/events/stream/route.ts index 2e9cff2eb37..2142e4a1d93 100644 --- a/apps/sim/app/api/table/[tableId]/events/stream/route.ts +++ b/apps/sim/app/api/table/[tableId]/events/stream/route.ts @@ -1,26 +1,13 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' import { type NextRequest, NextResponse } from 'next/server' import { tableEventStreamContract } from '@/lib/api/contracts/tables' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' -import { SSE_HEADERS } from '@/lib/core/utils/sse' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - getLatestTableEventId, - readTableEventsSince, - type TableEventEntry, -} from '@/lib/table/events' +import { createEventStreamResponse } from '@/lib/realtime/event-stream-route' +import { getLatestTableEventId, readTableEventsSince } from '@/lib/table/events' import { accessError, checkAccess } from '@/app/api/table/utils' -const logger = createLogger('TableEventStreamAPI') - -const POLL_INTERVAL_MS = 500 -const HEARTBEAT_INTERVAL_MS = 15_000 -const MAX_STREAM_DURATION_MS = 4 * 60 * 60 * 1000 // 4 hours; client reconnects past this - export const runtime = 'nodejs' export const dynamic = 'force-dynamic' @@ -30,12 +17,9 @@ interface RouteContext { /** GET /api/table/[tableId]/events/stream?from= * - * SSE stream of cell-state transitions. Replay-on-reconnect via `from`; - * absent `from` tails from the latest event id (fresh mount — the client has - * just fetched current state, so replaying history would rewind it). - * Pruning (buffer cap exceeded or TTL expired) sends a `pruned` event and - * closes; the client responds with a full row-query refetch and reconnects - * tailing from latest. */ + * SSE stream of cell-state transitions over the shared durable event log. Auth + * and access are checked here; the replay/tail/poll/heartbeat/prune mechanics + * come from `createEventStreamResponse`. */ export const GET = withRouteHandler(async (req: NextRequest, context: RouteContext) => { const requestId = generateRequestId() const parsed = await parseRequest(tableEventStreamContract, req, context) @@ -51,123 +35,13 @@ export const GET = withRouteHandler(async (req: NextRequest, context: RouteConte const access = await checkAccess(tableId, auth.userId, 'read') if (!access.ok) return accessError(access, requestId, tableId) - logger.info(`[${requestId}] Table event stream opened`, { tableId, fromEventId }) - - const encoder = new TextEncoder() - let closed = false - - const stream = new ReadableStream({ - async start(controller) { - let lastEventId = fromEventId ?? 0 - const deadline = Date.now() + MAX_STREAM_DURATION_MS - let nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS - - const enqueue = (text: string) => { - if (closed) return - try { - controller.enqueue(encoder.encode(text)) - } catch { - closed = true - } - } - - const sendEvents = (events: TableEventEntry[]) => { - for (const entry of events) { - if (closed) return - enqueue(`data: ${JSON.stringify(entry)}\n\n`) - lastEventId = entry.eventId - } - } - - const sendPrunedAndClose = (earliestEventId: number | undefined) => { - enqueue( - `event: pruned\ndata: ${JSON.stringify({ earliestEventId: earliestEventId ?? null })}\n\n` - ) - if (!closed) { - closed = true - try { - controller.close() - } catch {} - } - } - - const sendHeartbeat = () => { - // SSE comment line — keeps proxies (ALB default 60s idle) from closing - // the connection during quiet periods. - enqueue(`: ping ${Date.now()}\n\n`) - } - - try { - // No replay cursor → tail from the latest event id. Resolved inside - // the try so a Redis failure errors the stream (client reconnects - // with backoff) rather than silently replaying the whole buffer. - if (fromEventId === undefined) { - lastEventId = await getLatestTableEventId(tableId) - } - // Initial replay from buffer. - const initial = await readTableEventsSince(tableId, lastEventId) - if (initial.status === 'pruned') { - sendPrunedAndClose(initial.earliestEventId) - return - } - if (initial.status === 'unavailable') { - throw new Error(`Table event buffer unavailable: ${initial.error}`) - } - sendEvents(initial.events) - - // Stream loop — poll the buffer and forward new events. Workflow - // execution stream uses the same shape; pub/sub wakeups are an - // optimization we can add later if 500ms polling becomes a problem. - while (!closed && Date.now() < deadline) { - await sleep(POLL_INTERVAL_MS) - if (closed) return - - const result = await readTableEventsSince(tableId, lastEventId) - if (result.status === 'pruned') { - sendPrunedAndClose(result.earliestEventId) - return - } - if (result.status === 'unavailable') { - throw new Error(`Table event buffer unavailable: ${result.error}`) - } - if (result.events.length > 0) { - sendEvents(result.events) - } - - if (Date.now() >= nextHeartbeatAt) { - sendHeartbeat() - nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS - } - } - - // Reached the defensive duration ceiling — close cleanly so the client - // reconnects with the latest lastEventId. - if (!closed) { - enqueue(`event: rotate\ndata: {}\n\n`) - closed = true - try { - controller.close() - } catch {} - } - } catch (error) { - logger.error(`[${requestId}] Table event stream error`, { - tableId, - error: toError(error).message, - }) - if (!closed) { - try { - controller.error(error) - } catch {} - } - } - }, - cancel() { - closed = true - logger.info(`[${requestId}] Client disconnected from table event stream`, { tableId }) - }, - }) - - return new NextResponse(stream, { - headers: { ...SSE_HEADERS, 'X-Table-Id': tableId }, + return createEventStreamResponse({ + requestId, + label: 'table', + streamId: tableId, + fromEventId, + getLatestEventId: getLatestTableEventId, + readEventsSince: readTableEventsSince, + extraHeaders: { 'X-Table-Id': tableId }, }) }) diff --git a/apps/sim/lib/realtime/event-log.test.ts b/apps/sim/lib/realtime/event-log.test.ts new file mode 100644 index 00000000000..34d113d79f5 --- /dev/null +++ b/apps/sim/lib/realtime/event-log.test.ts @@ -0,0 +1,86 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/config/env', () => ({ env: { REDIS_URL: undefined } })) +vi.mock('@/lib/core/config/redis', () => ({ getRedisClient: () => null })) + +import { + appendEvent, + type EventLogConfig, + type EventLogEntry, + getLatestEventId, + readEventsSince, + resetEventLogMemoryForTesting, +} from '@/lib/realtime/event-log' + +interface TestEntry extends EventLogEntry { + eventId: number + streamId: string + value: string +} + +const config: EventLogConfig = { prefix: 'test:stream:', ttlSeconds: 3600, cap: 3, readChunk: 500 } + +function serializerFor(streamId: string, value: string) { + return { + entryPrefix: '{"eventId":', + entrySuffix: `,"streamId":${JSON.stringify(streamId)},"value":${JSON.stringify(value)}}`, + buildMemory: (eventId: number): TestEntry => ({ eventId, streamId, value }), + } +} + +describe('event-log (memory fallback)', () => { + beforeEach(() => resetEventLogMemoryForTesting()) + + it('assigns monotonically increasing event ids', async () => { + const first = await appendEvent(config, 's1', serializerFor('s1', 'a')) + const second = await appendEvent(config, 's1', serializerFor('s1', 'b')) + expect(first?.eventId).toBe(1) + expect(second?.eventId).toBe(2) + }) + + it('isolates streams by id', async () => { + await appendEvent(config, 's1', serializerFor('s1', 'a')) + const other = await appendEvent(config, 's2', serializerFor('s2', 'x')) + expect(other?.eventId).toBe(1) + expect(await getLatestEventId(config, 's1')).toBe(1) + expect(await getLatestEventId(config, 's2')).toBe(1) + }) + + it('reads only events after the cursor', async () => { + await appendEvent(config, 's1', serializerFor('s1', 'a')) + await appendEvent(config, 's1', serializerFor('s1', 'b')) + const result = await readEventsSince(config, 's1', 1) + expect(result.status).toBe('ok') + if (result.status === 'ok') { + expect(result.events).toHaveLength(1) + expect(result.events[0].eventId).toBe(2) + expect(result.events[0].value).toBe('b') + } + }) + + it('tails from the latest id and returns nothing for a fresh cursor', async () => { + await appendEvent(config, 's1', serializerFor('s1', 'a')) + await appendEvent(config, 's1', serializerFor('s1', 'b')) + const latest = await getLatestEventId(config, 's1') + const result = await readEventsSince(config, 's1', latest) + expect(result).toEqual({ status: 'ok', events: [] }) + }) + + it('reports pruned when the cursor falls behind the cap-trimmed buffer', async () => { + // cap = 3; append 5, so the earliest retained id is 3. + for (const v of ['a', 'b', 'c', 'd', 'e']) { + await appendEvent(config, 's1', serializerFor('s1', v)) + } + const result = await readEventsSince(config, 's1', 1) + expect(result.status).toBe('pruned') + if (result.status === 'pruned') expect(result.earliestEventId).toBe(3) + }) + + it('reports pruned for a non-zero cursor against a never-seen stream', async () => { + const result = await readEventsSince(config, 'missing', 5) + expect(result.status).toBe('pruned') + }) +}) diff --git a/apps/sim/lib/realtime/event-log.ts b/apps/sim/lib/realtime/event-log.ts new file mode 100644 index 00000000000..fd107a774a2 --- /dev/null +++ b/apps/sim/lib/realtime/event-log.ts @@ -0,0 +1,280 @@ +/** + * Generic durable event log over Redis (sorted set + monotonic id + TTL), with an + * in-memory fallback for dev/tests. This is the reusable core extracted from the + * Tables cell-event buffer; a domain adapter (e.g. `lib/table/events.ts`) supplies + * its Redis key prefix and how to serialize an entry, and gets append/read/tail + * semantics for free — including replay-on-reconnect and prune detection. + * + * The core is deliberately domain-neutral: it only knows an entry has a numeric + * `eventId`. Everything else in the entry is opaque bytes the adapter owns, so a + * domain can keep its exact wire shape (and its existing Redis keys) unchanged. + * + * Modeled after `apps/sim/lib/execution/event-buffer.ts` but stripped of what an + * always-on stream doesn't need (no id-reservation batching, no write-queue + * serialization, no per-entity terminal lifecycle, no byte budgeting). + */ + +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { env } from '@/lib/core/config/env' +import { getRedisClient } from '@/lib/core/config/redis' + +const logger = createLogger('EventLog') + +/** + * Atomic append: INCR the seq counter to mint a new eventId, splice it into the + * adapter-supplied entry JSON, ZADD it, refresh TTLs, trim to cap, and record the + * resulting earliestEventId in meta — one round-trip. Without atomicity a slow + * reader could observe the trim before the meta update and miss the prune signal. + * + * KEYS: [events, seq, meta] + * ARGV: [ttlSec, cap, updatedAtIso, entryPrefix, entrySuffix] + * The new eventId is spliced between prefix/suffix to form the entry JSON. + * Returns the new eventId. + */ +const APPEND_EVENT_SCRIPT = ` +local eventId = redis.call('INCR', KEYS[2]) +local entry = ARGV[4] .. eventId .. ARGV[5] +redis.call('ZADD', KEYS[1], eventId, entry) +redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1])) +redis.call('EXPIRE', KEYS[2], tonumber(ARGV[1])) +redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -tonumber(ARGV[2]) - 1) +local oldest = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES') +if oldest[2] then + redis.call('HSET', KEYS[3], 'earliestEventId', tostring(math.floor(tonumber(oldest[2]))), 'updatedAt', ARGV[3]) + redis.call('EXPIRE', KEYS[3], tonumber(ARGV[1])) +end +return eventId +` + +/** Configuration for a durable event-log stream family (e.g. Tables). */ +export interface EventLogConfig { + /** + * Redis key prefix, e.g. `table:stream:`. STABLE per family — renaming it resets + * the seq counter, which silently strands live clients holding a higher + * in-memory `lastEventId` (their `?from=` never matches). Never change it. + */ + prefix: string + ttlSeconds: number + cap: number + /** Max entries returned by one read; the SSE route drains in chunks. */ + readChunk: number +} + +/** Minimal shape the core requires; adapters extend it with their own fields. */ +export interface EventLogEntry { + eventId: number +} + +/** + * How an adapter serializes one entry. `entryPrefix`/`entrySuffix` are spliced + * around the minted `eventId` in Lua (`prefix + eventId + suffix`); `buildMemory` + * must produce the byte-identical object for the in-memory fallback. The two MUST + * agree — a divergence makes dev/no-Redis behave differently from prod. + */ +export interface EntrySerializer { + entryPrefix: string + entrySuffix: string + buildMemory: (eventId: number) => E +} + +export type EventLogReadResult = + | { status: 'ok'; events: E[] } + | { status: 'pruned'; earliestEventId: number | undefined } + | { status: 'unavailable'; error: string } + +function eventsKey(config: EventLogConfig, streamId: string) { + return `${config.prefix}${streamId}:events` +} +function seqKey(config: EventLogConfig, streamId: string) { + return `${config.prefix}${streamId}:seq` +} +function metaKey(config: EventLogConfig, streamId: string) { + return `${config.prefix}${streamId}:meta` +} + +interface MemoryStream { + events: E[] + earliestEventId?: number + nextEventId: number + expiresAt: number +} + +/** In-memory fallback keyed by `${prefix}${streamId}`, shared across all families. */ +const memoryStreams = new Map>() + +function memoryKey(config: EventLogConfig, streamId: string) { + return `${config.prefix}${streamId}` +} + +function canUseMemoryBuffer(): boolean { + return typeof window === 'undefined' && !env.REDIS_URL +} + +function pruneExpiredMemoryStreams(now = Date.now()): void { + for (const [key, stream] of memoryStreams) { + if (stream.expiresAt <= now) memoryStreams.delete(key) + } +} + +function getMemoryStream(config: EventLogConfig, streamId: string): MemoryStream { + pruneExpiredMemoryStreams() + const key = memoryKey(config, streamId) + let stream = memoryStreams.get(key) + if (!stream) { + stream = { events: [], nextEventId: 1, expiresAt: Date.now() + config.ttlSeconds * 1000 } + memoryStreams.set(key, stream) + } + return stream +} + +/** + * Append an event. Fire-and-forget from the caller — never throws, returns null on + * failure. A Redis blip must not fail the originating mutation. + */ +export async function appendEvent( + config: EventLogConfig, + streamId: string, + serializer: EntrySerializer +): Promise { + const redis = getRedisClient() + if (!redis) { + if (canUseMemoryBuffer()) { + try { + const stream = getMemoryStream(config, streamId) + const entry = serializer.buildMemory(stream.nextEventId++) + stream.events.push(entry) + if (stream.events.length > config.cap) { + stream.events = stream.events.slice(-config.cap) + stream.earliestEventId = stream.events[0]?.eventId + } + stream.expiresAt = Date.now() + config.ttlSeconds * 1000 + return entry + } catch (error) { + logger.warn('appendEvent: memory append failed', { + streamId, + error: toError(error).message, + }) + return null + } + } + return null + } + try { + const result = await redis.eval( + APPEND_EVENT_SCRIPT, + 3, + eventsKey(config, streamId), + seqKey(config, streamId), + metaKey(config, streamId), + config.ttlSeconds, + config.cap, + new Date().toISOString(), + serializer.entryPrefix, + serializer.entrySuffix + ) + const eventId = typeof result === 'number' ? result : Number(result) + if (!Number.isFinite(eventId)) return null + return serializer.buildMemory(eventId) + } catch (error) { + logger.warn('appendEvent: Redis append failed', { streamId, error: toError(error).message }) + return null + } +} + +/** + * The latest eventId assigned for a stream, or 0 when empty/expired. Used by the + * stream route to tail from "now" when a client connects without a replay cursor. + * Redis errors propagate so the route errors the stream instead of replaying the + * whole buffer over freshly-fetched state. + */ +export async function getLatestEventId(config: EventLogConfig, streamId: string): Promise { + const redis = getRedisClient() + if (!redis) { + if (canUseMemoryBuffer()) { + const stream = memoryStreams.get(memoryKey(config, streamId)) + return stream ? stream.nextEventId - 1 : 0 + } + return 0 + } + const raw = await redis.get(seqKey(config, streamId)) + if (!raw) return 0 + const parsed = Number.parseInt(raw, 10) + return Number.isFinite(parsed) && parsed > 0 ? parsed : 0 +} + +/** + * Read events where eventId > afterEventId. Returns 'pruned' if the caller has + * fallen off the back of the buffer (TTL expired or cap rolled past their cursor); + * the caller should full-refetch and resume from the new earliest id. + */ +export async function readEventsSince( + config: EventLogConfig, + streamId: string, + afterEventId: number +): Promise> { + const redis = getRedisClient() + if (!redis) { + if (canUseMemoryBuffer()) { + pruneExpiredMemoryStreams() + const stream = memoryStreams.get(memoryKey(config, streamId)) + if (!stream) { + if (afterEventId > 0) return { status: 'pruned', earliestEventId: undefined } + return { status: 'ok', events: [] } + } + if (stream.earliestEventId !== undefined && afterEventId + 1 < stream.earliestEventId) { + return { status: 'pruned', earliestEventId: stream.earliestEventId } + } + return { + status: 'ok', + events: stream.events + .filter((entry) => entry.eventId > afterEventId) + .slice(0, config.readChunk) as E[], + } + } + return { status: 'unavailable', error: 'Redis client unavailable' } + } + try { + const meta = await redis.hgetall(metaKey(config, streamId)) + const earliestEventId = + meta?.earliestEventId !== undefined ? Number(meta.earliestEventId) : undefined + if (earliestEventId !== undefined && afterEventId + 1 < earliestEventId) { + return { status: 'pruned', earliestEventId } + } + const raw = await redis.zrangebyscore( + eventsKey(config, streamId), + afterEventId + 1, + '+inf', + 'LIMIT', + 0, + config.readChunk + ) + if (raw.length === 0 && afterEventId > 0) { + const seqExists = await redis.exists(seqKey(config, streamId)) + if (seqExists === 0) { + return { status: 'pruned', earliestEventId: undefined } + } + } + return { + status: 'ok', + events: raw + .map((entry) => { + try { + return JSON.parse(entry) as E + } catch { + return null + } + }) + .filter((entry): entry is E => entry !== null), + } + } catch (error) { + const message = toError(error).message + logger.warn('readEventsSince failed', { streamId, error: message }) + return { status: 'unavailable', error: message } + } +} + +/** Test-only: clear the in-memory streams between cases. */ +export function resetEventLogMemoryForTesting(): void { + memoryStreams.clear() +} diff --git a/apps/sim/lib/realtime/event-stream-route.ts b/apps/sim/lib/realtime/event-stream-route.ts new file mode 100644 index 00000000000..ad4fd2e8a82 --- /dev/null +++ b/apps/sim/lib/realtime/event-stream-route.ts @@ -0,0 +1,160 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { NextResponse } from 'next/server' +import { SSE_HEADERS } from '@/lib/core/utils/sse' +import type { EventLogEntry, EventLogReadResult } from '@/lib/realtime/event-log' + +const logger = createLogger('EventStreamRoute') + +const POLL_INTERVAL_MS = 500 +const HEARTBEAT_INTERVAL_MS = 15_000 +/** Defensive ceiling; the client reconnects (resuming from lastEventId) past this. */ +const MAX_STREAM_DURATION_MS = 4 * 60 * 60 * 1000 + +export interface EventStreamResponseOptions { + requestId: string + /** The durable-log stream id (e.g. a tableId). */ + streamId: string + /** Replay cursor from `?from=`; `undefined` tails from the latest event id. */ + fromEventId: number | undefined + getLatestEventId: (streamId: string) => Promise + readEventsSince: (streamId: string, afterEventId: number) => Promise> + /** Extra response headers (e.g. `{ 'X-Table-Id': id }`). */ + extraHeaders?: Record + /** Short label for logs (e.g. 'table'). */ + label: string +} + +/** + * Shared SSE stream for any durable event log (`@/lib/realtime/event-log`). Handles + * replay-on-reconnect (`?from=`), tail-from-latest on a fresh mount, chunked poll + + * forward, heartbeats, graceful `pruned`/`rotate` close, and error propagation. + * + * Auth and contract parsing stay in the route (they are domain-specific); this + * owns only the streaming mechanics, so every durable-log surface streams + * identically. The poll loop mirrors the execution stream; pub/sub wakeups are an + * optimization that can replace the 500ms poll later without changing this shape. + */ +export function createEventStreamResponse( + options: EventStreamResponseOptions +): NextResponse { + const { requestId, streamId, fromEventId, getLatestEventId, readEventsSince, label } = options + + logger.info(`[${requestId}] ${label} event stream opened`, { streamId, fromEventId }) + + const encoder = new TextEncoder() + let closed = false + + const stream = new ReadableStream({ + async start(controller) { + let lastEventId = fromEventId ?? 0 + const deadline = Date.now() + MAX_STREAM_DURATION_MS + let nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS + + const enqueue = (text: string) => { + if (closed) return + try { + controller.enqueue(encoder.encode(text)) + } catch { + closed = true + } + } + + const sendEvents = (events: E[]) => { + for (const entry of events) { + if (closed) return + enqueue(`data: ${JSON.stringify(entry)}\n\n`) + lastEventId = entry.eventId + } + } + + const sendPrunedAndClose = (earliestEventId: number | undefined) => { + enqueue( + `event: pruned\ndata: ${JSON.stringify({ earliestEventId: earliestEventId ?? null })}\n\n` + ) + if (!closed) { + closed = true + try { + controller.close() + } catch {} + } + } + + const sendHeartbeat = () => { + // SSE comment line — keeps proxies (ALB default 60s idle) from closing + // the connection during quiet periods. + enqueue(`: ping ${Date.now()}\n\n`) + } + + try { + // No replay cursor → tail from the latest event id. Resolved inside the + // try so a Redis failure errors the stream (client reconnects with + // backoff) rather than silently replaying the whole buffer. + if (fromEventId === undefined) { + lastEventId = await getLatestEventId(streamId) + } + + const initial = await readEventsSince(streamId, lastEventId) + if (initial.status === 'pruned') { + sendPrunedAndClose(initial.earliestEventId) + return + } + if (initial.status === 'unavailable') { + throw new Error(`${label} event buffer unavailable: ${initial.error}`) + } + sendEvents(initial.events) + + while (!closed && Date.now() < deadline) { + await sleep(POLL_INTERVAL_MS) + if (closed) return + + const result = await readEventsSince(streamId, lastEventId) + if (result.status === 'pruned') { + sendPrunedAndClose(result.earliestEventId) + return + } + if (result.status === 'unavailable') { + throw new Error(`${label} event buffer unavailable: ${result.error}`) + } + if (result.events.length > 0) { + sendEvents(result.events) + } + + if (Date.now() >= nextHeartbeatAt) { + sendHeartbeat() + nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS + } + } + + // Reached the defensive duration ceiling — close cleanly so the client + // reconnects with the latest lastEventId. + if (!closed) { + enqueue(`event: rotate\ndata: {}\n\n`) + closed = true + try { + controller.close() + } catch {} + } + } catch (error) { + logger.error(`[${requestId}] ${label} event stream error`, { + streamId, + error: toError(error).message, + }) + if (!closed) { + try { + controller.error(error) + } catch {} + } + } + }, + cancel() { + closed = true + logger.info(`[${requestId}] Client disconnected from ${label} event stream`, { streamId }) + }, + }) + + return new NextResponse(stream, { + headers: { ...SSE_HEADERS, ...(options.extraHeaders ?? {}) }, + }) +} diff --git a/apps/sim/lib/table/events.ts b/apps/sim/lib/table/events.ts index 7d1c8135fe0..27d9b3bb0ec 100644 --- a/apps/sim/lib/table/events.ts +++ b/apps/sim/lib/table/events.ts @@ -1,69 +1,38 @@ /** * Per-table event buffer for live cell-state updates. * - * The grid subscribes to a per-table SSE stream and patches its React Query - * cache as events arrive. This buffer is the durable mid-tier between the - * cell-write paths (`writeWorkflowGroupState`, `cancelWorkflowGroupRuns`) and - * the SSE consumers — every status transition appends here with a monotonic - * eventId; SSE clients resume on reconnect via `?from=` and the - * server replays from this buffer. + * The grid subscribes to a per-table SSE stream and patches its React Query cache + * as events arrive. This is a thin domain adapter over the generic durable event + * log (`@/lib/realtime/event-log`): it owns the Redis key prefix (`table:stream:`) + * and the entry wire shape (`{ eventId, tableId, event }`), and gets append/read/ + * tail + replay + prune semantics from the core. Every status transition appends + * here with a monotonic eventId; SSE clients resume on reconnect via + * `?from=` and the server replays from this buffer. * - * Modeled after `apps/sim/lib/execution/event-buffer.ts` but stripped of - * complexity tables don't need: no per-execution lifecycle, no id reservation - * batching, no write-queue serialization. Tables are always-on; cell writes - * are sparse and independent. + * The `table:stream:` prefix and the entry shape are a wire contract — renaming + * the prefix resets the seq counter and silently strands connected clients (their + * in-memory `lastEventId` no longer matches), so both are intentionally fixed here. */ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { env } from '@/lib/core/config/env' -import { getRedisClient } from '@/lib/core/config/redis' +import { + appendEvent, + type EventLogConfig, + type EventLogReadResult, + getLatestEventId, + readEventsSince, +} from '@/lib/realtime/event-log' -const logger = createLogger('TableEventBuffer') - -const REDIS_PREFIX = 'table:stream:' export const TABLE_EVENT_TTL_SECONDS = 60 * 60 // 1 hour export const TABLE_EVENT_CAP = 5000 /** Max events returned by a single read; the SSE route drains in chunks. */ export const TABLE_EVENT_READ_CHUNK = 500 -/** - * Atomic append: INCR the seq counter to mint a new eventId, build the entry - * JSON inline, ZADD it, refresh TTL on events + seq + meta, trim to cap, then - * write the resulting earliestEventId to meta. Single round-trip per event. - * Without atomicity a slow reader could observe the trim before the meta - * update and miss the prune signal. - * - * KEYS: [events, seq, meta] - * ARGV: [ttlSec, cap, updatedAtIso, entryPrefix, entrySuffix] - * The new eventId is spliced between prefix/suffix to form the entry JSON. - * Returns the new eventId. - */ -const APPEND_EVENT_SCRIPT = ` -local eventId = redis.call('INCR', KEYS[2]) -local entry = ARGV[4] .. eventId .. ARGV[5] -redis.call('ZADD', KEYS[1], eventId, entry) -redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1])) -redis.call('EXPIRE', KEYS[2], tonumber(ARGV[1])) -redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -tonumber(ARGV[2]) - 1) -local oldest = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES') -if oldest[2] then - redis.call('HSET', KEYS[3], 'earliestEventId', tostring(math.floor(tonumber(oldest[2]))), 'updatedAt', ARGV[3]) - redis.call('EXPIRE', KEYS[3], tonumber(ARGV[1])) -end -return eventId -` - -function getEventsKey(tableId: string) { - return `${REDIS_PREFIX}${tableId}:events` -} - -function getSeqKey(tableId: string) { - return `${REDIS_PREFIX}${tableId}:seq` -} - -function getMetaKey(tableId: string) { - return `${REDIS_PREFIX}${tableId}:meta` +/** Wire contract — see the file header; the prefix must never change. */ +const TABLE_EVENT_LOG: EventLogConfig = { + prefix: 'table:stream:', + ttlSeconds: TABLE_EVENT_TTL_SECONDS, + cap: TABLE_EVENT_CAP, + readChunk: TABLE_EVENT_READ_CHUNK, } export type TableCellStatus = 'pending' | 'queued' | 'running' | 'completed' | 'cancelled' | 'error' @@ -148,220 +117,38 @@ export interface TableEventEntry { event: TableEvent } -export type TableEventsReadResult = - | { status: 'ok'; events: TableEventEntry[] } - | { status: 'pruned'; earliestEventId: number | undefined } - | { status: 'unavailable'; error: string } - -/** In-memory fallback for dev/tests when Redis isn't configured. */ -interface MemoryTableStream { - events: TableEventEntry[] - earliestEventId?: number - nextEventId: number - expiresAt: number -} - -const memoryTableStreams = new Map() - -function canUseMemoryBuffer(): boolean { - return typeof window === 'undefined' && !env.REDIS_URL -} - -function pruneExpiredMemoryStreams(now = Date.now()): void { - for (const [tableId, stream] of memoryTableStreams) { - if (stream.expiresAt <= now) { - memoryTableStreams.delete(tableId) - } - } -} - -function getMemoryStream(tableId: string): MemoryTableStream { - pruneExpiredMemoryStreams() - let stream = memoryTableStreams.get(tableId) - if (!stream) { - stream = { - events: [], - nextEventId: 1, - expiresAt: Date.now() + TABLE_EVENT_TTL_SECONDS * 1000, - } - memoryTableStreams.set(tableId, stream) - } - return stream -} - -function appendMemory(event: TableEvent): TableEventEntry { - const stream = getMemoryStream(event.tableId) - const entry: TableEventEntry = { - eventId: stream.nextEventId++, - tableId: event.tableId, - event, - } - stream.events.push(entry) - if (stream.events.length > TABLE_EVENT_CAP) { - stream.events = stream.events.slice(-TABLE_EVENT_CAP) - stream.earliestEventId = stream.events[0]?.eventId - } - stream.expiresAt = Date.now() + TABLE_EVENT_TTL_SECONDS * 1000 - return entry -} - -function readMemory(tableId: string, afterEventId: number): TableEventsReadResult { - pruneExpiredMemoryStreams() - const stream = memoryTableStreams.get(tableId) - if (!stream) { - // Mirror the Redis path: a non-zero afterEventId with no buffer at all - // means TTL expired or the stream never existed; either way the caller's - // cursor is stale. - if (afterEventId > 0) return { status: 'pruned', earliestEventId: undefined } - return { status: 'ok', events: [] } - } - if (stream.earliestEventId !== undefined && afterEventId + 1 < stream.earliestEventId) { - return { status: 'pruned', earliestEventId: stream.earliestEventId } - } - return { - status: 'ok', - events: stream.events - .filter((entry) => entry.eventId > afterEventId) - .slice(0, TABLE_EVENT_READ_CHUNK), - } -} +export type TableEventsReadResult = EventLogReadResult /** - * Append an event to the table's buffer. Fire-and-forget from the caller — - * this never throws, returns null on failure. A Redis blip must not fail a - * cell-write. + * Append an event to the table's buffer. Fire-and-forget — never throws, returns + * null on failure (a Redis blip must not fail a cell-write). The Redis (Lua splice) + * and in-memory paths are built to produce byte-identical entries. */ export async function appendTableEvent(event: TableEvent): Promise { - const redis = getRedisClient() - if (!redis) { - if (canUseMemoryBuffer()) { - try { - return appendMemory(event) - } catch (error) { - logger.warn('appendTableEvent: memory append failed', { - tableId: event.tableId, - error: toError(error).message, - }) - return null - } - } - return null - } - try { - // Build the entry JSON in two halves so Lua can splice the new eventId - // between them without us needing a round-trip just to mint the id first. - const tail = `,"tableId":${JSON.stringify(event.tableId)},"event":${JSON.stringify(event)}}` - const head = `{"eventId":` - const result = await redis.eval( - APPEND_EVENT_SCRIPT, - 3, - getEventsKey(event.tableId), - getSeqKey(event.tableId), - getMetaKey(event.tableId), - TABLE_EVENT_TTL_SECONDS, - TABLE_EVENT_CAP, - new Date().toISOString(), - head, - tail - ) - const eventId = typeof result === 'number' ? result : Number(result) - if (!Number.isFinite(eventId)) return null - return { eventId, tableId: event.tableId, event } - } catch (error) { - logger.warn('appendTableEvent: Redis append failed', { - tableId: event.tableId, - error: toError(error).message, - }) - return null - } + return appendEvent(TABLE_EVENT_LOG, event.tableId, { + entryPrefix: '{"eventId":', + entrySuffix: `,"tableId":${JSON.stringify(event.tableId)},"event":${JSON.stringify(event)}}`, + buildMemory: (eventId) => ({ eventId, tableId: event.tableId, event }), + }) } /** * The latest eventId assigned for a table, or 0 when the buffer is empty or * expired. Used by the stream route to tail from "now" when a client connects - * without a replay cursor (fresh mount — its caches were just fetched from - * the DB, so replaying history would only rewind them). - * - * Redis errors propagate: silently falling back to 0 would replay the whole - * buffer over fresh state — the exact churn tail-from-latest exists to avoid. - * The stream route errors the stream instead and the client reconnects with - * backoff. + * without a replay cursor. */ -export async function getLatestTableEventId(tableId: string): Promise { - const redis = getRedisClient() - if (!redis) { - if (canUseMemoryBuffer()) { - // Pure read — getMemoryStream() would allocate a stream as a side effect. - const stream = memoryTableStreams.get(tableId) - return stream ? stream.nextEventId - 1 : 0 - } - return 0 - } - const raw = await redis.get(getSeqKey(tableId)) - if (!raw) return 0 - const parsed = Number.parseInt(raw, 10) - return Number.isFinite(parsed) && parsed > 0 ? parsed : 0 +export function getLatestTableEventId(tableId: string): Promise { + return getLatestEventId(TABLE_EVENT_LOG, tableId) } /** - * Read events for a table where eventId > afterEventId. Returns 'pruned' if - * the caller has fallen off the back of the buffer (TTL expired or cap rolled - * past their lastEventId). Caller should respond by full-refetching from DB - * and resuming streaming from the new earliestEventId. + * Read events for a table where eventId > afterEventId. Returns 'pruned' if the + * caller has fallen off the back of the buffer (TTL expired or cap rolled past + * their lastEventId). */ -export async function readTableEventsSince( +export function readTableEventsSince( tableId: string, afterEventId: number ): Promise { - const redis = getRedisClient() - if (!redis) { - if (canUseMemoryBuffer()) { - return readMemory(tableId, afterEventId) - } - return { status: 'unavailable', error: 'Redis client unavailable' } - } - try { - const meta = await redis.hgetall(getMetaKey(tableId)) - const earliestEventId = - meta?.earliestEventId !== undefined ? Number(meta.earliestEventId) : undefined - if (earliestEventId !== undefined && afterEventId + 1 < earliestEventId) { - return { status: 'pruned', earliestEventId } - } - // Read in capped chunks so a 5000-event backlog doesn't materialize as one - // multi-MB Redis reply + JSON parse + SSE flush. The route loop drains - // chunks across ticks. - const raw = await redis.zrangebyscore( - getEventsKey(tableId), - afterEventId + 1, - '+inf', - 'LIMIT', - 0, - TABLE_EVENT_READ_CHUNK - ) - if (raw.length === 0 && afterEventId > 0) { - // Total TTL expiry: events + meta both gone. The seq counter has the - // same TTL — its absence means the buffer was wiped and the caller's - // `afterEventId` is stale. Signal pruned so the client refetches. - const seqExists = await redis.exists(getSeqKey(tableId)) - if (seqExists === 0) { - return { status: 'pruned', earliestEventId: undefined } - } - } - return { - status: 'ok', - events: raw - .map((entry) => { - try { - return JSON.parse(entry) as TableEventEntry - } catch { - return null - } - }) - .filter((entry): entry is TableEventEntry => Boolean(entry)), - } - } catch (error) { - const message = toError(error).message - logger.warn('readTableEventsSince failed', { tableId, error: message }) - return { status: 'unavailable', error: message } - } + return readEventsSince(TABLE_EVENT_LOG, tableId, afterEventId) } From 9a899d8efecbd46bb6a6627b9b85e6a5399aa5e3 Mon Sep 17 00:00:00 2001 From: Waleed Date: Fri, 24 Jul 2026 13:47:00 -0700 Subject: [PATCH 05/53] fix(realtime): address post-merge review-comment findings (#5937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(realtime): address post-merge review-comment findings A re-audit of every inline review comment on the merged stack surfaced real issues that the thread-resolutions and prior audits missed. Fixes: Presence server (#5930 comments): - connection.ts: snapshot `socket.rooms` SYNCHRONOUSLY before the first await. Socket.IO clears the room set once the synchronous part of a `disconnecting` handler returns, so reading it after `await removeSocketFromAllRooms` saw an empty set — the eviction fallback was dead. (Cursor: "Disconnect fallback misses live rooms".) - workflow-room-service: restore the original managers' final unconditional room-state wipe via a new `deleteRoom(room)` manager method, so a deleted workflow leaves no lingering presence/meta even if a per-socket removal failed or a socket joined mid-teardown. (Cursor: "Deletion skips final room wipe".) Files (#5932 comments): - workspace-file-manager.uploadWorkspaceFile now fans out the live-tree signal (all direct-upload paths: multipart fallback, copilot create, /api/files/upload, v1 files — the presigned path already notified). (Cursor: "Creates miss live tree fan-out".) - use-workspace-files-room: clear the pending retry timer on join success; and a module-scoped intended-room guard defers the unmount `leave` so a rapid remount re-claims the room and skips a stale leave — fixing presence flap + a leave-after-join race. (Cursor: "Retry timer survives join success" + "Remount churns files presence".) - workspace-files handler: roll back a partial join (leave room + remove presence) in the catch, mirroring the workflow join. (Cursor: "Join failure skips membership rollback".) +2 tests (deleteRoom). 127 realtime tests pass, both apps tsc clean, api-validation + boundaries green. * fix(files): scope workspace-files leave to a workspace (deferred-leave safety) Self-review of the deferred-leave guard found a real bug: leave-workspace-files was not workspace-scoped, so after a workspace switch (A->B) the deferred leave from A would evict the socket from its new room B. The leave now carries the workspaceId and the server no-ops if the socket's current files room differs. Also excludes the leaving socket from the leave broadcast (consistent with disconnect). * fix(realtime): close files-room presence leak + validate join payload Architecture-audit findings: - S1 (real Redis leak): the files room inherited the shared manager but not the workflow join's liveness sweep, so an UNGRACEFUL disconnect (pod crash — no `disconnecting` event) left its presence entry in the no-TTL room hash forever. Added a shared `sweepStalePresence(manager, room)` (fetchSockets liveness + remove not-live-AND-stale entries, matching the workflow 75min threshold) and run it on files join; also filter the join ack through `filterVisiblePresence` so a joiner never briefly sees an un-swept ghost. - S2: validate the client-supplied `workspaceId` on files join before it reaches the DB query (matches the /api/workspace-files-changed guard; fails closed). - N2: corrected the notify doc — it is awaited (guaranteed dispatch before a Node route returns) and hard-bounded to NOTIFY_TIMEOUT_MS, not "never block". +1 test (sweepStalePresence keeps live/fresh, reclaims not-live-stale). 128 realtime tests pass, both apps tsc clean, biome clean. * fix(realtime): workflow-deletion always notifies + cleans by socket.io membership Review-round findings on #5937: - Always emit `workflow-deleted` (was guarded by users.length>0), so a socket still in the Socket.IO room after a Redis presence eviction is told the workflow is gone before socketsLeave kicks it — the editor no longer keeps showing a deleted workflow. (Cursor: "Silent kick skips deletion event".) - Clean per-socket state for the UNION of live Socket.IO members and presence-tracked sockets, so an evicted/late-joined socket's room mapping + session are dropped too — not just presence-snapshot sockets. (Greptile: "Room deletion leaves reverse state".) - deleteRoom now logs AND rethrows on Redis failure (like addUserToRoom) so a failed wipe isn't reported as a clean deletion; the request surfaces it. (Greptile: "Room deletion failures are suppressed".) The two "deferred leave drops new membership" P1s were already fixed by the workspace-scoped leave in a prior commit (leave carries { workspaceId }; server no-ops on mismatch). 128 tests pass, tsc + biome clean. * refactor(files): drop module-scoped deferred-leave; rely on workspace-scoped leave Removes the one non-idiomatic construct (a module-level mutable `intendedFilesWorkspaceId` + queueMicrotask). It only guarded a same-workspace CONCURRENT remount, which doesn't occur in production (folder nav is shallow/no remount; list<->detail is sequential) — a dev-StrictMode-only case. The real cross-workspace race is already handled by the workspace-scoped leave: if B's join runs first (auto-leaving A), A's leave no-ops because the socket's current files room is B. Simpler, idiomatic, prod-correct. --- apps/realtime/src/handlers/connection.ts | 11 ++++- apps/realtime/src/handlers/workflow.test.ts | 1 + .../src/handlers/workspace-files.test.ts | 1 + apps/realtime/src/handlers/workspace-files.ts | 42 +++++++++++++++++-- .../realtime/src/rooms/memory-manager.test.ts | 34 +++++++++++++++ apps/realtime/src/rooms/memory-manager.ts | 4 ++ .../realtime/src/rooms/presence-visibility.ts | 38 +++++++++++++++++ apps/realtime/src/rooms/redis-manager.ts | 11 +++++ apps/realtime/src/rooms/types.ts | 7 ++++ .../src/rooms/workflow-room-service.ts | 35 +++++++++++----- .../files/hooks/use-workspace-files-room.ts | 13 +++++- apps/sim/lib/realtime/notify.ts | 11 +++-- .../workspace/workspace-file-manager.ts | 6 +++ 13 files changed, 195 insertions(+), 19 deletions(-) diff --git a/apps/realtime/src/handlers/connection.ts b/apps/realtime/src/handlers/connection.ts index 2f180e7fa53..8b0d646ee73 100644 --- a/apps/realtime/src/handlers/connection.ts +++ b/apps/realtime/src/handlers/connection.ts @@ -21,6 +21,13 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager // evicted or TTL-expired (which would leave the manager's stored rooms empty). socket.on('disconnecting', async (reason) => { try { + // Snapshot the live Socket.IO room membership SYNCHRONOUSLY, before any + // await: Socket.IO clears `socket.rooms` via leaveAll() as soon as the + // synchronous portion of this `disconnecting` handler returns (i.e. at the + // first await below), so reading it afterwards would see an empty set and + // the eviction fallback would be dead. + const liveRoomNames = [...socket.rooms] + // Clean up pending debounce entries for this socket to prevent memory leaks cleanupPendingSubblocksForSocket(socket.id) cleanupPendingVariablesForSocket(socket.id) @@ -29,13 +36,13 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager // room the manager knows about. const removedRooms = await roomManager.removeSocketFromAllRooms(socket.id) - // Union with the live Socket.IO membership (authoritative here, and it + // Union with the snapshotted Socket.IO membership (authoritative, and it // survives a Redis eviction/TTL lapse that would leave the manager's tracked // rooms empty). Attempt removal for any room the manager didn't already // remove — best-effort, since a transient Redis error can't be recovered here. const wasInRooms = new Map() for (const room of removedRooms) wasInRooms.set(roomName(room), room) - for (const name of socket.rooms) { + for (const name of liveRoomNames) { // `wasInRooms.has(name)` already excludes every room the manager removed // (same room-name key via the roomName/parseRoomName bijection), so any // room reaching here was NOT in `removedRooms` and needs a removal attempt. diff --git a/apps/realtime/src/handlers/workflow.test.ts b/apps/realtime/src/handlers/workflow.test.ts index 89ae5521e2f..417960c6e5d 100644 --- a/apps/realtime/src/handlers/workflow.test.ts +++ b/apps/realtime/src/handlers/workflow.test.ts @@ -61,6 +61,7 @@ function createRoomManager(overrides?: Partial): IRoomManager { broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), getRoomUsers: vi.fn().mockResolvedValue([]), hasRoom: vi.fn().mockResolvedValue(false), + deleteRoom: vi.fn().mockResolvedValue(undefined), addUserToRoom: vi.fn().mockResolvedValue(undefined), getUserSession: vi.fn().mockResolvedValue(null), updateUserActivity: vi.fn().mockResolvedValue(undefined), diff --git a/apps/realtime/src/handlers/workspace-files.test.ts b/apps/realtime/src/handlers/workspace-files.test.ts index 9ecc943c8bb..76f20d9a7e5 100644 --- a/apps/realtime/src/handlers/workspace-files.test.ts +++ b/apps/realtime/src/handlers/workspace-files.test.ts @@ -55,6 +55,7 @@ function createRoomManager(overrides?: Partial): IRoomManager { broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), getRoomUsers: vi.fn().mockResolvedValue([]), hasRoom: vi.fn().mockResolvedValue(false), + deleteRoom: vi.fn().mockResolvedValue(undefined), addUserToRoom: vi.fn().mockResolvedValue(undefined), getUserSession: vi.fn().mockResolvedValue(null), updateUserActivity: vi.fn().mockResolvedValue(undefined), diff --git a/apps/realtime/src/handlers/workspace-files.ts b/apps/realtime/src/handlers/workspace-files.ts index c88d403e44e..964857e0633 100644 --- a/apps/realtime/src/handlers/workspace-files.ts +++ b/apps/realtime/src/handlers/workspace-files.ts @@ -5,6 +5,7 @@ import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms import { eq } from 'drizzle-orm' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager, UserPresence } from '@/rooms' +import { filterVisiblePresence, sweepStalePresence } from '@/rooms/presence-visibility' const logger = createLogger('WorkspaceFilesHandlers') @@ -77,6 +78,19 @@ export function setupWorkspaceFilesHandlers( return } + // Validate the client-supplied id before it reaches the DB query (matches + // the /api/workspace-files-changed guard; join payloads are otherwise raw + // client input). + if (typeof workspaceId !== 'string' || workspaceId.length === 0) { + socket.emit('join-workspace-files-error', { + workspaceId: typeof workspaceId === 'string' ? workspaceId : '', + error: 'Invalid workspace id', + code: 'INVALID_PAYLOAD', + retryable: false, + }) + return + } + const room = filesRoom(workspaceId) let authorized: Awaited> @@ -130,6 +144,10 @@ export function setupWorkspaceFilesHandlers( } } + // Reclaim any presence orphaned by an ungraceful disconnect (pod crash + // fires no `disconnecting` event; the room hashes have no TTL). + await sweepStalePresence(roomManager, room) + socket.join(roomName(room)) const presence: UserPresence = { @@ -147,7 +165,13 @@ export function setupWorkspaceFilesHandlers( await roomManager.addUserToRoom(room, socket.id, presence) - const presenceUsers = await roomManager.getRoomUsers(room) + // Filter the join ack to live members so a new joiner never briefly sees a + // ghost from an entry the sweep hasn't reclaimed yet. + const presenceUsers = await filterVisiblePresence( + roomManager.io, + room, + await roomManager.getRoomUsers(room) + ) socket.emit('join-workspace-files-success', { workspaceId, socketId: socket.id, @@ -159,6 +183,14 @@ export function setupWorkspaceFilesHandlers( logger.info(`User ${userId} (${userName}) joined files room for workspace ${workspaceId}`) } catch (error) { logger.error('Error joining workspace files room:', error) + // Roll back any partial join so a failed attempt can't leave the socket in + // the Socket.IO room or a stale presence entry behind (mirrors the workflow + // join's rollback), before signalling a retryable failure. + try { + const room = filesRoom(workspaceId) + socket.leave(roomName(room)) + await roomManager.removeUserFromRoom(room, socket.id) + } catch {} socket.emit('join-workspace-files-error', { workspaceId, error: 'Failed to join workspace files', @@ -169,14 +201,18 @@ export function setupWorkspaceFilesHandlers( } ) - socket.on('leave-workspace-files', async () => { + socket.on('leave-workspace-files', async (payload?: { workspaceId?: string }) => { try { if (!roomManager.isReady()) return const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKSPACE_FILES) if (!room) return + // Scope the leave to a specific workspace when the client provides one: a + // deferred leave from a prior page must not evict the socket from a room it + // has since switched into (workspace A→B leaves A's leave targeting B). + if (payload?.workspaceId && payload.workspaceId !== room.id) return socket.leave(roomName(room)) await roomManager.removeUserFromRoom(room, socket.id) - await roomManager.broadcastPresenceUpdate(room) + await roomManager.broadcastPresenceUpdate(room, socket.id) } catch (error) { logger.error('Error leaving workspace files room:', error) } diff --git a/apps/realtime/src/rooms/memory-manager.test.ts b/apps/realtime/src/rooms/memory-manager.test.ts index d5d2512a2d8..af98414f540 100644 --- a/apps/realtime/src/rooms/memory-manager.test.ts +++ b/apps/realtime/src/rooms/memory-manager.test.ts @@ -8,6 +8,7 @@ import { ROOM_TYPES, type RoomRef } from '@sim/realtime-protocol/rooms' import { beforeEach, describe, expect, it, vi } from 'vitest' import { MemoryRoomManager } from '@/rooms/memory-manager' +import { sweepStalePresence } from '@/rooms/presence-visibility' import type { UserPresence } from '@/rooms/types' function fakeIo(liveSocketIds: string[] = []) { @@ -120,6 +121,39 @@ describe('MemoryRoomManager multi-room', () => { expect(await manager.getUserSession('socket-2')).not.toBeNull() }) + it('sweepStalePresence reclaims not-live stale entries but keeps live and fresh ones', async () => { + const { io } = fakeIo(['socket-live']) + const m = new MemoryRoomManager(io) + await m.initialize() + + const staleMs = 76 * 60 * 1000 + await m.addUserToRoom(FILES, 'socket-live', presence(FILES, 'socket-live', 'u1')) + await m.addUserToRoom(FILES, 'socket-dead', { + ...presence(FILES, 'socket-dead', 'u2'), + joinedAt: Date.now() - staleMs, + lastActivity: Date.now() - staleMs, + }) + await m.addUserToRoom(FILES, 'socket-recent', presence(FILES, 'socket-recent', 'u3')) + + await sweepStalePresence(m, FILES) + + const remaining = (await m.getRoomUsers(FILES)).map((u) => u.socketId).sort() + // socket-dead: not live + stale → removed. socket-live: live → kept. + // socket-recent: not live but fresh (transient) → kept. + expect(remaining).toEqual(['socket-live', 'socket-recent']) + }) + + it('deleteRoom unconditionally drops all room state', async () => { + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + expect(await manager.hasRoom(FILES)).toBe(true) + + await manager.deleteRoom(FILES) + + expect(await manager.hasRoom(FILES)).toBe(false) + expect(await manager.getRoomUsers(FILES)).toHaveLength(0) + }) + it('ignores removal of a room the socket is not in (id-guarded)', async () => { await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) diff --git a/apps/realtime/src/rooms/memory-manager.ts b/apps/realtime/src/rooms/memory-manager.ts index 0e25921a47e..90001f53b53 100644 --- a/apps/realtime/src/rooms/memory-manager.ts +++ b/apps/realtime/src/rooms/memory-manager.ts @@ -149,6 +149,10 @@ export class MemoryRoomManager implements IRoomManager { return this.rooms.has(roomKey(room)) } + async deleteRoom(room: RoomRef): Promise { + this.rooms.delete(roomKey(room)) + } + async updateUserActivity( room: RoomRef, socketId: string, diff --git a/apps/realtime/src/rooms/presence-visibility.ts b/apps/realtime/src/rooms/presence-visibility.ts index 4680a581964..7e43b9fde29 100644 --- a/apps/realtime/src/rooms/presence-visibility.ts +++ b/apps/realtime/src/rooms/presence-visibility.ts @@ -1,5 +1,14 @@ import { type RoomRef, roomName } from '@sim/realtime-protocol/rooms' import type { Server } from 'socket.io' +import type { IRoomManager } from '@/rooms/types' + +/** + * How stale a not-live presence entry must be before a join-time sweep reclaims + * it. Kept above the 1h socket-key TTL so a normally-idle collaborator is never + * evicted; only genuinely-orphaned entries (e.g. a crashed pod that never fired + * `disconnecting`) are cleared. Matches the workflow join sweep. + */ +const STALE_PRESENCE_THRESHOLD_MS = 75 * 60 * 1000 /** * Filters a room's stored presence down to what should actually be broadcast: @@ -34,3 +43,32 @@ export async function filterVisiblePresence( return candidates } } + +/** + * Reclaims orphaned presence entries in a room: any stored socket that is no + * longer a live Socket.IO member AND has been idle past + * {@link STALE_PRESENCE_THRESHOLD_MS} is removed. This is how a room-users hash + * (which has no TTL) is bounded against ungraceful disconnects — a pod crash + * fires no `disconnecting` event, so its entries would otherwise persist forever. + * Run on join, like the workflow room does. No-op when the liveness lookup fails + * (so a transient adapter blip can't evict live collaborators). + */ +export async function sweepStalePresence(manager: IRoomManager, room: RoomRef): Promise { + let liveIds: Set + try { + const liveSockets = await manager.io.in(roomName(room)).fetchSockets() + liveIds = new Set(liveSockets.map((socket) => socket.id)) + } catch { + return + } + + const now = Date.now() + const users = await manager.getRoomUsers(room) + for (const user of users) { + if (liveIds.has(user.socketId)) continue + const lastSeen = user.lastActivity || user.joinedAt || 0 + if (now - lastSeen > STALE_PRESENCE_THRESHOLD_MS) { + await manager.removeUserFromRoom(room, user.socketId) + } + } +} diff --git a/apps/realtime/src/rooms/redis-manager.ts b/apps/realtime/src/rooms/redis-manager.ts index 2de65bc14c5..945d03d539c 100644 --- a/apps/realtime/src/rooms/redis-manager.ts +++ b/apps/realtime/src/rooms/redis-manager.ts @@ -311,6 +311,17 @@ export class RedisRoomManager implements IRoomManager { return exists > 0 } + async deleteRoom(room: RoomRef): Promise { + // Log AND rethrow (like addUserToRoom): a failed wipe must not be reported as a + // clean deletion by the caller — the request surfaces it (and can be retried). + try { + await this.redis.del([KEYS.roomUsers(room), KEYS.roomMeta(room)]) + } catch (error) { + logger.error(`Failed to delete room ${room.type}:${room.id}:`, error) + throw error + } + } + async updateUserActivity( room: RoomRef, socketId: string, diff --git a/apps/realtime/src/rooms/types.ts b/apps/realtime/src/rooms/types.ts index 80525195ef8..c9c8ee18704 100644 --- a/apps/realtime/src/rooms/types.ts +++ b/apps/realtime/src/rooms/types.ts @@ -101,6 +101,13 @@ export interface IRoomManager { /** Whether a room currently has any presence. */ hasRoom(room: RoomRef): Promise + /** + * Unconditionally drop all state for a room (presence + metadata). Used when a + * room's underlying resource is destroyed (e.g. a deleted workflow) to guarantee + * no state lingers even if per-socket removals failed or a socket joined mid-teardown. + */ + deleteRoom(room: RoomRef): Promise + /** Update a socket's activity (cursor, selection, lastActivity) within a room. */ updateUserActivity( room: RoomRef, diff --git a/apps/realtime/src/rooms/workflow-room-service.ts b/apps/realtime/src/rooms/workflow-room-service.ts index 4623647a7da..f12a6c337d4 100644 --- a/apps/realtime/src/rooms/workflow-room-service.ts +++ b/apps/realtime/src/rooms/workflow-room-service.ts @@ -22,29 +22,44 @@ export class WorkflowRoomService { logger.info(`Handling workflow deletion notification for ${workflowId}`) const room = workflowRoom(workflowId) - const users = await this.manager.getRoomUsers(room) - if (users.length === 0) { - logger.debug(`No active users found for deleted workflow ${workflowId}`) - return - } + const name = roomName(room) + // Always notify — reach every socket still in the Socket.IO room so the client + // clears the deleted workflow, even if that socket's Redis presence was evicted + // (in which case it would be missing from getRoomUsers). Emitting to an empty + // room is a harmless no-op. this.manager.emitToRoom(room, 'workflow-deleted', { workflowId, message: 'This workflow has been deleted', timestamp: Date.now(), }) + // Clean per-socket state for every socket that is either a live Socket.IO member + // OR still has presence — so an evicted/late-joined socket's room mapping and + // session are dropped too, not just the presence-tracked ones. + const socketIds = new Set() + try { + const liveSockets = await this.manager.io.in(name).fetchSockets() + for (const s of liveSockets) socketIds.add(s.id) + } catch (error) { + logger.warn(`Could not enumerate sockets for deleted workflow ${workflowId}`, error) + } + for (const user of await this.manager.getRoomUsers(room)) socketIds.add(user.socketId) + // Remove every socket from the Socket.IO room (cross-pod via the Redis adapter). - const name = roomName(room) await this.manager.io.in(name).socketsLeave(name) - // Drop presence state for each socket; empty-room cleanup is handled by the manager. - for (const user of users) { - await this.manager.removeUserFromRoom(room, user.socketId) + for (const socketId of socketIds) { + await this.manager.removeUserFromRoom(room, socketId) } + // Final unconditional wipe — the workflow is gone, so no room state may linger + // even if a per-socket removal failed (matches the pre-refactor managers, which + // ended deletion with an unconditional room drop). + await this.manager.deleteRoom(room) + logger.info( - `Cleaned up workflow room ${workflowId} after deletion (${users.length} users disconnected)` + `Cleaned up workflow room ${workflowId} after deletion (${socketIds.size} sockets removed)` ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts index e2c4a61befc..795d233f4e0 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts @@ -77,6 +77,12 @@ export function useWorkspaceFilesRoom( }) => { if (data.workspaceId !== workspaceId) return retries = 0 + // Cancel any retry scheduled by a prior retryable error so it can't fire an + // extra join after we're already in. + if (retryTimer) { + clearTimeout(retryTimer) + retryTimer = null + } setPresenceUsers(data.presenceUsers ?? []) } const handleJoinError = (data: JoinErrorPayload) => { @@ -103,13 +109,18 @@ export function useWorkspaceFilesRoom( return () => { if (retryTimer) clearTimeout(retryTimer) - socket.emit('leave-workspace-files') socket.off('connect', join) socket.off('join-workspace-files-success', handleJoinSuccess) socket.off('join-workspace-files-error', handleJoinError) socket.off('workspace-files:presence-update', handlePresence) socket.off('workspace-files-changed', handleChanged) setPresenceUsers([]) + + // Leave the room, scoped to THIS workspace: the server no-ops if the socket + // has already switched to another workspace's files room (so a workspace + // A→B switch, where B's join runs first and auto-leaves A, can't have A's + // leave evict the fresh B membership). + socket.emit('leave-workspace-files', { workspaceId }) } }, [socket, workspaceId, queryClient]) diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts index e738130b6de..71ef65c1ced 100644 --- a/apps/sim/lib/realtime/notify.ts +++ b/apps/sim/lib/realtime/notify.ts @@ -11,9 +11,14 @@ const NOTIFY_TIMEOUT_MS = 2000 /** * Best-effort fan-out to the realtime server that a workspace's file tree changed, * so every browser currently viewing that workspace's files refetches. File - * mutations happen over the HTTP API (not the socket); this is the lossy liveness - * signal — a dropped notification only degrades to stale-until-refetch, so it must - * never throw or block the originating mutation. + * mutations happen over the HTTP API (not the socket); this is a lossy liveness + * signal — a dropped notification only degrades to stale-until-refetch. + * + * Never throws. Callers `await` it (rather than fire-and-forget) so the fetch is + * guaranteed to dispatch before a Node route handler returns — a floating promise + * can be dropped after the response is sent. It is a normally-sub-millisecond + * local call and is hard-bounded to {@link NOTIFY_TIMEOUT_MS}, so it adds that + * latency only when the socket pod is unreachable. */ export async function notifyWorkspaceFilesChanged(workspaceId: string): Promise { try { diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 6967a7d313f..662885c2ec5 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -23,6 +23,7 @@ import { resolveWorkflowAliasForWorkspace } from '@/lib/copilot/vfs/workflow-ali import { isReservedWorkflowAliasBackingDisplayPath } from '@/lib/copilot/vfs/workflow-aliases' import { generateRestoreName } from '@/lib/core/utils/restore-name' import type { DbOrTx } from '@/lib/db/types' +import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { getServePathPrefix } from '@/lib/uploads' import { deleteFile, @@ -380,6 +381,11 @@ export async function uploadWorkspaceFile( const pathPrefix = getServePathPrefix() const serveUrl = `${pathPrefix}${encodeURIComponent(uploadResult.key)}?context=workspace` + // Fan out the live-tree signal for the direct-upload paths (multipart + // fallback, copilot create, /api/files/upload, v1 files) — the presigned + // path already notifies from its register route. + await notifyWorkspaceFilesChanged(workspaceId) + return { id: fileId, name: uniqueName, From a1bb9f329c2320bca6fe029fb3336bda1de33020 Mon Sep 17 00:00:00 2001 From: Waleed Date: Fri, 24 Jul 2026 16:36:55 -0700 Subject: [PATCH 06/53] feat(realtime): Yjs relay server for collaborative document editing [4/N] (#5941) Server-side Yjs relay for collaborative document editing (live carets + text selection) in the Files rich-markdown editor. Faithful y-websocket-style relay over the existing authenticated Socket.IO connection + shared room abstraction; in-memory Y.Doc + Awareness per file; awareness ownership binding, userId-keyed client-id uniqueness, seeder election with deadline re-election, concurrent-JOIN generation guard. 25 relay tests. Reviewed to Greptile 5/5 + Cursor pass across multiple rounds, plus an independent 4-lens audit (correctness/security/conventions/simplicity) and /simplify + /cleanup passes. --- apps/realtime/package.json | 3 + apps/realtime/src/handlers/connection.ts | 4 + apps/realtime/src/handlers/file-doc.test.ts | 577 +++++++++++++++++++ apps/realtime/src/handlers/file-doc.ts | 514 +++++++++++++++++ apps/realtime/src/handlers/index.ts | 2 + bun.lock | 11 + packages/platform-authz/src/rooms.ts | 22 +- packages/realtime-protocol/package.json | 4 + packages/realtime-protocol/src/file-doc.ts | 134 +++++ packages/realtime-protocol/src/rooms.test.ts | 15 + packages/realtime-protocol/src/rooms.ts | 7 + 11 files changed, 1292 insertions(+), 1 deletion(-) create mode 100644 apps/realtime/src/handlers/file-doc.test.ts create mode 100644 apps/realtime/src/handlers/file-doc.ts create mode 100644 packages/realtime-protocol/src/file-doc.ts diff --git a/apps/realtime/package.json b/apps/realtime/package.json index 633ffb60827..35e45fc29e6 100644 --- a/apps/realtime/package.json +++ b/apps/realtime/package.json @@ -33,9 +33,12 @@ "@sim/workflow-types": "workspace:*", "@socket.io/redis-adapter": "8.3.0", "drizzle-orm": "^0.45.2", + "lib0": "0.2.117", "postgres": "^3.4.5", "redis": "5.10.0", "socket.io": "^4.8.1", + "y-protocols": "1.0.7", + "yjs": "13.6.31", "zod": "4.3.6" }, "devDependencies": { diff --git a/apps/realtime/src/handlers/connection.ts b/apps/realtime/src/handlers/connection.ts index 8b0d646ee73..aa134d8a771 100644 --- a/apps/realtime/src/handlers/connection.ts +++ b/apps/realtime/src/handlers/connection.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { parseRoomName, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { cleanupFileDocForSocket } from '@/handlers/file-doc' import { cleanupPendingSubblocksForSocket } from '@/handlers/subblocks' import { cleanupPendingVariablesForSocket } from '@/handlers/variables' import type { AuthenticatedSocket } from '@/middleware/auth' @@ -31,6 +32,9 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager // Clean up pending debounce entries for this socket to prevent memory leaks cleanupPendingSubblocksForSocket(socket.id) cleanupPendingVariablesForSocket(socket.id) + // Clear the socket's collaborative-document awareness (removes its caret for + // everyone else) and drop the room if it was the last editor. + cleanupFileDocForSocket(socket.id, roomManager.io) // A socket may occupy multiple rooms (one per type). Remove it from every // room the manager knows about. diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts new file mode 100644 index 00000000000..48da76f22ac --- /dev/null +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -0,0 +1,577 @@ +/** + * @vitest-environment node + */ +import { FILE_DOC_EVENTS, FILE_DOC_MESSAGE_TYPE } from '@sim/realtime-protocol/file-doc' +import * as encoding from 'lib0/encoding' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as awarenessProtocol from 'y-protocols/awareness' +import * as syncProtocol from 'y-protocols/sync' +import * as Y from 'yjs' +import type { IRoomManager } from '@/rooms' + +const { mockAuthorizeRoom } = vi.hoisted(() => ({ + mockAuthorizeRoom: vi.fn(), +})) + +vi.mock('@sim/platform-authz/rooms', () => ({ + authorizeRoom: mockAuthorizeRoom, +})) + +import { cleanupFileDocForSocket, setupWorkspaceFileDocHandlers } from '@/handlers/file-doc' + +type Handler = (payload?: unknown) => Promise | void + +const ROOM_NAME = 'workspace-file-doc:file-1' + +interface SentMessage { + target: string + except?: string + event: string + payload: unknown +} + +/** An `io` mock that records every server-originated emit with its target/except. */ +function createIo() { + const sent: SentMessage[] = [] + const to = vi.fn((target: string) => ({ + except: (exclude: string) => ({ + emit: (event: string, payload: unknown) => + sent.push({ target, except: exclude, event, payload }), + }), + emit: (event: string, payload: unknown) => sent.push({ target, event, payload }), + })) + return { io: { to } as unknown as IRoomManager['io'], sent } +} + +/** Every socket id a test created, so `afterEach` can drop their rooms without a + * hardcoded list drifting out of sync with the tests. */ +const createdSocketIds = new Set() + +function createSocket(id: string, overrides?: Record) { + createdSocketIds.add(id) + const handlers: Record = {} + const socket = { + id, + userId: 'user-1', + userName: 'Test User', + disconnected: false, + on: vi.fn((event: string, handler: Handler) => { + handlers[event] = handler + }), + emit: vi.fn(), + join: vi.fn(), + leave: vi.fn(), + ...overrides, + } + return { handlers, socket } +} + +function createRoomManager( + io: IRoomManager['io'], + overrides?: Partial +): IRoomManager { + return { + isReady: vi.fn().mockReturnValue(true), + io, + ...overrides, + } as unknown as IRoomManager +} + +function setup(id: string, io: IRoomManager['io'], socketOverrides?: Record) { + const { socket, handlers } = createSocket(id, socketOverrides) + setupWorkspaceFileDocHandlers( + socket as unknown as Parameters[0], + createRoomManager(io) + ) + return { socket, handlers } +} + +/** Frame a Yjs message with its type tag, exactly as the client provider would. */ +function frame(type: number, write: (encoder: encoding.Encoder) => void): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, type) + write(encoder) + return encoding.toUint8Array(encoder) +} + +/** Build a real awareness frame carrying a single client's state. */ +function awarenessFrame(clientId: number, name: string): { frame: Uint8Array; clientId: number } { + const doc = new Y.Doc() + // Force a specific clientID so the test can bind/spoof deliberately. + doc.clientID = clientId + const awareness = new awarenessProtocol.Awareness(doc) + awareness.setLocalStateField('user', { name }) + const update = awarenessProtocol.encodeAwarenessUpdate(awareness, [clientId]) + return { + frame: frame(FILE_DOC_MESSAGE_TYPE.AWARENESS, (e) => encoding.writeVarUint8Array(e, update)), + clientId, + } +} + +function joinSuccessFileId(socket: { emit: ReturnType }) { + const calls = socket.emit.mock.calls.filter( + (call: unknown[]) => call[0] === FILE_DOC_EVENTS.JOIN_SUCCESS + ) + const last = calls[calls.length - 1] + return (last?.[1] as { fileId: string } | undefined)?.fileId +} + +describe('setupWorkspaceFileDocHandlers', () => { + beforeEach(() => { + vi.clearAllMocks() + // The seed deadline uses setTimeout; fake it so tests can drive it and so a + // real timer can never fire into a later test. + vi.useFakeTimers() + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'write', + }) + }) + + afterEach(() => { + // The room store is module-global; drop every room the test's sockets opened. + const { io } = createIo() + for (const id of createdSocketIds) cleanupFileDocForSocket(id, io) + createdSocketIds.clear() + vi.clearAllTimers() + vi.useRealTimers() + }) + + it('rejects join when the socket is not authenticated', async () => { + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io, { userId: undefined, userName: undefined }) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'AUTHENTICATION_REQUIRED', retryable: false }) + ) + }) + + it('rejects join with a retryable error when realtime is unavailable', async () => { + const { io } = createIo() + const { socket, handlers } = createSocket('socket-1') + setupWorkspaceFileDocHandlers( + socket as unknown as Parameters[0], + createRoomManager(io, { isReady: vi.fn().mockReturnValue(false) }) + ) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) + ) + }) + + it('rejects a payload missing the file id or client id before authorizing', async () => { + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: '', clientId: 1 }) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'INVALID_PAYLOAD', retryable: false }) + ) + expect(mockAuthorizeRoom).not.toHaveBeenCalled() + }) + + it('requires write permission and reports 404 as NOT_FOUND', async () => { + mockAuthorizeRoom.mockResolvedValue({ allowed: false, status: 404, workspacePermission: null }) + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(mockAuthorizeRoom).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'write', + room: { type: 'workspace-file-doc', id: 'file-1' }, + }) + ) + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'NOT_FOUND', retryable: false }) + ) + }) + + it('joins the room, sends sync step 1, and asks the first client to seed', async () => { + const { io, sent } = createIo() + const { socket, handlers } = setup('socket-1', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(socket.join).toHaveBeenCalledWith(ROOM_NAME) + expect(joinSuccessFileId(socket)).toBe('file-1') + + // A binary sync-step-1 message (type tag 0) is sent to kick off the handshake. + const syncMessage = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + expect((syncMessage?.[1] as Uint8Array)[0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + + // The lone joiner is elected to seed the empty document. + const seed = sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) + expect(seed).toEqual({ + target: 'socket-1', + event: FILE_DOC_EVENTS.SEED_REQUEST, + payload: { fileId: 'file-1' }, + }) + }) + + it('asks only one client to seed across concurrent joiners of the same file', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + const b = setup('socket-b', io) + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + const seeds = sent.filter((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) + expect(seeds).toHaveLength(1) + expect(seeds[0].target).toBe('socket-a') + }) + + it('relays a document update to the rest of the room, excluding the sender', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + const b = setup('socket-b', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + sent.length = 0 + + const clientDoc = new Y.Doc() + clientDoc.getText('default').insert(0, 'hello') + const update = Y.encodeStateAsUpdate(clientDoc) + a.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeUpdate(e, update)) + ) + + const relayed = sent.find((m) => m.event === FILE_DOC_EVENTS.MESSAGE) + expect(relayed?.target).toBe(ROOM_NAME) + expect(relayed?.except).toBe('socket-a') + expect((relayed?.payload as Uint8Array)[0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + }) + + it('relays an owned awareness update to the room, excluding the sender', async () => { + const { io, sent } = createIo() + const { frame: awFrame, clientId } = awarenessFrame(4242, 'Ada') + const a = setup('socket-a', io) + const b = setup('socket-b', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + sent.length = 0 + + a.handlers[FILE_DOC_EVENTS.MESSAGE](awFrame) + + const relayed = sent.find( + (m) => + m.event === FILE_DOC_EVENTS.MESSAGE && + (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS + ) + expect(relayed?.except).toBe('socket-a') + }) + + it('drops an awareness frame that spoofs another client id', async () => { + const { io, sent } = createIo() + // socket-a binds client id 100 at join, but sends awareness for client 999. + const { frame: spoof } = awarenessFrame(999, 'Mallory') + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 100 }) + sent.length = 0 + + a.handlers[FILE_DOC_EVENTS.MESSAGE](spoof) + + const relayed = sent.find( + (m) => + m.event === FILE_DOC_EVENTS.MESSAGE && + (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS + ) + expect(relayed).toBeUndefined() + }) + + it("rejects a DIFFERENT user binding a peer's client id (spoof)", async () => { + const { io } = createIo() + const a = setup('socket-a', io) + const b = setup('socket-b', io, { userId: 'attacker' }) + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + + expect(b.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'CLIENT_ID_IN_USE' }) + ) + expect(b.socket.join).not.toHaveBeenCalled() + }) + + it('reclaims a client id for the SAME user reconnecting (reused Yjs client id)', async () => { + const { io } = createIo() + // The same user's dropped socket still owns client id 7 (its disconnect + // cleanup has not run yet) when it reconnects on a new socket reusing id 7. + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + const b = setup('socket-b', io) // same default userId 'user-1' + + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + + expect(joinSuccessFileId(b.socket)).toBe('file-1') + expect(b.socket.join).toHaveBeenCalledWith(ROOM_NAME) + }) + + it('clears a departed caret when a socket rejoins the room with a new client id', async () => { + const { io, sent } = createIo() + const { frame: awFrame } = awarenessFrame(500, 'A') + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 500 }) + a.handlers[FILE_DOC_EVENTS.MESSAGE](awFrame) + sent.length = 0 + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 501 }) + + // The old client (500) caret removal is broadcast to the room. + const removal = sent.find( + (m) => + m.event === FILE_DOC_EVENTS.MESSAGE && + (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS + ) + expect(removal).toBeDefined() + }) + + it('preserves the existing caret when a rebind to a foreign client id is rejected', async () => { + const { io, sent } = createIo() + const { frame: awFrame } = awarenessFrame(10, 'A') + const a = setup('socket-a', io) + const b = setup('socket-b', io, { userId: 'user-b' }) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 10 }) + a.handlers[FILE_DOC_EVENTS.MESSAGE](awFrame) // a publishes its caret for client 10 + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 20 }) + sent.length = 0 + + // socket-a (owns 10) tries to rebind to 20, owned by a different user → reject. + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 20 }) + + expect(a.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'CLIENT_ID_IN_USE' }) + ) + // The rejected rebind must NOT have removed a's existing caret (no awareness + // removal broadcast fires). + const removal = sent.find( + (m) => + m.event === FILE_DOC_EVENTS.MESSAGE && + (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS + ) + expect(removal).toBeUndefined() + }) + + it('drops a malformed frame without throwing', async () => { + const { io } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(() => + a.handlers[FILE_DOC_EVENTS.MESSAGE](new Uint8Array([255, 254, 253, 200])) + ).not.toThrow() + }) + + it('hands the seeder role to a remaining client when the elected one leaves before seeding', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + const b = setup('socket-b', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + sent.length = 0 + + // The seeder leaves before it ever seeded; b remains. + cleanupFileDocForSocket('socket-a', io) + + const seed = sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) + expect(seed?.target).toBe('socket-b') + }) + + it('drops the document when the last editor leaves, re-seeding a fresh joiner', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + cleanupFileDocForSocket('socket-a', io) + sent.length = 0 + + const b = setup('socket-b', io) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + const seed = sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) + expect(seed?.target).toBe('socket-b') + }) + + it('aborts a join superseded by a newer join during authorization (no cross-binding)', async () => { + const { io } = createIo() + let resolveFirst: (v: unknown) => void = () => {} + mockAuthorizeRoom + .mockReturnValueOnce(new Promise((resolve) => (resolveFirst = resolve))) + .mockResolvedValueOnce({ allowed: true, status: 200, workspacePermission: 'write' }) + const s = setup('socket-a', io) + + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) + resolveFirst({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + // The socket is bound only to the newer file, never cross-bound to file-1. + expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') + expect(s.socket.join).not.toHaveBeenCalledWith('workspace-file-doc:file-1') + }) + + it('does not register a socket that disconnected during authorization', async () => { + const { io, sent } = createIo() + let resolveAuth: (v: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValueOnce(new Promise((resolve) => (resolveAuth = resolve))) + const s = setup('socket-a', io) + + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + s.socket.disconnected = true + cleanupFileDocForSocket('socket-a', io) // disconnect cleanup — no-op, nothing registered yet + resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + expect(s.socket.join).not.toHaveBeenCalled() + // No room leaked: a fresh joiner starts a new document and is elected to seed. + const b = setup('socket-b', io) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') + }) + + it('does not abort an in-flight join when a leave for a different file arrives', async () => { + const { io } = createIo() + let resolveAuth: (v: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValueOnce(new Promise((resolve) => (resolveAuth = resolve))) + const s = setup('socket-a', io) + + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) + // A stale leave for a DIFFERENT file must not invalidate the in-flight join. + s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + expect(joinSuccessFileId(s.socket)).toBe('file-2') + expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') + }) + + it('scopes LEAVE to the named file (a leave for a different file is a no-op)', async () => { + const { io } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + a.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'other' }) + expect(a.socket.leave).not.toHaveBeenCalledWith(ROOM_NAME) + + a.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + expect(a.socket.leave).toHaveBeenCalledWith(ROOM_NAME) + }) + + it('replies with a sync step 2 to the sender on a sync step 1 frame', async () => { + const { io } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + // Give the server doc some content so a step-1 request yields a non-empty step 2. + const seeded = new Y.Doc() + seeded.getText('default').insert(0, 'hi') + a.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(seeded)) + ) + ) + a.socket.emit.mockClear() + + a.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + + const reply = a.socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + expect((reply?.[1] as Uint8Array)[0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + }) + + it('does not re-elect a seeder once the document is marked seeded', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) // a elected seeder + // a seeds: set the CRDT initialContentLoaded flag on the server doc. + const seeded = new Y.Doc() + seeded.getMap('config').set('initialContentLoaded', true) + a.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(seeded)) + ) + ) + const b = setup('socket-b', io) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + sent.length = 0 + + // The seeder leaves a SEEDED doc → no re-election (no duplicate seed). + cleanupFileDocForSocket('socket-a', io) + expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)).toBeUndefined() + }) + + it('leaves the previous document when a socket switches files', async () => { + const { io, sent } = createIo() + const s = setup('socket-a', io) + await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) + + expect(s.socket.leave).toHaveBeenCalledWith('workspace-file-doc:file-1') + expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') + + // file-1's room was dropped (socket-a was its only owner): a fresh joiner of + // file-1 starts a new document and is elected to seed. + sent.length = 0 + const b = setup('socket-b', io) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') + }) + + it('re-elects a new seeder when the elected one misses the seed deadline', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + const b = setup('socket-b', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-a') + sent.length = 0 + + // socket-a never seeds; the deadline lapses. + vi.advanceTimersByTime(10_000) + + // The remaining un-tried client is asked to seed instead. + expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') + }) + + it('cancels the seed deadline once the document is seeded', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + const b = setup('socket-b', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + // socket-a seeds within the deadline. + const seeded = new Y.Doc() + seeded.getMap('config').set('initialContentLoaded', true) + a.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(seeded)) + ) + ) + sent.length = 0 + + vi.advanceTimersByTime(10_000) + + // No re-election: the successful seed cancelled the deadline. + expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)).toBeUndefined() + }) +}) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts new file mode 100644 index 00000000000..a6c43c00cfa --- /dev/null +++ b/apps/realtime/src/handlers/file-doc.ts @@ -0,0 +1,514 @@ +import { createLogger } from '@sim/logger' +import { authorizeRoom } from '@sim/platform-authz/rooms' +import { + FILE_DOC_EVENTS, + FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SEED, + type JoinFileDocPayload, + type LeaveFileDocPayload, + toFileDocBytes, +} from '@sim/realtime-protocol/file-doc' +import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import type { Server } from 'socket.io' +import * as awarenessProtocol from 'y-protocols/awareness' +import * as syncProtocol from 'y-protocols/sync' +import * as Y from 'yjs' +import type { AuthenticatedSocket } from '@/middleware/auth' +import type { IRoomManager } from '@/rooms' + +const logger = createLogger('FileDocHandlers') + +/** + * How long an elected seeder has to import the initial content before the server + * re-elects another client. Seeding is a local, synchronous editor write, so this + * only trips when a seeder never completes it (a bug, or a client withholding the + * seed) — it keeps the document from staying empty for the rest of the room. + */ +const SEED_DEADLINE_MS = 10_000 + +/** + * Collaborative document editing (live carets + text selection) for a single + * file's rich-text editor. This is the standard Yjs "websocket server" relay — + * an authoritative in-memory {@link Y.Doc} + {@link awarenessProtocol.Awareness} + * per file — carried over the shared, already-authenticated Socket.IO connection + * and the room abstraction, rather than a separate ws server. Clients speak the + * `y-protocols` sync + awareness protocols; the server applies and relays them. + * + * No durable Yjs state is kept yet: the document lives only while at least one + * collaborator is connected, and is re-seeded from the file's stored markdown on + * the next cold open (the markdown, saved by a client through the content API, is + * the durable source of truth). Durable Yjs snapshots are a separate follow-up. + * + * Single-writer assumption: the authoritative {@link Y.Doc} is held in this + * process's memory, so correctness assumes one realtime replica per file (Helm + * pins `realtime.replicaCount: 1`). Horizontal scaling would need a shared Yjs + * backend (y-redis / Hocuspocus) — out of scope here. + */ +/** A socket's presence ownership within a room. */ +interface FileDocOwner { + /** + * The awareness clientID the socket declared at join. It owns exactly this one + * and may only publish/remove awareness for it, so an authenticated peer cannot + * forge or clear another collaborator's presence. + */ + clientId: number + /** The owning user — used to tell a reconnect (same user reusing its Yjs client + * id) from a spoof (a different user binding a peer's id). */ + userId: string +} + +interface FileDocRoom { + /** The `workspace_files.id` this room edits (for seed-request payloads). */ + fileId: string + doc: Y.Doc + awareness: awarenessProtocol.Awareness + /** socketId → its presence ownership. */ + owners: Map + /** The socket currently elected to seed initial content, or `null`. */ + seederSocketId: string | null + /** Deadline timer for the current seeder to complete the seed, or `null`. */ + seedTimer: ReturnType | null + /** Sockets that were elected but failed to seed within the deadline; skipped + * on re-election so a single stuck/withholding client can't block the room. */ + triedSeeders: Set +} + +/** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */ +const fileDocRooms = new Map() +/** socketId → its current file-doc room name (a socket edits at most one doc). */ +const socketToRoomName = new Map() +/** + * socketId → a monotonic join generation. A JOIN bumps it on arrival and, after + * the async authorization, proceeds only if the generation is still its own — so + * a newer JOIN (a fast document switch) or a disconnect (which drops the entry in + * cleanup) that occurred during authorization aborts the now-stale JOIN. Without + * this, an out-of-order authorize completion could bind the socket to the wrong + * document, or a disconnect-during-authorize could register a dead socket and + * leak its room. + */ +const joinGeneration = new Map() + +interface AwarenessChange { + added: number[] + updated: number[] + removed: number[] +} + +const fileDocRoom = (fileId: string): RoomRef => ({ + type: ROOM_TYPES.WORKSPACE_FILE_DOC, + id: fileId, +}) + +/** + * A `y-protocols` transaction/awareness origin is the emitting socket id (a + * string) when it came from a client, and something else (`null` / `'local'` / + * `'timeout'`) for server-internal changes. Returns the socket id to exclude + * from a relay, or `null` to broadcast to the whole room. + */ +function originSocketId(origin: unknown): string | null { + return typeof origin === 'string' ? origin : null +} + +function broadcast(io: Server, name: string, payload: Uint8Array, exceptSocketId: string | null) { + const channel = exceptSocketId ? io.to(name).except(exceptSocketId) : io.to(name) + channel.emit(FILE_DOC_EVENTS.MESSAGE, payload) +} + +/** Whether the client has recorded that it seeded the document's initial content. */ +function isDocSeeded(doc: Y.Doc): boolean { + return doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag) === true +} + +/** + * Decode the client IDs an awareness update carries, without applying it, to + * check a frame only touches its sender's own presence. Mirrors the wire format + * of `awarenessProtocol.encodeAwarenessUpdate`: a count, then per client a + * varUint id, a varUint clock, and a varString state. + */ +function awarenessUpdateClientIds(update: Uint8Array): number[] { + const decoder = decoding.createDecoder(update) + const count = decoding.readVarUint(decoder) + const ids: number[] = [] + for (let i = 0; i < count; i++) { + ids.push(decoding.readVarUint(decoder)) + decoding.readVarUint(decoder) // clock + decoding.readVarUint8Array(decoder) // state bytes — advanced past, only ids matter + } + return ids +} + +function clearSeedTimer(room: FileDocRoom) { + if (room.seedTimer !== null) { + clearTimeout(room.seedTimer) + room.seedTimer = null + } +} + +/** + * Drop a room's document + awareness (and its seed timer) once it has no owners, + * so an idle file holds no memory. A later joiner re-creates and re-seeds it. + */ +function destroyRoomIfIdle(name: string) { + const room = fileDocRooms.get(name) + if (!room || room.owners.size > 0) return + clearSeedTimer(room) + room.awareness.destroy() + room.doc.destroy() + fileDocRooms.delete(name) +} + +/** + * Elect a client to seed an unseeded document and ask it to import the stored + * markdown, if one is needed and not already assigned. Called after a join (to + * elect the newcomer), after the elected seeder leaves (to hand the role to a + * remaining client), and after a seed deadline lapses (to pass over a client that + * never seeded). Skips clients that already failed to seed, and arms a deadline so + * a stuck or withholding seeder can't leave the document empty for the whole room. + * A no-op once the document is seeded, a seeder is already assigned, or no + * un-tried client remains. + */ +function electSeederIfNeeded(io: Server, room: FileDocRoom) { + if (room.seederSocketId !== null || isDocSeeded(room.doc)) return + + let elected: string | null = null + for (const socketId of room.owners.keys()) { + if (!room.triedSeeders.has(socketId)) { + elected = socketId + break + } + } + if (elected === null) return + + room.seederSocketId = elected + io.to(elected).emit(FILE_DOC_EVENTS.SEED_REQUEST, { fileId: room.fileId }) + + clearSeedTimer(room) + room.seedTimer = setTimeout(() => { + room.seedTimer = null + // Only act if this election is still the pending one and it never seeded. + if (room.seederSocketId !== elected || isDocSeeded(room.doc)) return + room.triedSeeders.add(elected) + room.seederSocketId = null + electSeederIfNeeded(io, room) + }, SEED_DEADLINE_MS) +} + +/** + * Get (or lazily create) the authoritative document for a room, wiring the two + * relay handlers exactly once: document updates and awareness changes are + * broadcast to the room, excluding the origin socket (it already applied them). + */ +function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { + const name = roomName(ref) + const existing = fileDocRooms.get(name) + if (existing) return existing + + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + // The server holds no cursor of its own; it only relays clients' awareness. + awareness.setLocalState(null) + + const room: FileDocRoom = { + fileId: ref.id, + doc, + awareness, + owners: new Map(), + seederSocketId: null, + seedTimer: null, + triedSeeders: new Set(), + } + fileDocRooms.set(name, room) + + doc.on('update', (update: Uint8Array, origin: unknown) => { + // Once the document is seeded, the seed deadline is moot — cancel it. + if (room.seedTimer !== null && isDocSeeded(doc)) clearSeedTimer(room) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, update) + broadcast(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) + }) + + awareness.on('update', ({ added, updated, removed }: AwarenessChange, origin: unknown) => { + const changed = added.concat(updated, removed) + if (changed.length === 0) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(awareness, changed) + ) + broadcast(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) + }) + + return room +} + +function emitJoinError( + socket: AuthenticatedSocket, + fileId: unknown, + error: string, + code: string, + retryable: boolean +) { + socket.emit(FILE_DOC_EVENTS.JOIN_ERROR, { + fileId: typeof fileId === 'string' ? fileId : '', + error, + code, + retryable, + }) +} + +function handleMessage(socket: AuthenticatedSocket, data: unknown) { + const name = socketToRoomName.get(socket.id) + if (!name) return + const room = fileDocRooms.get(name) + if (!room) return + + const bytes = toFileDocBytes(data) + if (!bytes) return + + // A malformed frame from any client must never escape as a process-level + // exception; drop it and keep the relay running. + try { + const decoder = decoding.createDecoder(bytes) + const messageType = decoding.readVarUint(decoder) + + switch (messageType) { + case FILE_DOC_MESSAGE_TYPE.SYNC: { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + // `socket.id` is the transaction origin, so the doc's `update` handler + // excludes this sender when relaying the applied update to the room. + syncProtocol.readSyncMessage(decoder, encoder, room.doc, socket.id) + // A reply longer than the 1-byte type tag is a sync step 2 (or step 1) + // destined for the sender only; applied updates fan out via `doc.on`. + if (encoding.length(encoder) > 1) { + socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + break + } + case FILE_DOC_MESSAGE_TYPE.AWARENESS: { + const update = decoding.readVarUint8Array(decoder) + // Enforce presence ownership: a socket may only publish/remove awareness + // for the clientID it bound at join, so a peer cannot spoof or clear + // another collaborator's caret. + const owned = room.owners.get(socket.id)?.clientId + if (owned === undefined || awarenessUpdateClientIds(update).some((id) => id !== owned)) { + logger.warn('Dropping awareness frame for an unowned client id', { socketId: socket.id }) + return + } + awarenessProtocol.applyAwarenessUpdate(room.awareness, update, socket.id) + break + } + default: + logger.warn('Unknown file-doc message type', { messageType }) + } + } catch (error) { + logger.warn('Dropping malformed file-doc frame', { socketId: socket.id, error }) + } +} + +/** + * Remove a socket from its file-doc room: clear its awareness state (so its caret + * disappears for everyone else), hand off the seeder role if it left before + * seeding, and drop the room's document when the last collaborator leaves. + * Exported for the disconnect handler; safe to call for a socket in no room. + */ +export function cleanupFileDocForSocket(socketId: string, io: Server): void { + // Drop the join generation so an in-flight JOIN for this socket aborts after + // its authorize resolves, and the map never leaks across the socket's life. + joinGeneration.delete(socketId) + + const name = socketToRoomName.get(socketId) + if (!name) return + socketToRoomName.delete(socketId) + + const room = fileDocRooms.get(name) + if (!room) return + + const owner = room.owners.get(socketId) + room.owners.delete(socketId) + if (owner !== undefined) { + // Fires the awareness `update` handler with a non-socket origin → the removal + // is broadcast to every remaining client, so the departed caret vanishes. + awarenessProtocol.removeAwarenessStates(room.awareness, [owner.clientId], null) + } + + // Hand off the seeder role: if the elected seeder left before it seeded, elect + // a remaining client so the document doesn't stay permanently empty. + if (room.seederSocketId === socketId) { + room.seederSocketId = null + electSeederIfNeeded(io, room) + } + + destroyRoomIfIdle(name) +} + +/** + * Registers the collaborative file-document handlers on a socket. Room id is the + * file id; joining requires workspace `write` (editing a document). Mirrors the + * workspace-files join shape (auth → readiness → validate → authorize → join), + * then runs the Yjs sync/awareness handshake. + */ +export function setupWorkspaceFileDocHandlers( + socket: AuthenticatedSocket, + roomManager: IRoomManager +) { + const io = roomManager.io + + socket.on(FILE_DOC_EVENTS.JOIN, async ({ fileId, clientId }: JoinFileDocPayload) => { + try { + const userId = socket.userId + const userName = socket.userName + + if (!userId || !userName) { + emitJoinError(socket, fileId, 'Authentication required', 'AUTHENTICATION_REQUIRED', false) + return + } + if (!roomManager.isReady()) { + emitJoinError(socket, fileId, 'Realtime unavailable', 'ROOM_MANAGER_UNAVAILABLE', true) + return + } + if (typeof fileId !== 'string' || fileId.length === 0 || typeof clientId !== 'number') { + emitJoinError(socket, fileId, 'Invalid join payload', 'INVALID_PAYLOAD', false) + return + } + + // Claim this JOIN's generation before the async authorize below. + const generation = (joinGeneration.get(socket.id) ?? 0) + 1 + joinGeneration.set(socket.id, generation) + + const room = fileDocRoom(fileId) + const name = roomName(room) + + let authorized: Awaited> + try { + authorized = await authorizeRoom({ userId, room, action: 'write' }) + } catch (error) { + logger.warn(`Error authorizing file-doc room for ${userId}:`, error) + emitJoinError( + socket, + fileId, + 'Failed to verify workspace access', + 'VERIFY_ACCESS_FAILED', + true + ) + return + } + if (!authorized.allowed) { + emitJoinError( + socket, + fileId, + authorized.status === 404 ? 'File not found' : 'Access denied to file', + authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', + false + ) + return + } + + // Abort a JOIN superseded during authorization: the socket disconnected, or + // a newer JOIN (a document switch) bumped the generation. Registering here + // would leak a dead socket's room or bind the socket to the wrong document. + if (socket.disconnected || joinGeneration.get(socket.id) !== generation) return + + // Switched documents on the same socket — leave the previous one first (a + // socket edits at most one document). A duplicate join of the SAME room + // falls through and simply re-runs the sync handshake, idempotently. + const currentName = socketToRoomName.get(socket.id) + if (currentName && currentName !== name) { + socket.leave(currentName) + cleanupFileDocForSocket(socket.id, io) + } + + const entry = getOrCreateRoom(io, room) + + // A client id must be owned by at most one user, or a peer could bind an + // active collaborator's id and pass the per-frame ownership check to + // spoof/clear its caret. Distinguish a reconnect from a spoof by the owning + // user: the same user reclaiming its own client id (a dropped socket + // reconnecting reuses the Yjs client id, and its prior socket may not be + // cleaned up yet) takes over the stale binding; a DIFFERENT user is + // rejected. This runs BEFORE any state mutation below, so a rejected rebind + // leaves the socket's existing binding and caret untouched. + for (const [otherSid, owner] of entry.owners) { + if (owner.clientId !== clientId || otherSid === socket.id) continue + if (owner.userId !== userId) { + emitJoinError(socket, fileId, 'Client id already in use', 'CLIENT_ID_IN_USE', false) + return + } + entry.owners.delete(otherSid) + awarenessProtocol.removeAwarenessStates(entry.awareness, [owner.clientId], null) + } + + // Accepted: a same socket rebinding to a NEW client id clears its old caret + // so it doesn't linger as a ghost after the binding is overwritten. + const previous = entry.owners.get(socket.id) + if (previous !== undefined && previous.clientId !== clientId) { + awarenessProtocol.removeAwarenessStates(entry.awareness, [previous.clientId], null) + } + + entry.owners.set(socket.id, { clientId, userId }) + socketToRoomName.set(socket.id, name) + socket.join(name) + + socket.emit(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId }) + + // Begin the sync handshake: send the server's state (sync step 1). The + // client replies with its updates and requests the server's in return. + const syncEncoder = encoding.createEncoder() + encoding.writeVarUint(syncEncoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep1(syncEncoder, entry.doc) + socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(syncEncoder)) + + // Send existing awareness so the new client immediately sees others' carets. + const states = entry.awareness.getStates() + if (states.size > 0) { + const awarenessEncoder = encoding.createEncoder() + encoding.writeVarUint(awarenessEncoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array( + awarenessEncoder, + awarenessProtocol.encodeAwarenessUpdate(entry.awareness, Array.from(states.keys())) + ) + socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(awarenessEncoder)) + } + + electSeederIfNeeded(io, entry) + + logger.info(`User ${userId} joined file-doc room ${fileId}`) + } catch (error) { + logger.error('Error joining file-doc room:', error) + try { + const name = roomName(fileDocRoom(fileId)) + socket.leave(name) + cleanupFileDocForSocket(socket.id, io) + // If the failure happened after `getOrCreateRoom` but before the socket + // registered as an owner, `cleanupFileDocForSocket` (which keys off + // `socketToRoomName`) can't drop the freshly-created empty room — do it here. + destroyRoomIfIdle(name) + } catch {} + emitJoinError(socket, fileId, 'Failed to join file document', 'JOIN_FAILED', true) + } + }) + + socket.on(FILE_DOC_EVENTS.MESSAGE, (data: unknown) => handleMessage(socket, data)) + + socket.on(FILE_DOC_EVENTS.LEAVE, (payload?: LeaveFileDocPayload) => { + try { + // Only affect a REGISTERED room; never touch the join generation here. A + // leave that raced ahead of an in-flight join (no room registered yet) is a + // no-op — bumping the generation would silently abort an unrelated join for + // a different file (a document switch), leaving the socket bound to nothing. + const name = socketToRoomName.get(socket.id) + if (!name) return + // Scope the leave to the named file when provided: a deferred leave from a + // prior document must not evict the socket from one it has since opened. + if (payload?.fileId && roomName(fileDocRoom(payload.fileId)) !== name) return + socket.leave(name) + cleanupFileDocForSocket(socket.id, io) + } catch (error) { + logger.error('Error leaving file-doc room:', error) + } + }) +} diff --git a/apps/realtime/src/handlers/index.ts b/apps/realtime/src/handlers/index.ts index 91573f1a1fc..9e034d77db1 100644 --- a/apps/realtime/src/handlers/index.ts +++ b/apps/realtime/src/handlers/index.ts @@ -1,4 +1,5 @@ import { setupConnectionHandlers } from '@/handlers/connection' +import { setupWorkspaceFileDocHandlers } from '@/handlers/file-doc' import { setupOperationsHandlers } from '@/handlers/operations' import { setupPresenceHandlers } from '@/handlers/presence' import { setupSubblocksHandlers } from '@/handlers/subblocks' @@ -15,5 +16,6 @@ export function setupAllHandlers(socket: AuthenticatedSocket, roomManager: IRoom setupVariablesHandlers(socket, roomManager) setupPresenceHandlers(socket, roomManager) setupWorkspaceFilesHandlers(socket, roomManager) + setupWorkspaceFileDocHandlers(socket, roomManager) setupConnectionHandlers(socket, roomManager) } diff --git a/bun.lock b/bun.lock index 033da3635b8..f1e8d8bde90 100644 --- a/bun.lock +++ b/bun.lock @@ -84,9 +84,12 @@ "@sim/workflow-types": "workspace:*", "@socket.io/redis-adapter": "8.3.0", "drizzle-orm": "^0.45.2", + "lib0": "0.2.117", "postgres": "^3.4.5", "redis": "5.10.0", "socket.io": "^4.8.1", + "y-protocols": "1.0.7", + "yjs": "13.6.31", "zod": "4.3.6", }, "devDependencies": { @@ -2955,6 +2958,8 @@ "isomorphic-ws": ["isomorphic-ws@5.0.0", "", { "peerDependencies": { "ws": "*" } }, "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw=="], + "isomorphic.js": ["isomorphic.js@0.2.5", "", {}, "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw=="], + "istanbul-lib-coverage": ["istanbul-lib-coverage@3.2.2", "", {}, "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg=="], "istanbul-lib-report": ["istanbul-lib-report@3.0.1", "", { "dependencies": { "istanbul-lib-coverage": "^3.0.0", "make-dir": "^4.0.0", "supports-color": "^7.1.0" } }, "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw=="], @@ -3017,6 +3022,8 @@ "leac": ["leac@0.6.0", "", {}, "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg=="], + "lib0": ["lib0@0.2.117", "", { "dependencies": { "isomorphic.js": "^0.2.4" }, "bin": { "0serve": "bin/0serve.js", "0gentesthtml": "bin/gentesthtml.js", "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js" } }, "sha512-DeXj9X5xDCjgKLU/7RR+/HQEVzuuEUiwldwOGsHK/sfAfELGWEyTcf0x+uOvCvK3O2zPmZePXWL85vtia6GyZw=="], + "libbase64": ["libbase64@1.3.0", "", {}, "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg=="], "libmime": ["libmime@5.3.7", "", { "dependencies": { "encoding-japanese": "2.2.0", "iconv-lite": "0.6.3", "libbase64": "1.3.0", "libqp": "2.1.1" } }, "sha512-FlDb3Wtha8P01kTL3P9M+ZDNDWPKPmKHWaU/cG/lg5pfuAwdflVpZE+wm9m7pKmC5ww6s+zTxBKS1p6yl3KpSw=="], @@ -4147,6 +4154,8 @@ "xpath": ["xpath@0.0.34", "", {}, "sha512-FxF6+rkr1rNSQrhUNYrAFJpRXNzlDoMxeXN5qI84939ylEv3qqPFKa85Oxr6tDaJKqwW6KKyo2v26TSv3k6LeA=="], + "y-protocols": ["y-protocols@1.0.7", "", { "dependencies": { "lib0": "^0.2.85" }, "peerDependencies": { "yjs": "^13.0.0" } }, "sha512-YSVsLoXxO67J6eE/nV4AtFtT3QEotZf5sK5BHxFBXso7VDUT3Tx07IfA6hsu5Q5OmBdMkQVmFZ9QOA7fikWvnw=="], + "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], "yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="], @@ -4159,6 +4168,8 @@ "yauzl": ["yauzl@3.4.0", "", { "dependencies": { "pend": "~1.2.0" } }, "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw=="], + "yjs": ["yjs@13.6.31", "", { "dependencies": { "lib0": "^0.2.99" } }, "sha512-Eq+5BRfbeGyqGVrTJL3bEcr8gKkxPuyuoHmAwpk52fDb8kOVMrfVSTRPd6yiGgX5Fskb96qCRjzjbRjrL4YEnw=="], + "yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], "yoctocolors": ["yoctocolors@2.1.2", "", {}, "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug=="], diff --git a/packages/platform-authz/src/rooms.ts b/packages/platform-authz/src/rooms.ts index 1fd9949333e..f8676cfde45 100644 --- a/packages/platform-authz/src/rooms.ts +++ b/packages/platform-authz/src/rooms.ts @@ -1,4 +1,4 @@ -import { db, workspace } from '@sim/db' +import { db, workspace, workspaceFiles } from '@sim/db' import { ROOM_TYPES, type RoomRef, type RoomType } from '@sim/realtime-protocol/rooms' import { and, eq, isNull } from 'drizzle-orm' import { getActiveWorkflowContext } from './workflow' @@ -37,6 +37,24 @@ async function resolveWorkspaceRoomWorkspace(workspaceId: string): Promise { + const [file] = await db + .select({ workspaceId: workspaceFiles.workspaceId }) + .from(workspaceFiles) + .where(and(eq(workspaceFiles.id, fileId), isNull(workspaceFiles.deletedAt))) + .limit(1) + + if (!file?.workspaceId) return null + return resolveWorkspaceRoomWorkspace(file.workspaceId) +} + /** * Single source of truth mapping each room type to its resource→workspace * lookup. Every realtime room is workspace-scoped and authorizes through the @@ -54,6 +72,8 @@ const ROOM_WORKSPACE_RESOLVERS: Record = { }, // A workspace-files room is addressed directly by its workspace id. [ROOM_TYPES.WORKSPACE_FILES]: resolveWorkspaceRoomWorkspace, + // A file-doc room is addressed by file id; resolve it to its workspace. + [ROOM_TYPES.WORKSPACE_FILE_DOC]: resolveFileDocWorkspace, } /** Resolves a room's owning workspace, or `null` if the room resource is gone. */ diff --git a/packages/realtime-protocol/package.json b/packages/realtime-protocol/package.json index 4f0ea685ebe..7023ec6a12a 100644 --- a/packages/realtime-protocol/package.json +++ b/packages/realtime-protocol/package.json @@ -25,6 +25,10 @@ "./rooms": { "types": "./src/rooms.ts", "default": "./src/rooms.ts" + }, + "./file-doc": { + "types": "./src/file-doc.ts", + "default": "./src/file-doc.ts" } }, "scripts": { diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts new file mode 100644 index 00000000000..738af5bc123 --- /dev/null +++ b/packages/realtime-protocol/src/file-doc.ts @@ -0,0 +1,134 @@ +/** + * Wire protocol for the collaborative file-document room + * ({@link ROOM_TYPES.WORKSPACE_FILE_DOC}). Live carets and text selection ride + * Yjs document sync + awareness over the shared Socket.IO connection. These are + * the event names, binary message tags, and join payloads that the server + * (`apps/realtime/src/handlers/file-doc.ts`) and the client provider + * (`apps/sim/.../file-doc`) must agree on exactly — the single source of truth + * for both sides so they can never drift. + * + * The binary channel uses the standard Yjs "websocket" framing: every + * {@link FILE_DOC_EVENTS.MESSAGE} payload is a `Uint8Array` whose first varUint + * is a {@link FILE_DOC_MESSAGE_TYPE} tag (sync protocol vs awareness protocol), + * so the client provider can reuse `y-protocols` verbatim. + */ + +/** Socket.IO event names for the file-document collaboration channel. */ +export const FILE_DOC_EVENTS = { + /** Client → server: join a file's collaborative session ({@link JoinFileDocPayload}). */ + JOIN: 'join-file-doc', + /** Server → client: join accepted ({@link JoinFileDocSuccess}). */ + JOIN_SUCCESS: 'join-file-doc-success', + /** Server → client: join rejected ({@link JoinFileDocError}). */ + JOIN_ERROR: 'join-file-doc-error', + /** + * Server → client: this client is elected to seed the document's initial + * content from the file's stored markdown ({@link SeedRequestPayload}). Sent to + * exactly one client of an unseeded document — at join, or later to a remaining + * client if the previously-elected seeder disconnects before it seeds. + */ + SEED_REQUEST: 'file-doc-seed-request', + /** Client → server: leave the session ({@link LeaveFileDocPayload}). */ + LEAVE: 'leave-file-doc', + /** Both directions: a framed Yjs message (binary), tagged by {@link FILE_DOC_MESSAGE_TYPE}. */ + MESSAGE: 'file-doc-message', +} as const + +/** + * The tag carried in the first varUint of a {@link FILE_DOC_EVENTS.MESSAGE} + * payload — the standard Yjs websocket framing distinguishing a document-sync + * message from an awareness (cursor/selection) message. + */ +export const FILE_DOC_MESSAGE_TYPE = { + SYNC: 0, + AWARENESS: 1, +} as const + +export type FileDocMessageType = (typeof FILE_DOC_MESSAGE_TYPE)[keyof typeof FILE_DOC_MESSAGE_TYPE] + +/** + * Where the client records that it has seeded the document's initial content, + * stored inside the Yjs document as `doc.getMap(configMap).get(flag) === true`. + * Because it lives in the CRDT it merges across clients and the server can read + * it — the server uses it to decide whether to re-elect a seeder when an elected + * one disconnects before seeding. Client and server MUST use these exact keys. + * + * The seeding client MUST write the imported content AND set this flag in a + * SINGLE Yjs transaction (`doc.transact(...)`). Otherwise a seeder that dies + * between the two writes can leave content with no flag, and a re-elected client + * would seed again and duplicate it. `configMap` is a reserved top-level Y.Map + * name; the editor must not use a top-level type of the same name (TipTap uses + * `getXmlFragment('default')`, so there is no collision today). + */ +export const FILE_DOC_SEED = { + configMap: 'config', + flag: 'initialContentLoaded', +} as const + +/** Client → server join request. `fileId` is the `workspace_files.id`. */ +export interface JoinFileDocPayload { + fileId: string + /** + * The joining Yjs document's `clientID`. The server binds it to this socket so + * a client can only publish/remove awareness (cursor/selection) for its own + * client — an authenticated peer cannot forge or clear another's presence. + */ + clientId: number +} + +/** Server → client acceptance of a {@link FILE_DOC_EVENTS.JOIN}. */ +export interface JoinFileDocSuccess { + fileId: string +} + +/** + * Server → client seed election ({@link FILE_DOC_EVENTS.SEED_REQUEST}). The + * recipient imports the file's stored markdown into the (empty) document. The + * client still guards on the CRDT `initialContentLoaded` flag so a re-election + * that races an in-flight seed can never duplicate content. + * + * Consumer (editor hook) contract — the relay depends on these to be safe: + * 1. **Never overwrite content with an unseeded doc.** Autosave (the markdown + * mirror written through the content API) MUST be gated on the document being + * both synced AND seeded (`initialContentLoaded === true`). Otherwise an empty + * or still-syncing doc could be saved over the real file — the one true + * data-loss path, and the reason a withholding seeder is only a liveness + * nuisance rather than destructive. + * 2. **Seed atomically, or leave.** Write content + the flag in ONE + * `doc.transact(...)`; if seeding fails, emit {@link FILE_DOC_EVENTS.LEAVE} + * (or destroy the provider) so the server re-elects another client. + * 3. **One provider per socket.** Destroy the previous {@link FILE_DOC_EVENTS} + * provider before creating the next (document switch), so a stale provider's + * binary-frame listener can't apply another document's updates. + * 4. **Re-mint on CLIENT_ID_IN_USE.** On a `CLIENT_ID_IN_USE` join error, + * recreate the Yjs doc (fresh `clientID`) and rejoin rather than giving up — + * the id is transiently held by another socket of the same user. + */ +export interface SeedRequestPayload { + fileId: string +} + +/** Server → client rejection of a {@link FILE_DOC_EVENTS.JOIN}. */ +export interface JoinFileDocError { + fileId: string + error: string + code: string + retryable?: boolean +} + +/** Client → server leave request. */ +export interface LeaveFileDocPayload { + fileId: string +} + +/** + * Coerce a Socket.IO binary payload to a `Uint8Array`, or `null` if it is + * neither. Shared by the server relay and the client provider so the two agree + * on how an inbound {@link FILE_DOC_EVENTS.MESSAGE} frame is read (Socket.IO may + * deliver a `Uint8Array`/`Buffer` or an `ArrayBuffer` depending on runtime). + */ +export function toFileDocBytes(data: unknown): Uint8Array | null { + if (data instanceof Uint8Array) return data + if (data instanceof ArrayBuffer) return new Uint8Array(data) + return null +} diff --git a/packages/realtime-protocol/src/rooms.test.ts b/packages/realtime-protocol/src/rooms.test.ts index 6ab07de542e..cb37c3fca45 100644 --- a/packages/realtime-protocol/src/rooms.test.ts +++ b/packages/realtime-protocol/src/rooms.test.ts @@ -18,6 +18,20 @@ describe('roomName', () => { expect(roomName({ type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-123' })).toBe( 'workspace-files:ws-123' ) + expect(roomName({ type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: 'file-123' })).toBe( + 'workspace-file-doc:file-123' + ) + }) + + it('keeps the file-doc and file-browser namespaces distinct for the same id', () => { + // `workspace-file-doc` must not be parsed as the `workspace-files` browser + // room (or vice versa): the prefix match is the whole segment before `:`. + const id = 'a1b2c3d4-uuid' + const doc = roomName({ type: ROOM_TYPES.WORKSPACE_FILE_DOC, id }) + const browser = roomName({ type: ROOM_TYPES.WORKSPACE_FILES, id }) + expect(doc).not.toBe(browser) + expect(parseRoomName(doc)).toEqual({ type: ROOM_TYPES.WORKSPACE_FILE_DOC, id }) + expect(parseRoomName(browser)).toEqual({ type: ROOM_TYPES.WORKSPACE_FILES, id }) }) it('never collides a namespaced room with a bare workflow id for real ids', () => { @@ -35,6 +49,7 @@ describe('parseRoomName', () => { const refs: RoomRef[] = [ { type: ROOM_TYPES.WORKFLOW, id: 'wf-123' }, { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-456' }, + { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: 'file-789' }, ] for (const ref of refs) { expect(parseRoomName(roomName(ref))).toEqual(ref) diff --git a/packages/realtime-protocol/src/rooms.ts b/packages/realtime-protocol/src/rooms.ts index c03ad621d45..b81b65e72bb 100644 --- a/packages/realtime-protocol/src/rooms.ts +++ b/packages/realtime-protocol/src/rooms.ts @@ -22,6 +22,13 @@ export const ROOM_TYPES = { WORKFLOW: 'workflow', /** The workspace file browser (one room per workspace). */ WORKSPACE_FILES: 'workspace-files', + /** + * A single collaborative file document — the rich-text editor for one file + * (one room per file). Carries Yjs document sync + awareness (live carets and + * text selection), so its id space is the file id, distinct from the + * workspace-scoped {@link ROOM_TYPES.WORKSPACE_FILES} browser room. + */ + WORKSPACE_FILE_DOC: 'workspace-file-doc', } as const export type RoomType = (typeof ROOM_TYPES)[keyof typeof ROOM_TYPES] From 45cb10be5212068e4d0be963f60965879077a2c7 Mon Sep 17 00:00:00 2001 From: Waleed Date: Fri, 24 Jul 2026 19:10:49 -0700 Subject: [PATCH 07/53] =?UTF-8?q?feat(files):=20collaborative=20document?= =?UTF-8?q?=20editing=20=E2=80=94=20client=20provider=20+=20editor=20(#594?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Client Yjs provider (FileDocProvider over the authenticated socket) + TipTap Collaboration/CollaborationCaret wiring for live carets + text-selection in the Files rich-markdown editor. Collaboration is a Files-page-only surface (explicit `collaborative` opt-in), disjoint from agent-streaming. Read-only + autosave-gated until synced+seeded. Merges into the realtime-rooms integration branch. --- .../components/file-viewer/file-viewer.tsx | 8 + .../collaboration/file-doc-provider.test.ts | 256 ++++++++++++++++++ .../collaboration/file-doc-provider.ts | 240 ++++++++++++++++ .../use-file-doc-collaboration.ts | 104 +++++++ .../rich-markdown-editor/editor-extensions.ts | 52 +++- .../rich-markdown-editor/extensions.ts | 8 +- .../rich-markdown-editor.css | 54 ++++ .../rich-markdown-editor.tsx | 230 +++++++++++++++- .../file-viewer/use-editable-file-content.ts | 10 +- .../workspace/[workspaceId]/files/files.tsx | 1 + apps/sim/lib/workspaces/colors.ts | 70 +---- apps/sim/package.json | 5 + bun.lock | 11 + 13 files changed, 961 insertions(+), 88 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index 9984e16b0df..b9bc4b4a302 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -110,6 +110,12 @@ interface FileViewerProps { streamIsIncremental?: boolean disableStreamingAutoScroll?: boolean previewContextKey?: string + /** + * Opt this surface into live collaborative editing (markdown files only). Set by the + * Files page; the agent/Chat surface leaves it off so collaboration and agent-streaming + * never target one editor. See {@link RichMarkdownEditorProps.collaborative}. + */ + collaborative?: boolean } export function FileViewer(props: FileViewerProps) { @@ -141,6 +147,7 @@ function FileViewerContent({ streamIsIncremental, disableStreamingAutoScroll = false, previewContextKey, + collaborative, }: FileViewerProps) { const category = resolveFileCategory(file.type, file.name) @@ -185,6 +192,7 @@ function FileViewerContent({ streamIsIncremental={streamIsIncremental} disableStreamingAutoScroll={disableStreamingAutoScroll} previewContextKey={previewContextKey} + collaborative={collaborative} /> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts new file mode 100644 index 00000000000..97fd5e13996 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -0,0 +1,256 @@ +/** + * @vitest-environment node + */ +import { FILE_DOC_EVENTS, FILE_DOC_MESSAGE_TYPE } from '@sim/realtime-protocol/file-doc' +import * as encoding from 'lib0/encoding' +import type { Socket } from 'socket.io-client' +import { describe, expect, it, vi } from 'vitest' +import * as awarenessProtocol from 'y-protocols/awareness' +import * as syncProtocol from 'y-protocols/sync' +import * as Y from 'yjs' +import { FileDocProvider } from './file-doc-provider' + +/** A minimal fake Socket.IO client whose server→client events can be fired in tests. */ +function createSocket(connected = true) { + const listeners = new Map void>>() + const emit = vi.fn() + const socket = { + connected, + emit, + on(event: string, cb: (...args: unknown[]) => void) { + let set = listeners.get(event) + if (!set) { + set = new Set() + listeners.set(event, set) + } + set.add(cb) + }, + off(event: string, cb: (...args: unknown[]) => void) { + listeners.get(event)?.delete(cb) + }, + } + const fire = (event: string, ...args: unknown[]) => { + for (const cb of listeners.get(event) ?? []) cb(...args) + } + return { socket: socket as unknown as Socket, emit, fire } +} + +function createProvider(connected = true) { + const { socket, emit, fire } = createSocket(connected) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness) + return { provider, doc, awareness, emit, fire } +} + +/** Messages emitted to the server, decoded to their `{ type, bytes }`. */ +function emittedMessages(emit: ReturnType) { + return emit.mock.calls + .filter( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + .map(([, payload]) => payload as Uint8Array) +} + +describe('FileDocProvider', () => { + it('joins immediately with its client id when the socket is already connected', () => { + const { doc, emit } = createProvider(true) + expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, { + fileId: 'file-1', + clientId: doc.clientID, + }) + }) + + it('waits for connect before joining when the socket is offline', () => { + const { emit, fire } = createProvider(false) + expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + fire('connect') + expect(emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN, + expect.objectContaining({ fileId: 'file-1' }) + ) + }) + + it('exchanges sync only after JOIN_SUCCESS', () => { + const { emit, fire } = createProvider(true) + emit.mockClear() + + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + + // A sync step 1 (type tag 0) is sent to exchange state with the server. + const messages = emittedMessages(emit) + expect(messages.length).toBeGreaterThan(0) + expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + }) + + it('emits seed-request and latches shouldSeed when the server elects it', () => { + const { provider, fire } = createProvider(true) + const seed = vi.fn() + provider.on('seed-request', seed) + expect(provider.shouldSeed).toBe(false) + + fire(FILE_DOC_EVENTS.SEED_REQUEST, { fileId: 'file-1' }) + + expect(seed).toHaveBeenCalledTimes(1) + // Latched so a consumer subscribing after the event can still detect election. + expect(provider.shouldSeed).toBe(true) + }) + + it('ignores acks and seed requests for a different file', () => { + const { provider, emit, fire } = createProvider(true) + const seed = vi.fn() + provider.on('seed-request', seed) + emit.mockClear() + + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'other-file' }) + fire(FILE_DOC_EVENTS.SEED_REQUEST, { fileId: 'other-file' }) + + expect(seed).not.toHaveBeenCalled() + expect(provider.shouldSeed).toBe(false) + expect(emittedMessages(emit)).toHaveLength(0) + }) + + it('applies a server sync step 2 and becomes synced', () => { + const { provider, doc, fire } = createProvider(true) + const synced = vi.fn() + provider.on('synced', synced) + + const serverDoc = new Y.Doc() + serverDoc.getText('default').insert(0, 'hello world') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + expect(doc.getText('default').toString()).toBe('hello world') + expect(provider.synced).toBe(true) + expect(synced).toHaveBeenCalledWith(true) + }) + + it('sends local document edits to the server as sync updates', () => { + const { doc, emit } = createProvider(true) + emit.mockClear() + + doc.getText('default').insert(0, 'x') + + const messages = emittedMessages(emit) + expect(messages.length).toBe(1) + expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + }) + + it('does not echo updates it applied from the server', () => { + const { provider, emit, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + emit.mockClear() + + const serverDoc = new Y.Doc() + serverDoc.getText('default').insert(0, 'remote') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(serverDoc)) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + // The applied remote update must not be re-emitted back to the server. + expect(emittedMessages(emit)).toHaveLength(0) + expect(provider.doc.getText('default').toString()).toBe('remote') + }) + + it('sends local awareness (cursor/selection) changes', () => { + const { awareness, emit } = createProvider(true) + emit.mockClear() + + awareness.setLocalStateField('user', { name: 'Ada', color: '#f783ac' }) + + const messages = emittedMessages(emit) + expect(messages.some((m) => m[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS)).toBe(true) + }) + + it('does not forward awareness it applied from the server', () => { + const { emit, fire } = createProvider(true) + emit.mockClear() + + const remoteDoc = new Y.Doc() + remoteDoc.clientID = 8888 + const remoteAwareness = new awarenessProtocol.Awareness(remoteDoc) + remoteAwareness.setLocalStateField('user', { name: 'Remote' }) + const update = awarenessProtocol.encodeAwarenessUpdate(remoteAwareness, [8888]) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array(encoder, update) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + // The remote peer's awareness (client 8888, not ours) must not be re-published. + expect(emittedMessages(emit).some((m) => m[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS)).toBe(false) + }) + + it('stops attempting to join and latches joinError after a non-retryable error', () => { + const { provider, emit, fire } = createProvider(true) + emit.mockClear() + + const error = { + fileId: 'file-1', + error: 'Access denied', + code: 'ACCESS_DENIED', + retryable: false, + } + fire(FILE_DOC_EVENTS.JOIN_ERROR, error) + fire('connect') + + expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + // Latched so a consumer subscribing after the event can still detect the failure. + expect(provider.joinError).toEqual(error) + }) + + it('still rejoins on reconnect after a retryable error', () => { + const { emit, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_ERROR, { + fileId: 'file-1', + error: 'Realtime unavailable', + code: 'ROOM_MANAGER_UNAVAILABLE', + retryable: true, + }) + emit.mockClear() + + fire('connect') + + expect(emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN, + expect.objectContaining({ fileId: 'file-1' }) + ) + }) + + it('resets synced and rejoins on a reconnect', () => { + const { provider, emit, fire } = createProvider(true) + // Become synced. + const serverDoc = new Y.Doc() + serverDoc.getText('default').insert(0, 'hi') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + emit.mockClear() + + // A reconnect must drop synced and re-issue JOIN so the doc re-syncs. + fire('connect') + + expect(provider.synced).toBe(false) + expect(emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN, + expect.objectContaining({ fileId: 'file-1' }) + ) + }) + + it('leaves the room and detaches on destroy', () => { + const { provider, doc, emit } = createProvider(true) + emit.mockClear() + + provider.destroy() + + expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.LEAVE, { fileId: 'file-1' }) + // After destroy, local edits are no longer forwarded. + emit.mockClear() + doc.getText('default').insert(0, 'y') + expect(emittedMessages(emit)).toHaveLength(0) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts new file mode 100644 index 00000000000..5472ab31668 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -0,0 +1,240 @@ +import { + FILE_DOC_EVENTS, + FILE_DOC_MESSAGE_TYPE, + type JoinFileDocError, + type JoinFileDocSuccess, + type SeedRequestPayload, + toFileDocBytes, +} from '@sim/realtime-protocol/file-doc' +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import { ObservableV2 } from 'lib0/observable' +import type { Socket } from 'socket.io-client' +import * as awarenessProtocol from 'y-protocols/awareness' +import * as syncProtocol from 'y-protocols/sync' +import type * as Y from 'yjs' + +/** + * Events emitted by {@link FileDocProvider}. + * - `synced`: the first full document sync with the server completed. + * - `seed-request`: the server elected this client to seed the document's initial + * content from the file's stored markdown (the editor does the import, guarded + * by the CRDT `initialContentLoaded` flag). + * - `join-error`: the server rejected the join (e.g. lost write access). + */ +interface FileDocProviderEvents { + synced: (synced: boolean) => void + 'seed-request': () => void + 'join-error': (error: JoinFileDocError) => void +} + +/** + * The client half of the collaborative file-document protocol: a Yjs provider + * that carries document sync + awareness over the shared, already-authenticated + * Socket.IO connection (the server relay lives in + * `apps/realtime/src/handlers/file-doc.ts`). It is the Socket.IO analogue of + * `y-websocket`'s `WebsocketProvider` — the same `y-protocols` message framing — + * so TipTap's `Collaboration` (bound to {@link doc}) and `CollaborationCaret` + * (bound to this provider's {@link awareness}) work unmodified. + * + * The document and awareness are owned by the caller (the hook) and are NOT + * destroyed here, so the provider can be torn down and rebuilt (e.g. on a socket + * reconnect) without discarding local edits. + */ +export class FileDocProvider extends ObservableV2 { + synced = false + /** + * Latched `true` when the server elects this client to seed the document. The + * `seed-request` event is transient and can fire before a consumer subscribes, + * so consumers read this flag on subscription rather than relying on the event. + */ + shouldSeed = false + /** + * The latched non-retryable join rejection, or `null`. Like {@link shouldSeed}, + * the `join-error` event is transient and can fire before a consumer subscribes, + * so consumers read this on subscription to detect a fatal failure they missed. + */ + joinError: JoinFileDocError | null = null + + private disposed = false + /** Set on a non-retryable join rejection (e.g. lost write access) so the + * provider stops attempting to (re)join until the owner tears it down. */ + private fatal = false + + constructor( + private readonly socket: Socket, + private readonly fileId: string, + readonly doc: Y.Doc, + readonly awareness: awarenessProtocol.Awareness + ) { + super() + + socket.on(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) + socket.on(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) + socket.on(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + socket.on(FILE_DOC_EVENTS.SEED_REQUEST, this.handleSeedRequest) + socket.on('connect', this.handleConnect) + doc.on('update', this.handleDocUpdate) + awareness.on('update', this.handleAwarenessUpdate) + + if (socket.connected) this.join() + } + + /** Join the room, binding our client id so the server only accepts awareness we own. */ + private join = () => { + if (this.fatal) return + this.socket.emit(FILE_DOC_EVENTS.JOIN, { fileId: this.fileId, clientId: this.doc.clientID }) + } + + /** + * Re-join after a (re)connect. The server re-registers the room before acking, + * so the sync/awareness exchange is deferred to {@link handleJoinSuccess}. + */ + private handleConnect = () => { + if (this.fatal) return + this.setSynced(false) + this.join() + } + + /** + * Handle the join ack. The server registers the room before acking, so an earlier + * send could be dropped — the initial sync + local awareness exchange begins here. + */ + private handleJoinSuccess = (data: JoinFileDocSuccess) => { + if (data.fileId !== this.fileId) return + this.sendSyncStep1() + this.sendLocalAwareness() + } + + /** + * Handle a join rejection. A non-retryable rejection (access denied, invalid) + * won't succeed on retry, so latch {@link fatal} to stop (re)joining and let the + * owner fall back to the non-collaborative view. + */ + private handleJoinError = (data: JoinFileDocError) => { + if (data.fileId !== this.fileId) return + if (data.retryable === false) { + this.fatal = true + this.joinError = data + } + this.emit('join-error', [data]) + } + + private handleSeedRequest = (data: SeedRequestPayload) => { + if (data.fileId !== this.fileId) return + this.shouldSeed = true + this.emit('seed-request', []) + } + + private handleMessage = (data: unknown) => { + const bytes = toFileDocBytes(data) + if (!bytes) return + + const decoder = decoding.createDecoder(bytes) + const messageType = decoding.readVarUint(decoder) + + switch (messageType) { + case FILE_DOC_MESSAGE_TYPE.SYNC: { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + // `this` is the transaction origin, so our own `doc.on('update')` skips + // re-sending updates we just applied from the server. + const syncType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this) + if (encoding.length(encoder) > 1) { + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + if (syncType === syncProtocol.messageYjsSyncStep2 && !this.synced) this.setSynced(true) + break + } + case FILE_DOC_MESSAGE_TYPE.AWARENESS: { + awarenessProtocol.applyAwarenessUpdate( + this.awareness, + decoding.readVarUint8Array(decoder), + this + ) + break + } + } + } + + private handleDocUpdate = (update: Uint8Array, origin: unknown) => { + // Updates we applied from the server carry `this` as origin — don't echo them. + if (origin === this) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, update) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + + private handleAwarenessUpdate = ( + { added, updated, removed }: { added: number[]; updated: number[]; removed: number[] }, + origin: unknown + ) => { + // Only ever publish OUR OWN awareness. Remote changes (origin === this) were + // applied from the server; and a local `Awareness` also emits 30s `timeout` + // removals for remote peers — forwarding either would be a frame for a client + // id we don't own, which the server (correctly) rejects. Filter to our own id + // so honest traffic never trips the ownership guard. + if (origin === this) return + const localId = this.doc.clientID + const changed = [...added, ...updated, ...removed].filter((id) => id === localId) + if (changed.length === 0) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(this.awareness, changed) + ) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + + private sendSyncStep1() { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep1(encoder, this.doc) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + + private sendLocalAwareness() { + if (this.awareness.getLocalState() === null) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(this.awareness, [this.doc.clientID]) + ) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + + private setSynced(synced: boolean) { + if (this.synced === synced) return + this.synced = synced + this.emit('synced', [synced]) + } + + /** + * Tear down the provider: leave the room, clear our awareness (so peers drop our + * caret immediately rather than after the server's 30s timeout), and detach all + * listeners. The document and awareness objects are the caller's and are left intact. + */ + destroy() { + if (this.disposed) { + super.destroy() + return + } + this.disposed = true + + awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'provider-destroy') + + this.socket.emit(FILE_DOC_EVENTS.LEAVE, { fileId: this.fileId }) + this.socket.off(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) + this.socket.off(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) + this.socket.off(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + this.socket.off(FILE_DOC_EVENTS.SEED_REQUEST, this.handleSeedRequest) + this.socket.off('connect', this.handleConnect) + this.doc.off('update', this.handleDocUpdate) + this.awareness.off('update', this.handleAwarenessUpdate) + + super.destroy() + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts new file mode 100644 index 00000000000..9fe069b56d0 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts @@ -0,0 +1,104 @@ +'use client' + +import { useEffect, useMemo, useRef, useState } from 'react' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { getUserColor } from '@/lib/workspaces/colors' +import { useSocket } from '@/app/workspace/providers/socket-provider' +import { FileDocProvider } from './file-doc-provider' + +/** The live collaboration binding the editor wires into TipTap's Collaboration + * (the {@link Y.Doc}) and CollaborationCaret (the awareness). */ +export interface FileDocCollaboration { + /** Bound to TipTap's Collaboration extension (created synchronously at mount). */ + doc: Y.Doc + /** Bound to CollaborationCaret (via `{ awareness }`); relayed by the provider. */ + awareness: Awareness + /** + * The realtime provider, or `null` until the socket is available. `doc` and + * `awareness` exist before it connects, so the editor can bind immediately; the + * provider is consumed for seeding events (`synced` / `seed-request`). + */ + provider: FileDocProvider | null + /** Local caret identity for CollaborationCaret: display name + assigned color. */ + user: { name: string; color: string } +} + +interface UseFileDocCollaborationParams { + fileId: string + userId: string + userName: string + /** + * Whether to establish collaboration. Decided once at editor mount — only for a + * live, editable, non-streaming workspace document. When `false` the hook + * returns `null` and the editor stays fully local. + */ + enabled: boolean +} + +/** + * Owns the per-file Yjs document, awareness, and {@link FileDocProvider} for + * collaborative editing. The document + awareness are created once (this hook + * lives inside an editor that is keyed by file id, so one instance == one file) + * and are the stable objects TipTap binds to; the provider connects them to the + * realtime relay over the shared socket. Returns `null` while disabled. + */ +export function useFileDocCollaboration({ + fileId, + userId, + userName, + enabled, +}: UseFileDocCollaborationParams): FileDocCollaboration | null { + const { socket } = useSocket() + + // The Y.Doc + Awareness are the editor's authoritative binding — created once + // and stable for the hook's life (see sim-react-performance: lazy-init ref). + // Only allocated when collaboration is enabled, so read-only / streaming / + // round-trip-unsafe views never build a Yjs document they won't use. + const docRef = useRef(null) + const awarenessRef = useRef(null) + if (enabled && docRef.current === null) { + docRef.current = new Y.Doc() + awarenessRef.current = new Awareness(docRef.current) + } + + const [provider, setProvider] = useState(null) + + // Declared BEFORE the provider effect so, on unmount, React runs this cleanup + // AFTER the provider effect's cleanup (cleanups run in reverse declaration + // order) — the provider detaches from the doc/awareness before they're destroyed. + useEffect(() => { + return () => { + awarenessRef.current?.destroy() + docRef.current?.destroy() + } + }, []) + + useEffect(() => { + if (!enabled || !socket) return + // Non-null: both refs are lazily set during render, before any effect runs. + const doc = docRef.current as Y.Doc + const awareness = awarenessRef.current as Awareness + const fileProvider = new FileDocProvider(socket, fileId, doc, awareness) + setProvider(fileProvider) + return () => { + fileProvider.destroy() + setProvider(null) + } + }, [enabled, socket, fileId]) + + const user = useMemo(() => ({ name: userName, color: getUserColor(userId) }), [userName, userId]) + + return useMemo( + () => + enabled + ? { + doc: docRef.current as Y.Doc, + awareness: awarenessRef.current as Awareness, + provider, + user, + } + : null, + [enabled, provider, user] + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts index 601d702c07c..12190e62718 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts @@ -1,5 +1,10 @@ import type { Extensions } from '@tiptap/core' +import Collaboration from '@tiptap/extension-collaboration' +import CollaborationCaret from '@tiptap/extension-collaboration-caret' import Placeholder from '@tiptap/extension-placeholder' +import type { Awareness } from 'y-protocols/awareness' +import type * as Y from 'yjs' +import { withAlpha } from '@/lib/workspaces/colors' import { BlockMover } from './block-mover' import { CodeBlockWithLanguage } from './code-block' import { CodeBlockHighlight } from './code-highlight' @@ -13,10 +18,20 @@ import { MentionChip } from './mention/mention-chip' import { FootnoteDefWithView, RawHtmlBlockWithView } from './raw-markdown-snippet' import { SlashCommand } from './slash-command/slash-command' +/** Live collaboration binding for the editor. When present, the editor's history + * is Yjs-backed and remote carets/selection render via CollaborationCaret. */ +export interface EditorCollaboration { + doc: Y.Doc + awareness: Awareness + user: { name: string; color: string } +} + interface MarkdownEditorExtensionOptions { placeholder: string /** Renders supported media links as live players beneath a standalone link. Off by default. */ embeds?: boolean + /** When set, wires TipTap Collaboration + CollaborationCaret onto the shared document. */ + collaboration?: EditorCollaboration } /** @@ -32,15 +47,38 @@ interface MarkdownEditorExtensionOptions { export function createMarkdownEditorExtensions({ placeholder, embeds = false, + collaboration, }: MarkdownEditorExtensionOptions): Extensions { return [ - ...createMarkdownContentExtensions({ - codeBlock: CodeBlockWithLanguage, - image: ResizableImage, - mention: MentionChip, - rawHtmlBlock: RawHtmlBlockWithView, - footnoteDef: FootnoteDefWithView, - }), + ...createMarkdownContentExtensions( + { + codeBlock: CodeBlockWithLanguage, + image: ResizableImage, + mention: MentionChip, + rawHtmlBlock: RawHtmlBlockWithView, + footnoteDef: FootnoteDefWithView, + }, + { disableHistory: Boolean(collaboration) } + ), + ...(collaboration + ? [ + Collaboration.configure({ document: collaboration.doc }), + // CollaborationCaret reads only `provider.awareness` (created synchronously, + // relayed by the socket provider once connected). The default caret + label + // color from `user.color`; only the selection tint needs an explicit override. + CollaborationCaret.configure({ + provider: { awareness: collaboration.awareness }, + user: collaboration.user, + selectionRender: (user) => { + const hex = typeof user.color === 'string' ? user.color : '#000000' + return { + class: 'collaboration-carets__selection', + style: `background-color: ${withAlpha(hex, 0.2)};`, + } + }, + }), + ] + : []), CodeBlockHighlight, SlashCommand, Mention, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts index 52e4731f27f..0f83b3a32ef 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts @@ -117,7 +117,10 @@ export interface ContentNodeViews { * registry. The live editor passes the node-view nodes via {@link createMarkdownEditorExtensions}; the * schema and markdown output are identical either way. */ -export function createMarkdownContentExtensions(nodeViews: ContentNodeViews = {}): Extensions { +export function createMarkdownContentExtensions( + nodeViews: ContentNodeViews = {}, + options: { disableHistory?: boolean } = {} +): Extensions { const codeBlock = (nodeViews.codeBlock ?? MarkdownCodeBlock).configure({ HTMLAttributes: { class: 'code-editor-theme' }, }) @@ -128,6 +131,9 @@ export function createMarkdownContentExtensions(nodeViews: ContentNodeViews = {} codeBlock: false, code: false, paragraph: false, + // Collaboration provides its own (Yjs-backed) undo/redo — disabling the + // built-in history avoids the two fighting over the shared document. + ...(options.disableHistory ? { undoRedo: false as const } : {}), }), BlockSafeParagraph, InlineCode, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css index cfb5e9a2ec3..2ec9890d8be 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css @@ -454,3 +454,57 @@ .rich-markdown-field-prose p.is-editor-empty:first-child::before { color: var(--text-muted); } + +/* + * Collaborative carets (TipTap CollaborationCaret). The caret border and the name + * label's background are colored inline from each user's assigned color; the + * selection is a translucent tint of that color (set via selectionRender). The + * name label is hidden until the caret is hovered — a quiet, Google-Docs-style + * presence that doesn't clutter the text. + */ +.rich-markdown-prose .collaboration-carets__caret { + position: relative; + margin-left: -1px; + margin-right: -1px; + border-left-width: 1px; + border-left-style: solid; + border-right-width: 1px; + border-right-style: solid; + word-break: normal; +} + +.rich-markdown-prose .collaboration-carets__label { + position: absolute; + top: -1.4em; + left: -1px; + padding: 0.1rem 0.35rem; + border-radius: 3px 3px 3px 0; + font-size: 11px; + font-weight: 600; + line-height: 1.2; + white-space: nowrap; + /* Fixed dark text: the label background is always a light user color (assigned + * inline, theme-independent), so a theme token would go unreadable in one mode. */ + color: #1a1a1a; + user-select: none; + pointer-events: none; + opacity: 0; + transition: opacity 0.12s ease; +} + +/* Widen the caret's hover target beyond its 1px width so the name label is easy to + * reveal; the label itself stays pointer-events:none and never intercepts clicks. */ +.rich-markdown-prose .collaboration-carets__caret::before { + content: ""; + position: absolute; + inset: -0.1em -2px; +} + +.rich-markdown-prose .collaboration-carets__caret:hover .collaboration-carets__label { + opacity: 1; +} + +.rich-markdown-prose .collaboration-carets__selection { + border-radius: 2px; + pointer-events: none; +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 9107dbd795d..2b9514b4f6a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -2,13 +2,15 @@ import { memo, useEffect, useRef, useState } from 'react' import { cn, toast } from '@sim/emcn' -import type { JSONContent } from '@tiptap/core' +import type { JoinFileDocError } from '@sim/realtime-protocol/file-doc' +import type { Extensions, JSONContent } from '@tiptap/core' import { Fragment, Slice } from '@tiptap/pm/model' import { NodeSelection } from '@tiptap/pm/state' import { dropPoint } from '@tiptap/pm/transform' import type { Editor } from '@tiptap/react' import { EditorContent, useEditor } from '@tiptap/react' import { useRouter } from 'next/navigation' +import { useSession } from '@/lib/auth/auth-client' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { extractEmbeddedFileRef } from '@/lib/uploads/utils/embedded-image-ref' import { useUploadWorkspaceFile } from '@/hooks/queries/workspace-files' @@ -16,6 +18,7 @@ import type { SaveStatus } from '@/hooks/use-autosave' import { useFileContentSource } from '@/hooks/use-file-content-source' import { PreviewLoadingFrame } from '../preview-shared' import { useEditableFileContent } from '../use-editable-file-content' +import { useFileDocCollaboration } from './collaboration/use-file-doc-collaboration' import { createMarkdownEditorExtensions } from './editor-extensions' import { findHeadingPos } from './heading-anchors' import { @@ -41,8 +44,10 @@ import { isRoundTripSafe } from './round-trip-safety' import '@sim/emcn/components/code/code.css' import './rich-markdown-editor.css' +const PLACEHOLDER = "Write something, or press '/' for commands…" + const EXTENSIONS = createMarkdownEditorExtensions({ - placeholder: "Write something, or press '/' for commands…", + placeholder: PLACEHOLDER, embeds: true, }) @@ -71,6 +76,13 @@ interface RichMarkdownEditorProps { previewContextKey?: string /** Disable the `@` tag-insertion menu (existing tags still render). Defaults off — the file editor keeps tagging. */ disableTagging?: boolean + /** + * Opt this surface into live collaborative editing. Set only by the Files page — + * the dedicated editing surface, which never streams agent output. The agent/Chat + * surface leaves it off, so collaboration and agent-streaming are disjoint by + * construction (they cannot both drive one editor and corrupt the shared doc). + */ + collaborative?: boolean } /** Inline WYSIWYG markdown editor: agent output streams in read-only, then the same instance becomes editable on settle. */ @@ -89,7 +101,20 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ disableStreamingAutoScroll = false, previewContextKey, disableTagging, + collaborative = false, }: RichMarkdownEditorProps) { + const { data: session, isPending: isSessionPending } = useSession() + const userId = session?.user?.id ?? '' + const userName = session?.user?.name?.trim() || 'Collaborator' + + /** + * Autosave gate for the collaborative path: the child reports `false` while its + * shared document is still syncing/seeding and `true` once it is safe to persist + * the markdown mirror — so an empty or partially-synced doc can never overwrite + * the real file. `true` for non-collaborative files (never gated). + */ + const [collabReady, setCollabReady] = useState(true) + const { content, setDraftContent, @@ -108,9 +133,14 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ saveRef, discardRef, normalizeBaseline: normalizeMarkdownContent, + canAutosave: collabReady, }) - if (isContentLoading) return + // Wait for the session too: the child decides collaboration ONCE at mount from + // `userId`, so mounting before the session resolves would latch collaboration off + // for a cold-loaded file (both users would then solo-save, last-write-wins). + if (isContentLoading || isSessionPending) + return if (hasContentError) { return ( @@ -128,12 +158,16 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ content={content} isStreaming={isStreamInteractionLocked} canEdit={canEdit} + userId={userId} + userName={userName} autoFocus={autoFocus} streamIsIncremental={streamIsIncremental} disableStreamingAutoScroll={disableStreamingAutoScroll} disableTagging={disableTagging} + collaborative={collaborative} onChange={setDraftContent} onSaveShortcut={saveImmediately} + onCollabReadyChange={setCollabReady} /> ) }) @@ -146,13 +180,20 @@ interface LoadedRichMarkdownEditorProps { /** True while agent output is streaming in: the editor renders it read-only and syncs each chunk. */ isStreaming: boolean canEdit: boolean + /** Current user id + display name, for the collaborative caret identity. */ + userId: string + userName: string autoFocus?: boolean /** See {@link RichMarkdownEditorProps.streamIsIncremental}. */ streamIsIncremental?: boolean disableStreamingAutoScroll?: boolean disableTagging?: boolean + /** See {@link RichMarkdownEditorProps.collaborative}. */ + collaborative?: boolean onChange: (markdown: string) => void onSaveShortcut: () => Promise + /** Reports whether the collaborative document is synced+seeded (autosave gate). */ + onCollabReadyChange: (ready: boolean) => void } interface SettledContent { @@ -172,12 +213,16 @@ export function LoadedRichMarkdownEditor({ content, isStreaming, canEdit, + userId, + userName, autoFocus, streamIsIncremental, disableStreamingAutoScroll, disableTagging, + collaborative = false, onChange, onSaveShortcut, + onCollabReadyChange, }: LoadedRichMarkdownEditorProps) { /** Whether this editor mounted mid-stream — if so it starts empty and syncs streamed chunks until settle. */ const streamingAtMountRef = useRef(isStreaming) @@ -187,11 +232,48 @@ export function LoadedRichMarkdownEditor({ if (!streamingAtMountRef.current && settledRef.current === null) { settledRef.current = lockSettled(content) } - const isEditable = canEdit && !isStreaming && (settledRef.current?.verdict ?? false) + /** + * Collaboration is decided once at mount from synchronously-available inputs + * (`settledRef` is set just above) via `useState`-init, and never changes — TipTap + * fixes the extension set at editor creation, so it cannot turn on later. Enabled + * only on a `collaborative` surface (the Files page, which never streams) for an + * editable, round-trip-safe, non-streaming workspace document with a known user. + */ + const [collaborationEnabled] = useState( + () => + collaborative && + canEdit && + !streamingAtMountRef.current && + (settledRef.current?.verdict ?? false) && + Boolean(userId) && + (file.storageContext ?? 'workspace') === 'workspace' + ) + /** + * Whether the collaborative document is safe to edit + persist: synced and seeded. + * Starts `false` for a collaborative document — so the editor is read-only and + * autosave gated until the shared content has arrived (a user must not type into an + * empty, unsynced doc, which the seed would then discard) — and `true` for a local one. + */ + const [collabReady, setCollabReady] = useState(!collaborationEnabled) + const isEditable = + canEdit && !isStreaming && (settledRef.current?.verdict ?? false) && collabReady + + const collaboration = useFileDocCollaboration({ + fileId: file.id, + userId, + userName, + enabled: collaborationEnabled, + }) - /** Seed the doc once via lazy init — chunked parse is linear vs the editor's ~O(n²) whole-body markdown parse. */ + /** + * Initial editor content. When collaborating, the Y.Doc is the source of truth — + * start empty and let the seed handshake fill it (below); otherwise seed from the + * parsed markdown (chunked parse is linear vs the editor's ~O(n²) whole-body parse). + */ const [initialContent] = useState(() => - streamingAtMountRef.current ? '' : parseMarkdownToDoc(splitFrontmatter(content).body) + streamingAtMountRef.current || collaborationEnabled + ? '' + : parseMarkdownToDoc(splitFrontmatter(content).body) ) /** * The body currently shown in the editor: seeded from a settled mount, updated on local edits (via @@ -289,8 +371,27 @@ export function LoadedRichMarkdownEditor({ } } + /** + * Extensions: the shared module set for the local path, or a per-instance set + * carrying this document's Collaboration + CollaborationCaret. Built once (collab + * is decided at mount), since `useEditor` fixes the extension set at creation. + */ + const [extensions] = useState(() => + collaboration + ? createMarkdownEditorExtensions({ + placeholder: PLACEHOLDER, + embeds: true, + collaboration: { + doc: collaboration.doc, + awareness: collaboration.awareness, + user: collaboration.user, + }, + }) + : EXTENSIONS + ) + const editor = useEditor({ - extensions: EXTENSIONS, + extensions, editable: isEditable, enablePasteRules: false, autofocus: streamingAtMountRef.current ? false : autoFocus ? 'end' : false, @@ -449,6 +550,99 @@ export function LoadedRichMarkdownEditor({ }) editorInstanceRef.current = editor + /** + * The loaded markdown to seed the shared doc from, held by pointer so the parse + * runs once at seed time rather than every render. + */ + const seedContentRef = useRef(content) + seedContentRef.current = content + + /** + * The collaborative document lifecycle. In one effect because the three concerns + * are one state machine keyed off the same provider events: + * - **seed** the doc from the loaded markdown when this client is elected and + * synced (content + `initialContentLoaded` flag in ONE Yjs transaction, so a + * re-election can never duplicate content — the relay's exactly-once contract); + * - **gate** the parent's autosave until the doc is synced AND seeded, so an + * empty/still-syncing doc can never overwrite the real file's markdown mirror; + * - **fall back** on a fatal join: seed the loaded content so it is SHOWN, but + * leave the editor read-only + gated. Every non-retryable failure (auth, access + * denied, not found, client-id conflict) either can't save or is moot, so the + * safe fallback is a read-only view of the content rather than editable-but- + * unsavable — which would silently drop the user's edits. + * + * `ready` (synced+seeded) gates BOTH the editor's editability (a user must never + * type into an empty/unsynced doc) and the parent's autosave. Non-collaborative + * documents are never gated. `provider.shouldSeed` / `provider.joinError` are + * latched, so events that fired before this subscription are not missed. + */ + useEffect(() => { + const setReady = (ready: boolean) => { + setCollabReady(ready) + onCollabReadyChange(ready) + } + if (!collaboration) { + setReady(true) + return + } + const { provider, doc } = collaboration + if (!editor) { + setReady(false) + return + } + const config = doc.getMap('config') + + const seedFromLoaded = () => { + if (config.get('initialContentLoaded') === true) return + doc.transact(() => { + editor.commands.setContent( + parseMarkdownToDoc(splitFrontmatter(seedContentRef.current).body), + { contentType: 'json', emitUpdate: false } + ) + config.set('initialContentLoaded', true) + }) + } + + if (!provider) { + setReady(false) + return + } + + const report = () => setReady(provider.synced && config.get('initialContentLoaded') === true) + const onProgress = () => { + if (provider.shouldSeed && provider.synced) seedFromLoaded() + report() + } + const onJoinError = (error: JoinFileDocError) => { + if (error.retryable === false) seedFromLoaded() + } + + provider.on('seed-request', onProgress) + provider.on('synced', onProgress) + provider.on('join-error', onJoinError) + config.observe(report) + onProgress() + if (provider.joinError) onJoinError(provider.joinError) + + return () => { + provider.off('seed-request', onProgress) + provider.off('synced', onProgress) + provider.off('join-error', onJoinError) + config.unobserve(report) + onCollabReadyChange(true) + } + }, [collaboration, editor, onCollabReadyChange, setCollabReady]) + + /** + * Owns editability for the collaborative lifecycle: `useEditor`'s `editable` is only + * the initial value, and the streaming/settle effect stays inert in collab mode — so + * re-apply here whenever collaboration readiness (synced + seeded) flips `isEditable`. + */ + useEffect(() => { + if (!editor || !collaborationEnabled) return + if (editor.isEditable !== isEditable) editor.setEditable(isEditable) + }, [editor, collaborationEnabled, isEditable]) + /** * Wire the `/Image` slash command to the hidden picker (per-editor storage, since the extension set is * shared across instances). Reads only refs, so the handler stays stable across the editor's life. @@ -473,6 +667,13 @@ export function LoadedRichMarkdownEditor({ const lastStreamParseAtRef = useRef(0) useEffect(() => { if (!editor) return + // Collaboration and agent-streaming are disjoint surfaces: collaboration is enabled + // only on the Files page, which never streams agent output, so in collab mode this + // reconcile loop stays fully inert. It must not run even defensively — its + // `setContent` would sync a full-document replace into the shared Y.Doc (the + // ySyncPlugin writes it regardless of `emitUpdate: false`), wiping peers' edits. + // Editability is owned by the reactive effect above. + if (collaborationEnabled) return const syncEditorBody = (body: string) => { if (body === lastSyncedBodyRef.current) return lastSyncedBodyRef.current = body @@ -542,13 +743,22 @@ export function LoadedRichMarkdownEditor({ // entirely. `setTextSelection` (not `.focus()`) so this never steals DOM focus from whatever the // user is doing outside the editor. editor.commands.setTextSelection(editor.state.doc.content.size) - editor.setEditable(canEdit && settledRef.current.verdict) + editor.setEditable(canEdit && settledRef.current.verdict && collabReady) if (isInitialSettle && autoFocus) editor.commands.focus('end') return } syncEditorBody(splitFrontmatter(content).body) - if (settledRef.current) editor.setEditable(canEdit && settledRef.current.verdict) - }, [editor, content, isStreaming, canEdit, autoFocus, disableStreamingAutoScroll]) + if (settledRef.current) editor.setEditable(canEdit && settledRef.current.verdict && collabReady) + }, [ + editor, + content, + isStreaming, + canEdit, + autoFocus, + disableStreamingAutoScroll, + collaborationEnabled, + collabReady, + ]) useEffect( () => () => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts index cb6defd8d95..f74e38660c2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts @@ -66,6 +66,13 @@ interface UseEditableFileContentOptions { * the at-rest baseline, never while an agent stream is in flight. Stable reference required. */ normalizeBaseline?: (raw: string) => string + /** + * Extra gate on autosave (and draft persistence). When `false`, saving is + * suppressed even when otherwise eligible — the collaborative editor uses it to + * hold saves until the shared document is synced AND seeded, so an empty or + * partially-synced doc can never overwrite the real file. Defaults to `true`. + */ + canAutosave?: boolean } interface EditableFileContent { @@ -141,6 +148,7 @@ export function useEditableFileContent({ saveRef, discardRef, normalizeBaseline, + canAutosave = true, }: UseEditableFileContentOptions): EditableFileContent { const onDirtyChangeRef = useRef(onDirtyChange) const onSaveStatusChangeRef = useRef(onSaveStatusChange) @@ -239,7 +247,7 @@ export function useEditableFileContent({ [workspaceId, file.id, markSavedContent] ) - const autosaveEnabled = canEdit && isInitialized && !isStreamInteractionLocked + const autosaveEnabled = canEdit && isInitialized && !isStreamInteractionLocked && canAutosave const { saveStatus, saveImmediately, isDirty, discard } = useAutosave({ content, diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 9128bebf31d..29531265471 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -1924,6 +1924,7 @@ export function Files() { onSaveStatusChange={handleSaveStatusChange} saveRef={saveRef} discardRef={discardRef} + collaborative /> { - const colorMap = new Map() - let colorIndex = 0 - - for (const userId of userIds) { - if (!colorMap.has(userId)) { - colorMap.set(userId, colorIndex++) - } - } - - return colorMap -} diff --git a/apps/sim/package.json b/apps/sim/package.json index f0aaa82fa3a..6914862cd0f 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -118,6 +118,8 @@ "@tanstack/react-virtual": "3.13.24", "@tiptap/core": "3.26.1", "@tiptap/extension-code-block": "3.26.1", + "@tiptap/extension-collaboration": "3.26.1", + "@tiptap/extension-collaboration-caret": "3.26.1", "@tiptap/extension-image": "3.26.1", "@tiptap/extension-list": "3.26.1", "@tiptap/extension-placeholder": "3.26.1", @@ -170,6 +172,7 @@ "js-yaml": "4.3.0", "json5": "2.2.3", "jszip": "3.10.1", + "lib0": "0.2.117", "lru-cache": "11.3.6", "lucide-react": "^0.479.0", "mammoth": "^1.9.0", @@ -222,6 +225,8 @@ "undici": "7.28.0", "unpdf": "1.4.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", + "y-protocols": "1.0.7", + "yjs": "13.6.31", "zod": "4.3.6", "zustand": "^5.0.13" }, diff --git a/bun.lock b/bun.lock index f1e8d8bde90..9fe1aa4b52c 100644 --- a/bun.lock +++ b/bun.lock @@ -189,6 +189,8 @@ "@tanstack/react-virtual": "3.13.24", "@tiptap/core": "3.26.1", "@tiptap/extension-code-block": "3.26.1", + "@tiptap/extension-collaboration": "3.26.1", + "@tiptap/extension-collaboration-caret": "3.26.1", "@tiptap/extension-image": "3.26.1", "@tiptap/extension-list": "3.26.1", "@tiptap/extension-placeholder": "3.26.1", @@ -241,6 +243,7 @@ "js-yaml": "4.3.0", "json5": "2.2.3", "jszip": "3.10.1", + "lib0": "0.2.117", "lru-cache": "11.3.6", "lucide-react": "^0.479.0", "mammoth": "^1.9.0", @@ -293,6 +296,8 @@ "undici": "7.28.0", "unpdf": "1.4.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", + "y-protocols": "1.0.7", + "yjs": "13.6.31", "zod": "4.3.6", "zustand": "^5.0.13", }, @@ -1774,6 +1779,10 @@ "@tiptap/extension-code-block": ["@tiptap/extension-code-block@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1", "@tiptap/pm": "3.26.1" } }, "sha512-NY7SYqcrqDVYTSWyaNGdSfCims6pOHoRQ2Rh4DEFb/rb8gLVkqbLZhcHzQCVfinlPqgV3xWF6cYMORwmnlBkXQ=="], + "@tiptap/extension-collaboration": ["@tiptap/extension-collaboration@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1", "@tiptap/pm": "3.26.1", "@tiptap/y-tiptap": "^3.0.5", "yjs": "^13" } }, "sha512-NLF3tWPla1bg9VAaICTrheEjDi9ZFGk1HhoALfVHWSwIqnUpVxSubyBxw/PFWyHYZNQrxaGvypf457NHHINolA=="], + + "@tiptap/extension-collaboration-caret": ["@tiptap/extension-collaboration-caret@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1", "@tiptap/pm": "3.26.1", "@tiptap/y-tiptap": "^3.0.5" } }, "sha512-fJpIO8eXYksiTyxck3e7vS2Fsddo+sJNaQdNHXuDykVZzH4LPXAHyGGdSRrLM4aMUsK1IZh8CPyti1CHwbz7vA=="], + "@tiptap/extension-document": ["@tiptap/extension-document@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1" } }, "sha512-6W2vZjvi0Mv+4xEtwMDGhWwo7FotWR6eKfmntmduvehWevFpMxOKcTtyotjLigfZv738y50YWmvbaPuAPJG3BA=="], "@tiptap/extension-dropcursor": ["@tiptap/extension-dropcursor@3.26.1", "", { "peerDependencies": { "@tiptap/extensions": "3.26.1" } }, "sha512-eVq3BvFIa3YD+pBIlj1i72vYEixlegGVKHnSYiVF2ovkQOSAH9sca7pkq6WgV1sMTCyWCU8e+WznTqtydvHUWA=="], @@ -1826,6 +1835,8 @@ "@tiptap/suggestion": ["@tiptap/suggestion@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1", "@tiptap/pm": "3.26.1" } }, "sha512-Bg8IyuDC92InSPzcHvCT3+ZDCJSMJIEINdFg513RPQzwZTw1dsrU0K00XYcDT6lOhZwLM2IVTiE6sZl2GY25Rg=="], + "@tiptap/y-tiptap": ["@tiptap/y-tiptap@3.0.7", "", { "dependencies": { "lib0": "^0.2.100" }, "peerDependencies": { "prosemirror-model": "^1.7.1", "prosemirror-state": "^1.2.3", "prosemirror-view": "^1.9.10", "y-protocols": "^1.0.1", "yjs": "^13.5.38" } }, "sha512-3VG01F7i2JDghWsBZSBKi7ypMiN5UVVSyD18IwoXx7ilm0ZynrTS+XNpmgxfuiSBjdauWk7tUlP04xfM8Bw4Vw=="], + "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], "@tootallnate/once": ["@tootallnate/once@2.0.1", "", {}, "sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ=="], From 790d93a88bb7738a85a7c52cc8a3e2284332baf1 Mon Sep 17 00:00:00 2001 From: Waleed Date: Sat, 25 Jul 2026 09:04:53 -0700 Subject: [PATCH 08/53] =?UTF-8?q?feat(tables):=20live=20collaboration=20?= =?UTF-8?q?=E2=80=94=20cell-selection=20presence=20+=20live=20mutation=20p?= =?UTF-8?q?ropagation=20(#5957)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(tables): live cell-selection presence — protocol + server + client hook The realtime spine for Google-Sheets-style table presence (mode A, socket): - @sim/realtime-protocol/table-presence: centralized wire protocol (events + TableCellSelection {anchor, focus, editing} + payloads) so server emits and client subscriptions can't drift. - ROOM_TYPES.TABLE + resolveTableWorkspace registered in ROOM_WORKSPACE_RESOLVERS (tableId -> workspace via userTableDefinitions, honoring archivedAt); roomName / presenceEventName / disconnect cleanup / authorizeRoom all derive automatically. - apps/realtime/src/handlers/tables.ts: join/leave (mirrors workspace-files) + a table-cell-selection relay (mirrors the workflow selection channel), broadcasting via roomName(room) since table rooms are namespaced. UserPresence gains a cell field threaded through the memory + Redis managers (Lua ARGV[7], null clears). - Extracted the duplicated resolveAvatarUrl into handlers/avatar.ts. - use-table-room.ts client hook: joins over the shared socket, tracks the roster (avatars) + patches per-socket cell deltas, exposes a throttled emitCellSelection. Grid UI (avatars + selection overlay) lands next; concurrent cell-value edits (last-write-wins via the durable log) are the follow-up PR. * feat(tables): render live cell-selection presence in the grid Wires the table presence room into the grid UI: - Page (table.tsx): useTableRoom (gated off in embedded/mothership mode) — renders in the header and passes remoteSelections + emitCellSelection down to the grid. - Grid emits its local selection: an effect resolves the index-based anchor/focus to stable (rowId, columnId) via refs and broadcasts it (with an editing flag for the active cell) through the throttled emitter. - RemoteSelectionOverlay: draws each remote viewer's selection in their color (getUserColor), a darker fill while editing, and name-on-hover — measured from live cell rects in the content wrapper's space (scrolls with the grid), hidden when rows are virtualized off-window, pointer-events-none so it never blocks cell clicks (hover via pointer hit-test). * test(tables): cover the table presence handler Mirrors workspace-files.test.ts: join auth/unavailable/denied/success, plus the cell-selection relay (asserts it persists via updateUserActivity and broadcasts on the namespaced roomName, not the bare id) and leave. * feat(tables): propagate manual cell edits live (last-write-wins) A manual row edit now appends a lightweight 'edit' event to the durable table stream; collaborators refetch the row (via the existing debounced rows-invalidate the job events use) so the winning value shows live. The event carries no value — peers refetch in their own wire format, so there's no auth-specific value translation on the wire, and last-write-wins falls out of the DB's committed order (the Google-Sheets model). Edits that also trigger a dispatch already emit dispatch/cell events; the debounce coalesces the two. * refactor(tables): apply /simplify findings - Drop the dead 'add unknown peer' upsert branch in use-table-room (Socket.IO ordering guarantees a peer is in the roster before their selection delta). - TableCellSelectionBroadcast = TablePresenceUser & { cell } (was a copy-paste). - Make TableGrid's presence props required + drop the unused empty-default/guard (only table.tsx mounts it, always passing both). - Drop the unused rowId from the 'edit' event (the handler invalidates all rows). - Overlay: subscribe scroll/resize/pointer listeners once per scroll element and cache the wrapper origin, so incoming deltas re-measure without re-subscribing and the pointer hit-test never forces a per-move layout read. - Server: cache the immutable socket session so a selection delta no longer reads it from Redis every time. * refactor(tables): apply /cleanup findings - Fix the remote-selection name label contrast: text-white is unreadable on the light-pastel user colors (same bug the Files caret fixed) → fixed dark #1a1a1a. - Re-measure via useLayoutEffect so a moving peer selection updates before paint (no one-frame position lag). - Drop 'mothership' from a comment (constitution copy rule). Six cleanup passes ran (effect, memo/callback, state, react-query, emcn, comment); the rest confirmed clean — all state/memos/callbacks/effects are load-bearing, presence correctly lives in useState (socket-pushed), and the edit→rows-invalidate granularity is right. * feat(tables): propagate every table mutation live (edit + schema signals) Comprehensive live collaboration for all user table mutations, via two value-less durable signals + named helpers (signalTableRowsChanged / signalTableSchemaChanged): - edit (rows refetch): single + batch row create, cell/row update, batch update, delete by id/filter, and upsert. - schema (definition + rows refetch): column add/update/delete, workflow-group add/update/delete, table rename, and CSV import (which can add columns). - Client handles 'schema' by invalidating the table detail (exact) + rows. Execution paths (column run, cancel-runs) and async jobs (delete/import-async, job-cancel) already propagate via cell/dispatch/job events — verified applyJob refetches on terminal. No reorder routes exist. Table archive (route DELETE) is a deliberate follow-up: it needs a table-deleted redirect event, not a refetch signal (which would 404). * refactor(tables): apply comprehensive /cleanup audit findings Holistic + react-query + comment audits over the whole PR: - Security/crash fix: a remote peer's rowId flowed unescaped into the overlay's querySelector — a hostile id ('x"]') threw SyntaxError inside a useLayoutEffect, crashing every other viewer's page. CSS.escape it, and validate + whitelist the untrusted cell payload server-side (shape + 200-char id bound) before it is stored/rebroadcast. - Simplify the CELL_SELECTION relay: the delta attached userId/userName/avatarUrl that the client discarded (identity comes from the roster). Drop them + the getUserSession lookup/cache entirely — the delta is now { socketId, cell }. - React Query: schema handler also invalidates lists() (parity with the local column-mutation set); document that the mutating client self-refetches by design. - Comment tightenings; biome fixed a stale import order in workspace-files.ts. * fix(tables): broadcast single-cell selections (focus falls back to anchor) Cursor High: a normal cell click leaves selectionFocus null (the grid treats it as a one-cell selection via focus ?? anchor), but the presence emit required BOTH anchor and focus to resolve — so the most common selection never broadcast and clicking even cleared a prior remote outline. Mirror the grid's focus ?? anchor semantics. * fix(tables): reviewer + regression + per-LOC audit findings Cursor review round (5 findings) + regression audit + per-LOC audit: - Presence roster snapshot now KEEPS the cell we already hold for a known socket, so a join/leave broadcast can't revert a fresher CELL_SELECTION delta. - Reset the selection throttle on table switch (was unmount-only), so a pending selection for table A can't flush into table B's room after a switch. - Metadata writes (column widths, display) use a new lightweight 'metadata' signal that refetches only the definition — a resize no longer forces peers to refetch rows. - Overlay re-measures on row add/remove/reorder via a tbody childList MutationObserver (a live refetch moves cells without a scroll/resize). - Document the actor self-refetch create caveat (scrolled multi-page insert) accurately. - isCellRef narrows to a partial instead of casting to the full type then re-checking; drop a redundant mount measure() (the layout effect covers it); text-[11px]→text-xs. * fix(tables): drop ineffective metadata propagation + re-measure overlay on column resize Cursor round on b8f28b04b: - Remove the 'metadata' signal entirely. The grid seeds columnWidths/pinnedColumns from metadata ONCE (metadataSeededRef) and deliberately never re-applies them (to avoid clobbering a local in-progress resize), so refetching the definition on a peer never surfaced their width/pin change — an ineffective path. Width/pin live-sync needs reconciliation that doesn't clobber a local resize; that's a deliberate follow-up, not a no-op refetch. Structural changes still propagate via 'schema'. - Overlay now also observes the content layer with the ResizeObserver, so a column resize (which grows the content, not the scroll container) re-measures remote outlines. - Presence-merge comment now states both sides of the trade-off. * fix(tables): re-broadcast local selection on (re)join Cursor Medium: a selection made before the room join completes (or held across a reconnect) was dropped server-side and never re-sent, so peers didn't see it until the local user moved it again. Track the current selection in a ref (set on every emit, cleared on table switch) and re-emit it from handleJoinSuccess once the room is joined. * fix(tables): re-broadcast selection when a peer's row change shifts it End-to-end lifecycle audit (Low-Med): the selection emit resolved the stable (rowId, columnId) only on selection/editing change, not when a live edit/schema refetch inserted/deleted/reordered rows. The index-based local selection then sat on a different logical row than the rowId peers held, so your outline showed on the old row until you moved. Re-run the emit on rows/displayColumns change and dedup an unchanged result (also drops the redundant null-on-open emit) so the broadcast stays consistent with the local highlight. * fix(tables): schema invalidates run-state/enrichment + guard stale join Cursor round on cdc8796b8 (2 Medium): - schema handler used detail exact:true, so it skipped the activeDispatches + enrichmentDetails sibling queries the local invalidateTableSchema refreshes via a prefix match. After a peer deletes/restructures a workflow group, peers could keep a stale running badge or enrichment panel. Now invalidates both siblings too (rows stay on the debounce). - Guard against a stale join stealing the room: a fast table A->B switch could let A's async authorize finish after B, leave B, and strand the socket in A. Added a per-socket monotonic join generation checked after authorize (mirrors the file-doc relay's guard) + a test. * feat(tables): live column width/pin/order sync Collaborators now see each other's column resizes, pins, and reorders live — the last piece of Google-Sheets-style layout parity. - New lightweight `metadata` durable event kind (distinct from `schema`): only the table definition carries UI metadata, so peers refetch the definition alone — no rows/run-state refetch. The metadata PUT route now signals it. - The grid reconciles server metadata against its in-progress gesture: the column being actively resized keeps its live local width, and an in-flight column drag blocks a reorder apply — so a peer's change never reverts the local action. Each field is reference-guarded (React Query structural sharing keeps unchanged sub-objects stable), so an unrelated peer change doesn't re-apply the others. * fix(tables): escalate to schema signal when a reorder scrubs group deps Independent audit of the metadata-sync commit found a stale-run-state hole: a columnOrder PUT that moves a column left of a workflow group's leftmost column makes updateTableMetadata scrub that group's dependencies and write a new schema — a real structural change. But the route only fired the lightweight 'metadata' signal (detail-only refetch), so peers' and the actor's activeDispatches / enrichmentDetails queries stayed stale (a lingering running badge / enrichment panel) — exactly what the 'schema' handler exists to prevent. updateTableMetadata now reports whether it scrubbed the schema; the route emits signalTableSchemaChanged in that case and the light signalTableMetadataChanged otherwise. Width/pin/plain-reorder stay on the cheap detail-only path. --- apps/realtime/src/handlers/avatar.ts | 29 ++ apps/realtime/src/handlers/index.ts | 2 + apps/realtime/src/handlers/tables.test.ts | 234 ++++++++++++++++ apps/realtime/src/handlers/tables.ts | 255 ++++++++++++++++++ apps/realtime/src/handlers/workspace-files.ts | 21 +- apps/realtime/src/rooms/memory-manager.ts | 3 +- apps/realtime/src/rooms/redis-manager.ts | 11 +- apps/realtime/src/rooms/types.ts | 7 +- .../app/api/table/[tableId]/columns/route.ts | 4 + .../app/api/table/[tableId]/groups/route.ts | 4 + .../app/api/table/[tableId]/import/route.ts | 3 + .../app/api/table/[tableId]/metadata/route.ts | 12 +- apps/sim/app/api/table/[tableId]/route.ts | 2 + .../api/table/[tableId]/rows/[rowId]/route.ts | 5 + .../sim/app/api/table/[tableId]/rows/route.ts | 7 + .../api/table/[tableId]/rows/upsert/route.ts | 2 + .../table-grid/remote-selection-overlay.tsx | 196 ++++++++++++++ .../components/table-grid/table-grid.tsx | 87 +++++- .../tables/[tableId]/hooks/index.ts | 1 + .../[tableId]/hooks/use-table-event-stream.ts | 21 ++ .../tables/[tableId]/hooks/use-table-room.ts | 217 +++++++++++++++ .../[workspaceId]/tables/[tableId]/table.tsx | 14 +- apps/sim/lib/table/events.ts | 60 +++++ apps/sim/lib/table/service.ts | 4 +- packages/platform-authz/src/rooms.ts | 21 +- packages/realtime-protocol/package.json | 4 + packages/realtime-protocol/src/rooms.ts | 6 + .../realtime-protocol/src/table-presence.ts | 103 +++++++ 28 files changed, 1292 insertions(+), 43 deletions(-) create mode 100644 apps/realtime/src/handlers/avatar.ts create mode 100644 apps/realtime/src/handlers/tables.test.ts create mode 100644 apps/realtime/src/handlers/tables.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts create mode 100644 packages/realtime-protocol/src/table-presence.ts diff --git a/apps/realtime/src/handlers/avatar.ts b/apps/realtime/src/handlers/avatar.ts new file mode 100644 index 00000000000..349b166e270 --- /dev/null +++ b/apps/realtime/src/handlers/avatar.ts @@ -0,0 +1,29 @@ +import { db, user } from '@sim/db' +import { createLogger } from '@sim/logger' +import { eq } from 'drizzle-orm' +import type { AuthenticatedSocket } from '@/middleware/auth' + +const logger = createLogger('PresenceAvatar') + +/** + * The avatar URL for a presence entry: the socket's authenticated image when + * present, otherwise a single lookup of the user's stored image. Never throws — + * presence must not fail on an avatar lookup, so a DB error resolves to `null`. + */ +export async function resolveAvatarUrl( + socket: AuthenticatedSocket, + userId: string +): Promise { + if (socket.userImage) return socket.userImage + try { + const [record] = await db + .select({ image: user.image }) + .from(user) + .where(eq(user.id, userId)) + .limit(1) + return record?.image ?? null + } catch (error) { + logger.warn('Failed to load user avatar for presence', { userId, error }) + return null + } +} diff --git a/apps/realtime/src/handlers/index.ts b/apps/realtime/src/handlers/index.ts index 9e034d77db1..485e2853170 100644 --- a/apps/realtime/src/handlers/index.ts +++ b/apps/realtime/src/handlers/index.ts @@ -3,6 +3,7 @@ import { setupWorkspaceFileDocHandlers } from '@/handlers/file-doc' import { setupOperationsHandlers } from '@/handlers/operations' import { setupPresenceHandlers } from '@/handlers/presence' import { setupSubblocksHandlers } from '@/handlers/subblocks' +import { setupTablesHandlers } from '@/handlers/tables' import { setupVariablesHandlers } from '@/handlers/variables' import { setupWorkflowHandlers } from '@/handlers/workflow' import { setupWorkspaceFilesHandlers } from '@/handlers/workspace-files' @@ -17,5 +18,6 @@ export function setupAllHandlers(socket: AuthenticatedSocket, roomManager: IRoom setupPresenceHandlers(socket, roomManager) setupWorkspaceFilesHandlers(socket, roomManager) setupWorkspaceFileDocHandlers(socket, roomManager) + setupTablesHandlers(socket, roomManager) setupConnectionHandlers(socket, roomManager) } diff --git a/apps/realtime/src/handlers/tables.test.ts b/apps/realtime/src/handlers/tables.test.ts new file mode 100644 index 00000000000..2e47099f09a --- /dev/null +++ b/apps/realtime/src/handlers/tables.test.ts @@ -0,0 +1,234 @@ +/** + * @vitest-environment node + */ +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { TABLE_PRESENCE_EVENTS } from '@sim/realtime-protocol/table-presence' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { IRoomManager } from '@/rooms' + +const { mockAuthorizeRoom } = vi.hoisted(() => ({ + mockAuthorizeRoom: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { select: vi.fn() }, + user: { image: 'image' }, +})) + +vi.mock('@sim/platform-authz/rooms', () => ({ + authorizeRoom: mockAuthorizeRoom, +})) + +import { setupTablesHandlers } from '@/handlers/tables' + +const TABLE_ROOM = { type: ROOM_TYPES.TABLE, id: 'table-1' } + +function createSocket(overrides?: Record) { + const handlers: Record Promise | void> = {} + const toEmit = vi.fn() + const socket = { + id: 'socket-1', + userId: 'user-1', + userName: 'Test User', + userImage: 'avatar.png', + on: vi.fn((event: string, handler: (payload: unknown) => Promise | void) => { + handlers[event] = handler + }), + emit: vi.fn(), + join: vi.fn(), + leave: vi.fn(), + to: vi.fn().mockReturnValue({ emit: toEmit }), + ...overrides, + } + return { handlers, socket, toEmit } +} + +function createRoomManager(overrides?: Partial): IRoomManager { + return { + isReady: vi.fn().mockReturnValue(true), + getRoomForSocket: vi.fn().mockResolvedValue(null), + getRoomsForSocket: vi.fn().mockResolvedValue([]), + removeUserFromRoom: vi.fn().mockResolvedValue(false), + removeSocketFromAllRooms: vi.fn().mockResolvedValue([]), + broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), + getRoomUsers: vi.fn().mockResolvedValue([]), + hasRoom: vi.fn().mockResolvedValue(false), + deleteRoom: vi.fn().mockResolvedValue(undefined), + addUserToRoom: vi.fn().mockResolvedValue(undefined), + getUserSession: vi.fn().mockResolvedValue(null), + updateUserActivity: vi.fn().mockResolvedValue(undefined), + updateRoomLastModified: vi.fn().mockResolvedValue(undefined), + emitToRoom: vi.fn(), + getUniqueUserCount: vi.fn().mockResolvedValue(1), + getTotalActiveConnections: vi.fn().mockResolvedValue(0), + shutdown: vi.fn().mockResolvedValue(undefined), + initialize: vi.fn().mockResolvedValue(undefined), + io: { + in: vi.fn().mockReturnValue({ socketsLeave: vi.fn().mockResolvedValue(undefined) }), + }, + ...overrides, + } as unknown as IRoomManager +} + +type SetupArg = Parameters[0] + +describe('setupTablesHandlers', () => { + beforeEach(() => { + vi.clearAllMocks() + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', + }) + }) + + it('rejects join when the socket is not authenticated', async () => { + const { socket, handlers } = createSocket({ userId: undefined, userName: undefined }) + setupTablesHandlers(socket as unknown as SetupArg, createRoomManager()) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + + expect(socket.emit).toHaveBeenCalledWith(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId: 'table-1', + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + }) + + it('rejects join with a retryable error when realtime is unavailable', async () => { + const { socket, handlers } = createSocket() + setupTablesHandlers( + socket as unknown as SetupArg, + createRoomManager({ isReady: vi.fn().mockReturnValue(false) }) + ) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + TABLE_PRESENCE_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) + ) + }) + + it('rejects join when table access is denied', async () => { + mockAuthorizeRoom.mockResolvedValue({ + allowed: false, + status: 403, + workspaceId: 'ws-1', + workspacePermission: null, + }) + const { socket, handlers } = createSocket() + setupTablesHandlers(socket as unknown as SetupArg, createRoomManager()) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + TABLE_PRESENCE_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) + ) + }) + + it('joins the table room and broadcasts presence on success', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1', tabSessionId: 'tab-1' }) + + expect(socket.join).toHaveBeenCalledWith('table:table-1') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + TABLE_ROOM, + 'socket-1', + expect.objectContaining({ userId: 'user-1', role: 'admin' }) + ) + expect(socket.emit).toHaveBeenCalledWith( + TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, + expect.objectContaining({ tableId: 'table-1', socketId: 'socket-1' }) + ) + expect(roomManager.broadcastPresenceUpdate).toHaveBeenCalledWith(TABLE_ROOM) + }) + + it('persists and relays a cell selection to the namespaced room (id + cell only)', async () => { + const { socket, handlers, toEmit } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue(TABLE_ROOM), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const cell = { + anchor: { rowId: 'row-1', columnId: 'col-a' }, + focus: { rowId: 'row-1', columnId: 'col-a' }, + editing: true, + } + await handlers[TABLE_PRESENCE_EVENTS.CELL_SELECTION]({ cell }) + + expect(roomManager.updateUserActivity).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1', { cell }) + // Namespaced room → broadcast targets roomName(room), not the bare id. + expect(socket.to).toHaveBeenCalledWith('table:table-1') + // The delta carries only the socket id + cell — identity comes from the roster. + expect(toEmit).toHaveBeenCalledWith(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { + socketId: 'socket-1', + cell, + }) + }) + + it('drops a malformed cell selection without storing or relaying it', async () => { + const { socket, handlers, toEmit } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue(TABLE_ROOM), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.CELL_SELECTION]({ cell: { anchor: 'x"]' } }) + + expect(roomManager.updateUserActivity).not.toHaveBeenCalled() + expect(toEmit).not.toHaveBeenCalled() + }) + + it('aborts a stale join whose authorize resolves after a newer join', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + // First join's authorize hangs until released; the second resolves immediately. + let releaseA: (value: unknown) => void = () => {} + const pendingA = new Promise((resolve) => { + releaseA = resolve + }) + mockAuthorizeRoom.mockReturnValueOnce(pendingA).mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const joinA = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-A' }) + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) + releaseA({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinA + + // The newer join (B) wins; the stale A join aborts before touching room state. + expect(socket.join).toHaveBeenCalledWith('table:table-B') + expect(socket.join).not.toHaveBeenCalledWith('table:table-A') + expect(roomManager.addUserToRoom).not.toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-A' }, + expect.anything(), + expect.anything() + ) + }) + + it('leaves the table room on leave', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue(TABLE_ROOM), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.LEAVE]({ tableId: 'table-1' }) + + expect(socket.leave).toHaveBeenCalledWith('table:table-1') + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1') + expect(roomManager.broadcastPresenceUpdate).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1') + }) +}) diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts new file mode 100644 index 00000000000..d1fcdd632c2 --- /dev/null +++ b/apps/realtime/src/handlers/tables.ts @@ -0,0 +1,255 @@ +import { createLogger } from '@sim/logger' +import { authorizeRoom } from '@sim/platform-authz/rooms' +import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { + type JoinTablePayload, + TABLE_PRESENCE_EVENTS, + type TableCellRef, + type TableCellSelection, +} from '@sim/realtime-protocol/table-presence' +import { resolveAvatarUrl } from '@/handlers/avatar' +import type { AuthenticatedSocket } from '@/middleware/auth' +import type { IRoomManager, UserPresence } from '@/rooms' +import { filterVisiblePresence, sweepStalePresence } from '@/rooms/presence-visibility' + +const logger = createLogger('TablePresenceHandlers') + +/** Longest accepted row/column id — real ids are UUIDs/short ids; this bounds a hostile payload. */ +const MAX_CELL_ID_LENGTH = 200 + +/** The table presence room ref for a table id. */ +const tableRoom = (tableId: string): RoomRef => ({ type: ROOM_TYPES.TABLE, id: tableId }) + +function isCellRef(value: unknown): value is TableCellRef { + if (typeof value !== 'object' || value === null) return false + const ref = value as { rowId?: unknown; columnId?: unknown } + return ( + typeof ref.rowId === 'string' && + ref.rowId.length <= MAX_CELL_ID_LENGTH && + typeof ref.columnId === 'string' && + ref.columnId.length <= MAX_CELL_ID_LENGTH + ) +} + +/** + * Validate + whitelist an untrusted peer's selection before it is stored and + * rebroadcast (it ultimately flows into a DOM query on every viewer). Returns the + * normalized selection — `null` for a legitimately cleared selection — or `undefined` + * for anything malformed, so the caller drops it. Only the known fields survive, so a + * hostile client can't amplify an oversized object through the room. + */ +function normalizeCellSelection(cell: unknown): TableCellSelection | undefined { + if (cell === null) return null + if (typeof cell !== 'object') return undefined + const candidate = cell as { anchor?: unknown; focus?: unknown; editing?: unknown } + if (!isCellRef(candidate.anchor) || !isCellRef(candidate.focus)) return undefined + return { + anchor: { rowId: candidate.anchor.rowId, columnId: candidate.anchor.columnId }, + focus: { rowId: candidate.focus.rowId, columnId: candidate.focus.columnId }, + ...(candidate.editing === true ? { editing: true } : {}), + } +} + +/** + * Live cell-selection presence for the table grid. Mirrors the workspace-files + * join flow but is table-scoped (room id = tableId) with a bidirectional + * cell-selection channel — the grid analog of the workflow cursor/selection + * relay. Table *data* still flows through the one-way durable event stream + * (`lib/table/events.ts`); this socket carries only ephemeral presence. + * + * Table rooms are namespaced (`table:${id}`), so every broadcast targets + * `roomName(room)`, never the bare `room.id` (which the workflow handler can use + * only because a workflow room's name equals its id). + */ +export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { + // Monotonic per-socket join counter: each JOIN captures its number and, after the async + // authorize, aborts if a newer JOIN has started — a fast table switch A→B can otherwise + // let A's late handler leave B and strand the socket in room A while the client views B. + let joinGeneration = 0 + + socket.on(TABLE_PRESENCE_EVENTS.JOIN, async ({ tableId, tabSessionId }: JoinTablePayload) => { + const joinAttempt = (joinGeneration += 1) + try { + const userId = socket.userId + const userName = socket.userName + + if (!userId || !userName) { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId, + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + return + } + + if (!roomManager.isReady()) { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId, + error: 'Realtime unavailable', + code: 'ROOM_MANAGER_UNAVAILABLE', + retryable: true, + }) + return + } + + // Validate the client-supplied id before it reaches the DB query. + if (typeof tableId !== 'string' || tableId.length === 0) { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId: typeof tableId === 'string' ? tableId : '', + error: 'Invalid table id', + code: 'INVALID_PAYLOAD', + retryable: false, + }) + return + } + + const room = tableRoom(tableId) + + let authorized: Awaited> + try { + authorized = await authorizeRoom({ userId, room, action: 'read' }) + } catch (error) { + logger.warn(`Error authorizing table room for ${userId}:`, error) + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId, + error: 'Failed to verify table access', + code: 'VERIFY_ACCESS_FAILED', + retryable: true, + }) + return + } + + if (!authorized.allowed) { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId, + error: authorized.status === 404 ? 'Table not found' : 'Access denied to table', + code: authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', + retryable: false, + }) + return + } + + // A newer JOIN started on this socket during authorize (or the socket dropped): + // abort so a stale join can't leave the room the client has since moved to. + if (joinGeneration !== joinAttempt || socket.disconnected) return + + // Leave a previously-joined table room if switching tables. + const currentRoom = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) + if (currentRoom && currentRoom.id !== tableId) { + socket.leave(roomName(currentRoom)) + await roomManager.removeUserFromRoom(currentRoom, socket.id) + await roomManager.broadcastPresenceUpdate(currentRoom) + } + + // Clean up the same user's stale socket from the same tab (a reconnect that + // raced the old socket's disconnect), so presence shows one entry. + if (tabSessionId) { + const existingUsers = await roomManager.getRoomUsers(room) + for (const existing of existingUsers) { + if ( + existing.socketId !== socket.id && + existing.userId === userId && + existing.tabSessionId === tabSessionId + ) { + await roomManager.removeUserFromRoom(room, existing.socketId) + await roomManager.io.in(existing.socketId).socketsLeave(roomName(room)) + } + } + } + + // Reclaim presence orphaned by an ungraceful disconnect (no `disconnecting` + // event fires on a pod crash; the room hashes have no TTL). + await sweepStalePresence(roomManager, room) + + socket.join(roomName(room)) + + const presence: UserPresence = { + userId, + room, + userName, + socketId: socket.id, + tabSessionId, + joinedAt: Date.now(), + lastActivity: Date.now(), + role: authorized.workspacePermission ?? 'read', + avatarUrl: await resolveAvatarUrl(socket, userId), + } + + await roomManager.addUserToRoom(room, socket.id, presence) + + // Filter the join ack to live members so a new joiner never briefly sees a + // ghost from an entry the sweep hasn't reclaimed yet. + const presenceUsers = await filterVisiblePresence( + roomManager.io, + room, + await roomManager.getRoomUsers(room) + ) + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, { + tableId, + socketId: socket.id, + presenceUsers, + }) + + await roomManager.broadcastPresenceUpdate(room) + + logger.info(`User ${userId} (${userName}) joined table room ${tableId}`) + } catch (error) { + logger.error('Error joining table room:', error) + // Roll back any partial join so a failed attempt can't leave the socket in the + // Socket.IO room or a stale presence entry behind, before signalling a retry. + try { + const room = tableRoom(tableId) + socket.leave(roomName(room)) + await roomManager.removeUserFromRoom(room, socket.id) + } catch {} + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId, + error: 'Failed to join table', + code: 'JOIN_FAILED', + retryable: true, + }) + } + }) + + socket.on(TABLE_PRESENCE_EVENTS.LEAVE, async (payload?: { tableId?: string }) => { + try { + if (!roomManager.isReady()) return + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) + if (!room) return + // Scope the leave to a specific table when the client provides one: a deferred + // leave from a prior view must not evict the socket from a room it has since + // switched into (table A→B leaves A's leave targeting B). + if (payload?.tableId && payload.tableId !== room.id) return + socket.leave(roomName(room)) + await roomManager.removeUserFromRoom(room, socket.id) + await roomManager.broadcastPresenceUpdate(room, socket.id) + } catch (error) { + logger.error('Error leaving table room:', error) + } + }) + + socket.on(TABLE_PRESENCE_EVENTS.CELL_SELECTION, async ({ cell }: { cell: unknown }) => { + try { + // Drop a malformed/oversized selection from an untrusted peer before it is stored + // or rebroadcast (`undefined` = invalid; `null` = a legitimately cleared selection). + const selection = normalizeCellSelection(cell) + if (selection === undefined) return + + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) + if (!room) return + + // Persist so a later joiner sees this viewer's current selection in the join ack. + await roomManager.updateUserActivity(room, socket.id, { cell: selection }) + + // Relay to peers (namespaced room → roomName, not room.id). Peers already know this + // socket's identity from the presence roster, so the delta carries only id + cell. + socket.to(roomName(room)).emit(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { + socketId: socket.id, + cell: selection, + }) + } catch (error) { + logger.error(`Error handling table cell selection for socket ${socket.id}:`, error) + } + }) +} diff --git a/apps/realtime/src/handlers/workspace-files.ts b/apps/realtime/src/handlers/workspace-files.ts index 964857e0633..0c7d3ef8c9b 100644 --- a/apps/realtime/src/handlers/workspace-files.ts +++ b/apps/realtime/src/handlers/workspace-files.ts @@ -1,8 +1,7 @@ -import { db, user } from '@sim/db' import { createLogger } from '@sim/logger' import { authorizeRoom } from '@sim/platform-authz/rooms' import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' -import { eq } from 'drizzle-orm' +import { resolveAvatarUrl } from '@/handlers/avatar' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager, UserPresence } from '@/rooms' import { filterVisiblePresence, sweepStalePresence } from '@/rooms/presence-visibility' @@ -21,24 +20,6 @@ interface JoinPayload { tabSessionId?: string } -async function resolveAvatarUrl( - socket: AuthenticatedSocket, - userId: string -): Promise { - if (socket.userImage) return socket.userImage - try { - const [record] = await db - .select({ image: user.image }) - .from(user) - .where(eq(user.id, userId)) - .limit(1) - return record?.image ?? null - } catch (error) { - logger.warn('Failed to load user avatar for files presence', { userId, error }) - return null - } -} - /** * Presence handlers for the workspace file browser. Mirrors the workflow join * flow but is workspace-scoped (room id = workspaceId) and read-only presence: diff --git a/apps/realtime/src/rooms/memory-manager.ts b/apps/realtime/src/rooms/memory-manager.ts index 90001f53b53..f7ebed862ac 100644 --- a/apps/realtime/src/rooms/memory-manager.ts +++ b/apps/realtime/src/rooms/memory-manager.ts @@ -156,13 +156,14 @@ export class MemoryRoomManager implements IRoomManager { async updateUserActivity( room: RoomRef, socketId: string, - updates: Partial> + updates: Partial> ): Promise { const presence = this.rooms.get(roomKey(room))?.users.get(socketId) if (!presence) return if (updates.cursor !== undefined) presence.cursor = updates.cursor if (updates.selection !== undefined) presence.selection = updates.selection + if (updates.cell !== undefined) presence.cell = updates.cell presence.lastActivity = updates.lastActivity ?? Date.now() } diff --git a/apps/realtime/src/rooms/redis-manager.ts b/apps/realtime/src/rooms/redis-manager.ts index 945d03d539c..cffb9a6d661 100644 --- a/apps/realtime/src/rooms/redis-manager.ts +++ b/apps/realtime/src/rooms/redis-manager.ts @@ -77,7 +77,7 @@ return removed * socket key TTLs to keep a long-lived session alive. * * KEYS: [roomUsers, socketRooms, socketSession] - * ARGV: [socketId, cursorJson, selectionJson, lastActivity, roomsTtl, sessionTtl] + * ARGV: [socketId, cursorJson, selectionJson, lastActivity, roomsTtl, sessionTtl, cellJson] * Returns 1 if the socket had presence in the room, else 0. */ const UPDATE_ACTIVITY_SCRIPT = ` @@ -90,6 +90,7 @@ local selectionJson = ARGV[3] local lastActivity = ARGV[4] local roomsTtl = tonumber(ARGV[5]) local sessionTtl = tonumber(ARGV[6]) +local cellJson = ARGV[7] local existingJson = redis.call('HGET', roomUsersKey, socketId) if not existingJson then @@ -103,6 +104,9 @@ end if selectionJson ~= '' then existing.selection = cjson.decode(selectionJson) end +if cellJson ~= '' then + existing.cell = cjson.decode(cellJson) +end existing.lastActivity = tonumber(lastActivity) redis.call('HSET', roomUsersKey, socketId, cjson.encode(existing)) @@ -325,7 +329,7 @@ export class RedisRoomManager implements IRoomManager { async updateUserActivity( room: RoomRef, socketId: string, - updates: Partial>, + updates: Partial>, retried = false ): Promise { if (!this.updateActivityScriptSha) { @@ -343,6 +347,9 @@ export class RedisRoomManager implements IRoomManager { (updates.lastActivity ?? Date.now()).toString(), SOCKET_ROOMS_TTL.toString(), SESSION_TTL.toString(), + // Trailing arg (ARGV[7]) so existing indices stay stable. `null` (cleared + // selection) serializes to 'null'; `undefined` (no cell change) to '' (skip). + updates.cell !== undefined ? JSON.stringify(updates.cell) : '', ], }) } catch (error) { diff --git a/apps/realtime/src/rooms/types.ts b/apps/realtime/src/rooms/types.ts index c9c8ee18704..f1a67d66abb 100644 --- a/apps/realtime/src/rooms/types.ts +++ b/apps/realtime/src/rooms/types.ts @@ -1,4 +1,5 @@ import type { RoomRef, RoomType } from '@sim/realtime-protocol/rooms' +import type { TableCellSelection } from '@sim/realtime-protocol/table-presence' import type { Server } from 'socket.io' /** @@ -18,6 +19,8 @@ export interface UserPresence { role: string cursor?: { x: number; y: number } selection?: { type: 'block' | 'edge' | 'none'; id?: string } + /** The viewer's current table cell selection, for table presence rooms. */ + cell?: TableCellSelection avatarUrl?: string | null /** * The subfolder the user is viewing, recorded at join for room types that track @@ -108,11 +111,11 @@ export interface IRoomManager { */ deleteRoom(room: RoomRef): Promise - /** Update a socket's activity (cursor, selection, lastActivity) within a room. */ + /** Update a socket's activity (cursor, selection, cell, lastActivity) within a room. */ updateUserActivity( room: RoomRef, socketId: string, - updates: Partial> + updates: Partial> ): Promise /** Bump a room's lastModified timestamp. */ diff --git a/apps/sim/app/api/table/[tableId]/columns/route.ts b/apps/sim/app/api/table/[tableId]/columns/route.ts index 7eecb5ee466..5b60573bd72 100644 --- a/apps/sim/app/api/table/[tableId]/columns/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/route.ts @@ -17,6 +17,7 @@ import { updateColumnConstraints, updateColumnType, } from '@/lib/table' +import { signalTableSchemaChanged } from '@/lib/table/events' import { accessError, checkAccess, normalizeColumn, rootErrorMessage } from '@/app/api/table/utils' const logger = createLogger('TableColumnsAPI') @@ -51,6 +52,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Colum } const updatedTable = await addTableColumn(tableId, validated.column, requestId) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, @@ -138,6 +140,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu if (!updatedTable) { return NextResponse.json({ error: 'No updates specified' }, { status: 400 }) } + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, @@ -202,6 +205,7 @@ export const DELETE = withRouteHandler( { tableId, columnName: validated.columnName }, requestId ) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/groups/route.ts b/apps/sim/app/api/table/[tableId]/groups/route.ts index 84aecb13c0c..23a2d5f0811 100644 --- a/apps/sim/app/api/table/[tableId]/groups/route.ts +++ b/apps/sim/app/api/table/[tableId]/groups/route.ts @@ -10,6 +10,7 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { signalTableSchemaChanged } from '@/lib/table/events' import { addWorkflowGroup, deleteWorkflowGroup, @@ -99,6 +100,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro }, requestId ) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, data: { @@ -161,6 +163,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, { params }: R }, requestId ) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, data: { @@ -194,6 +197,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, { params }: { tableId, groupId: validated.groupId }, requestId ) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, data: { diff --git a/apps/sim/app/api/table/[tableId]/import/route.ts b/apps/sim/app/api/table/[tableId]/import/route.ts index 1d6482390fd..eab163d583b 100644 --- a/apps/sim/app/api/table/[tableId]/import/route.ts +++ b/apps/sim/app/api/table/[tableId]/import/route.ts @@ -37,6 +37,7 @@ import { wouldExceedRowLimit, } from '@/lib/table' import { sniffCsvDelimiterFromStream } from '@/lib/table/csv-delimiter-stream' +import { signalTableSchemaChanged } from '@/lib/table/events' import { importAppendRows, importReplaceRows } from '@/lib/table/import-data' import { getUserSettings } from '@/lib/users/queries' import { @@ -322,6 +323,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro mappedColumns: validation.mappedHeaders.length, skippedHeaders: validation.skippedHeaders.length, }) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, @@ -378,6 +380,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro createdColumns: additions.length, mappedColumns: validation.mappedHeaders.length, }) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/metadata/route.ts b/apps/sim/app/api/table/[tableId]/metadata/route.ts index f85df0af4bd..6883b56038c 100644 --- a/apps/sim/app/api/table/[tableId]/metadata/route.ts +++ b/apps/sim/app/api/table/[tableId]/metadata/route.ts @@ -7,6 +7,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { TableMetadata } from '@/lib/table' import { updateTableMetadata } from '@/lib/table' +import { signalTableMetadataChanged, signalTableSchemaChanged } from '@/lib/table/events' import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableMetadataAPI') @@ -43,11 +44,20 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: TableR return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } - const updated = await updateTableMetadata( + const { metadata: updated, schemaChanged } = await updateTableMetadata( tableId, validated.metadata, table.metadata as TableMetadata | null ) + // Signal collaborators to re-apply the new column layout (width/pin/order) live; the + // grid reconciles against its in-progress resize/drag so a peer's change never clobbers + // the local gesture. A reorder that scrubs a workflow-group's dependencies also mutated + // the schema — escalate to the schema signal so peers refresh run-state/enrichment too. + if (schemaChanged) { + signalTableSchemaChanged(tableId) + } else { + signalTableMetadataChanged(tableId) + } return NextResponse.json({ success: true, data: { metadata: updated } }) } catch (error) { diff --git a/apps/sim/app/api/table/[tableId]/route.ts b/apps/sim/app/api/table/[tableId]/route.ts index 5d0069fa570..2dd9a759828 100644 --- a/apps/sim/app/api/table/[tableId]/route.ts +++ b/apps/sim/app/api/table/[tableId]/route.ts @@ -9,6 +9,7 @@ import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' import { deleteTable, renameTable, TableConflictError, type TableSchema } from '@/lib/table' import { getWorkspaceTableLimits } from '@/lib/table/billing' +import { signalTableSchemaChanged } from '@/lib/table/events' import { accessError, checkAccess, normalizeColumn } from '@/app/api/table/utils' const logger = createLogger('TableDetailAPI') @@ -126,6 +127,7 @@ export const PATCH = withRouteHandler( } const updated = await renameTable(tableId, validated.name, requestId, authResult.userId) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts index b1865223f83..4fc0da1871d 100644 --- a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts @@ -15,6 +15,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { RowData, TableSchema } from '@/lib/table' import { deleteRow, updateRow } from '@/lib/table' +import { signalTableRowsChanged } from '@/lib/table/events' import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, @@ -146,6 +147,9 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: RowR // Only `null` when a `cancellationGuard` is supplied and the SQL guard // rejects the write — this route doesn't pass one, so reaching null is a bug. if (!updatedRow) throw new Error('updateRow returned null without a cancellationGuard') + // An edit that also triggers a dispatch already emits dispatch/cell events; the + // debounced rows refetch on the peer coalesces the two. + signalTableRowsChanged(tableId) // Auto-dispatch for user edits is handled inside `updateRow` (mode: 'new'). // Firing a second mode: 'incomplete' dispatch here would race with the // `mode: 'new'` one AND bulk-clear sibling-group outputs (the incomplete @@ -212,6 +216,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Row } await deleteRow(tableId, rowId, validated.workspaceId, requestId) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index e200220ea11..49ba83bd322 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -25,6 +25,7 @@ import { validateRowData, validateRowSize, } from '@/lib/table' +import { signalTableRowsChanged } from '@/lib/table/events' import { queryRows } from '@/lib/table/rows/service' import { TableQueryValidationError } from '@/lib/table/sql' import { rowWireTranslators } from '@/app/api/table/row-wire' @@ -79,6 +80,7 @@ async function handleBatchInsert( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -164,6 +166,7 @@ export const POST = withRouteHandler( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -371,6 +374,7 @@ export const PUT = withRouteHandler( { status: 200 } ) } + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -436,6 +440,7 @@ export const DELETE = withRouteHandler( { tableId, rowIds: validated.rowIds, workspaceId: validated.workspaceId }, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -461,6 +466,7 @@ export const DELETE = withRouteHandler( }, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -534,6 +540,7 @@ export const PATCH = withRouteHandler( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts b/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts index bc97623ef9a..d6656c11ab4 100644 --- a/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts @@ -8,6 +8,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { RowData, TableSchema } from '@/lib/table' import { upsertRow } from '@/lib/table' +import { signalTableRowsChanged } from '@/lib/table/events' import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' @@ -54,6 +55,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Upser table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx new file mode 100644 index 00000000000..74ed697f76a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx @@ -0,0 +1,196 @@ +'use client' + +import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' +import { getUserColor, withAlpha } from '@/lib/workspaces/colors' +import type { RemoteTableSelection } from '@/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room' + +/** A measured remote selection, positioned in the grid content wrapper's space. */ +interface SelectionBox { + socketId: string + userName: string + color: string + editing: boolean + top: number + left: number + width: number + height: number +} + +interface RemoteSelectionOverlayProps { + remoteSelections: RemoteTableSelection[] + /** Column id → its rendered column index (matches the cells' `data-col`). */ + columnIndexById: Map + /** The grid's scroll container (`data-table-scroll`), queried for cell rects. */ + scrollElement: HTMLElement | null +} + +/** The cell `` for a (rowId, columnIndex), or undefined when virtualized off-window. */ +function cellRect( + scrollEl: HTMLElement, + rowId: string, + columnIndex: number | undefined +): DOMRect | undefined { + if (columnIndex === undefined) return undefined + // `rowId` is a remote peer's value — escape it so a hostile id can't break the + // selector and throw (`columnIndex` is a local numeric index, already safe). + const cell = scrollEl.querySelector( + `[data-row-id="${CSS.escape(rowId)}"][data-col="${columnIndex}"]` + ) + return cell?.getBoundingClientRect() +} + +/** + * Renders remote collaborators' cell selections over the table grid — a colored + * border per user (Google-Sheets style), a darker fill while they are editing, and + * their name on hover. Mounted inside the grid's `relative` content wrapper, so + * content-space coordinates scroll with the grid automatically. + * + * Positions are measured from the live cell rects (the same `[data-row-id][data-col]` + * idiom the reveal effect uses), keyed by stable ids so each client renders under its + * own sort/scroll. A selection whose rows are virtualized off-window is simply not + * drawn. The layer is `pointer-events-none` so it never intercepts cell clicks; the + * name-on-hover is driven by hit-testing pointer moves against the measured boxes. + */ +export function RemoteSelectionOverlay({ + remoteSelections, + columnIndexById, + scrollElement, +}: RemoteSelectionOverlayProps) { + const rootRef = useRef(null) + const [boxes, setBoxes] = useState([]) + const [hoveredSocketId, setHoveredSocketId] = useState(null) + + // Latest data read by the subscribe-once effect + the pointer hit-test, so neither + // re-subscribes on every incoming selection delta. + const boxesRef = useRef([]) + boxesRef.current = boxes + const remoteSelectionsRef = useRef(remoteSelections) + remoteSelectionsRef.current = remoteSelections + const columnIndexByIdRef = useRef(columnIndexById) + columnIndexByIdRef.current = columnIndexById + // Cached content-wrapper origin, refreshed on each measure (scroll/resize/data change), + // so the pointer hit-test never forces a layout read per mouse move. + const originRef = useRef({ top: 0, left: 0 }) + + const measure = useCallback(() => { + const scrollEl = scrollElement + const root = rootRef.current + if (!scrollEl || !root) return + const origin = root.getBoundingClientRect() + originRef.current = { top: origin.top, left: origin.left } + const next: SelectionBox[] = [] + for (const selection of remoteSelectionsRef.current) { + const { anchor, focus, editing } = selection.cell + const rects = [ + cellRect(scrollEl, anchor.rowId, columnIndexByIdRef.current.get(anchor.columnId)), + cellRect(scrollEl, focus.rowId, columnIndexByIdRef.current.get(focus.columnId)), + ].filter((rect): rect is DOMRect => rect !== undefined) + if (rects.length === 0) continue + + const top = Math.min(...rects.map((r) => r.top)) - origin.top + const left = Math.min(...rects.map((r) => r.left)) - origin.left + const bottom = Math.max(...rects.map((r) => r.bottom)) - origin.top + const right = Math.max(...rects.map((r) => r.right)) - origin.left + next.push({ + socketId: selection.socketId, + userName: selection.userName, + color: getUserColor(selection.userId), + editing: editing === true, + top, + left, + width: right - left, + height: bottom - top, + }) + } + setBoxes(next) + }, [scrollElement]) + + // Subscribe once per scroll element: re-measure on scroll/resize, and hit-test pointer + // moves against the cached boxes/origin — no layout read per move, stays pointer-events-none. + useEffect(() => { + const scrollEl = scrollElement + if (!scrollEl) return + + let raf = 0 + const schedule = () => { + if (!raf) + raf = requestAnimationFrame(() => { + raf = 0 + measure() + }) + } + const handleMove = (event: PointerEvent) => { + const { top, left } = originRef.current + const x = event.clientX - left + const y = event.clientY - top + const hit = boxesRef.current.find( + (b) => x >= b.left && x <= b.left + b.width && y >= b.top && y <= b.top + b.height + ) + setHoveredSocketId((prev) => + prev === (hit?.socketId ?? null) ? prev : (hit?.socketId ?? null) + ) + } + const handleLeave = () => setHoveredSocketId(null) + + // No measure() here — the re-measure layout effect below runs on mount and whenever + // `measure` changes (it depends on `scrollElement`), so it already covers the initial + // and scroll-element-changed measures without a redundant pass. + scrollEl.addEventListener('scroll', schedule, { passive: true }) + scrollEl.addEventListener('pointermove', handleMove, { passive: true }) + scrollEl.addEventListener('pointerleave', handleLeave) + const resizeObserver = new ResizeObserver(schedule) + resizeObserver.observe(scrollEl) + // Also observe the content layer (this overlay fills it): a column resize or a + // row-count change grows/shrinks the content without resizing the scroll container, + // yet moves cell rects — so measure off the content, not just the viewport. + if (rootRef.current) resizeObserver.observe(rootRef.current) + // Re-measure when rows are added/removed/reordered/virtualized (a live refetch moves + // cells without a scroll/resize) — childList only, so a cell-content edit doesn't fire. + const tbody = scrollEl.querySelector('tbody') + const rowObserver = new MutationObserver(schedule) + if (tbody) rowObserver.observe(tbody, { childList: true }) + + return () => { + scrollEl.removeEventListener('scroll', schedule) + scrollEl.removeEventListener('pointermove', handleMove) + scrollEl.removeEventListener('pointerleave', handleLeave) + resizeObserver.disconnect() + rowObserver.disconnect() + if (raf) cancelAnimationFrame(raf) + } + }, [scrollElement, measure]) + + // Re-measure when the selections or column layout change (listeners stay subscribed). + // Layout effect so positions update before paint — no one-frame lag as a peer moves. + useLayoutEffect(() => { + measure() + }, [remoteSelections, columnIndexById, measure]) + + return ( +
+ {boxes.map((box) => ( +
+ {hoveredSocketId === box.socketId && ( + + {box.userName} + + )} +
+ ))} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index 330df1c2d2d..8ae1abd1df7 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -5,6 +5,7 @@ import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } fr import { cn, toast, useToast } from '@sim/emcn' import { Loader, TableX } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' +import type { TableCellSelection } from '@sim/realtime-protocol/table-presence' import { useVirtualizer } from '@tanstack/react-virtual' import { useParams } from 'next/navigation' import { usePostHog } from 'posthog-js/react' @@ -14,6 +15,7 @@ import type { ColumnDefinition, Filter, TableRow as TableRowType, WorkflowGroup import { getColumnId } from '@/lib/table/column-keys' import { TABLE_LIMITS } from '@/lib/table/constants' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' +import type { RemoteTableSelection } from '@/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room' import { useTimezone } from '@/hooks/queries/general-settings' import { useAddTableColumn, @@ -47,6 +49,7 @@ import { ExpandedCellPopover } from './cells' import { ADD_COL_WIDTH, COL_WIDTH, SELECTION_TINT_BG } from './constants' import { DataRow } from './data-row' import { ColumnHeaderMenu, WorkflowGroupMetaCell } from './headers' +import { RemoteSelectionOverlay } from './remote-selection-overlay' import { TableFind } from './table-find' import { AddRowButton, SelectAllCheckbox, TableColGroup } from './table-primitives' import type { DisplayColumn } from './types' @@ -143,6 +146,10 @@ interface TableGridProps { workspaceId?: string tableId?: string embedded?: boolean + /** Remote collaborators' cell selections, rendered as presence overlays. */ + remoteSelections: RemoteTableSelection[] + /** Broadcast the local viewer's cell selection to the table presence room. */ + emitCellSelection: (cell: TableCellSelection) => void /** * Pixel width to reserve on the right of the table's scroll content for the * currently-open slideout panel (column config, workflow config, or log @@ -288,6 +295,8 @@ export function TableGrid({ workspaceId: propWorkspaceId, tableId: propTableId, embedded, + remoteSelections, + emitCellSelection, sidebarReservedPx, onOpenColumnConfig, onOpenWorkflowConfig, @@ -348,6 +357,8 @@ export function TableGrid({ const columnWidthsRef = useRef(columnWidths) columnWidthsRef.current = columnWidths const [resizingColumn, setResizingColumn] = useState(null) + const resizingColumnRef = useRef(resizingColumn) + resizingColumnRef.current = resizingColumn const [columnOrder, setColumnOrder] = useState(null) const columnOrderRef = useRef(columnOrder) columnOrderRef.current = columnOrder @@ -645,6 +656,13 @@ export function TableGrid({ return expandToDisplayColumns(ordered, tableWorkflowGroups) }, [columns, columnOrder, tableWorkflowGroups]) + /** Column id → its rendered index (matches the cells' `data-col`), for placing overlays. */ + const columnIndexById = useMemo(() => { + const map = new Map() + displayColumns.forEach((col, index) => map.set(col.key, index)) + return map + }, [displayColumns]) + const workflowGroupById = useMemo( () => new Map(tableWorkflowGroups.map((g) => [g.id, g])), [tableWorkflowGroups] @@ -819,6 +837,30 @@ export function TableGrid({ ? (rowsRef.current[selectionFocus.rowIndex]?.id ?? null) : null + // Broadcast the local viewer's cell selection to the presence room. Resolves the + // index-based selection to stable (rowId, columnId), re-running on `rows`/`displayColumns` + // too so a peer's row insert/delete/reorder re-broadcasts the shifted id under the same + // index (the emitter dedups an unchanged result). `editing` marks the active cell so + // peers darken it (the "someone is typing here" signal). + useEffect(() => { + const resolve = (coord: CellCoord | null) => { + if (!coord) return null + const rowId = rows[coord.rowIndex]?.id + const columnId = displayColumns[coord.colIndex]?.key + return rowId && columnId ? { rowId, columnId } : null + } + const anchor = resolve(selectionAnchor) + if (!anchor) { + emitCellSelection(null) + return + } + // A single-cell click leaves `selectionFocus` null; the grid treats that as a + // one-cell selection at the anchor (`focus ?? anchor`). Mirror that — otherwise the + // most common selection would never broadcast and would clear the prior outline. + const focus = resolve(selectionFocus) ?? anchor + emitCellSelection({ anchor, focus, editing: editingCell !== null }) + }, [selectionAnchor, selectionFocus, editingCell, rows, displayColumns, emitCellSelection]) + const { data: findData, isFetching: isFindFetching } = useFindTableRows({ workspaceId, tableId, @@ -1740,20 +1782,32 @@ export function TableGrid({ } return } - // After first load: only re-seed `columnOrder` when the *set of columns* - // changes (e.g. a workflow group adds/removes outputs server-side). Pure - // reorders are left alone so an in-flight optimistic drag isn't clobbered - // by a refetch returning the pre-drag order. + // After first load a collaborator (or our own committed edit) reshaped the layout. + // Re-apply it live, but never clobber the gesture the local user is mid-way through — + // their in-progress value leads the server's. Each field is guarded by reference: + // React Query structural sharing keeps an unchanged sub-object referentially stable, + // so an unrelated change (e.g. a peer's pin) doesn't re-apply widths/order. + // Width: keep the column being actively resized on its live local value. + const serverWidths = tableData.metadata.columnWidths + if (serverWidths && serverWidths !== columnWidthsRef.current) { + const resizing = resizingColumnRef.current + const localWidth = resizing ? columnWidthsRef.current[resizing] : undefined + setColumnWidths( + resizing && localWidth !== undefined + ? { ...serverWidths, [resizing]: localWidth } + : serverWidths + ) + } + // Pins toggle instantly (no in-progress gesture) — apply on change. + const serverPins = tableData.metadata.pinnedColumns + if (serverPins && serverPins !== pinnedColumnsRef.current) { + setPinnedColumns(serverPins) + } + // Order: apply unless a local column drag is in flight (an optimistic reorder would + // otherwise be reverted to the pre-drag order the refetch returns). const serverOrder = tableData.metadata.columnOrder - if (serverOrder) { - const localOrder = columnOrderRef.current - const serverSet = new Set(serverOrder) - const localSet = new Set(localOrder ?? []) - const setChanged = - !localOrder || serverSet.size !== localSet.size || serverOrder.some((n) => !localSet.has(n)) - if (setChanged) { - setColumnOrder(serverOrder) - } + if (serverOrder && serverOrder !== columnOrderRef.current && !dragColumnNameRef.current) { + setColumnOrder(serverOrder) } }, [tableData?.metadata]) @@ -3885,6 +3939,13 @@ export function TableGrid({ })()} + {remoteSelections.length > 0 && ( + + )} {resizingColumn && (
+} + +interface UseTableRoomResult { + /** Collaborators viewing this table, excluding the current socket (for avatars). */ + otherUsers: PresenceAvatarUser[] + /** Remote viewers that currently have a cell selected (for overlays). */ + remoteSelections: RemoteTableSelection[] + /** Broadcast the local viewer's current cell selection (`null` clears it). Throttled. */ + emitCellSelection: (cell: TableCellSelection) => void +} + +/** + * Joins the table presence room for live collaborator avatars + cell-selection + * highlights. Presence rides the shared socket (`useSocket`); table data is + * unchanged (it flows through the one-way durable event stream). The full roster + * arrives via the presence broadcast (join/leave); individual selection moves + * arrive as lower-latency {@link TABLE_PRESENCE_EVENTS.CELL_SELECTION} deltas that + * patch the matching roster entry. + */ +export function useTableRoom(tableId: string): UseTableRoomResult { + const { socket, currentSocketId } = useSocket() + + const [presenceUsers, setPresenceUsers] = useState([]) + + const tabSessionIdRef = useRef('') + if (!tabSessionIdRef.current) tabSessionIdRef.current = generateShortId() + + // The local viewer's current selection, re-broadcast on (re)join. Emits only fire on a + // selection change, and the server drops a CELL_SELECTION for a socket not yet in the + // room — so a selection made before the join completes (or held across a reconnect) + // would otherwise never reach peers until it next changes. + const currentCellRef = useRef(null) + + useEffect(() => { + if (!socket || !tableId) return + + let retries = 0 + let retryTimer: ReturnType | null = null + + const join = () => { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN, { tableId, tabSessionId: tabSessionIdRef.current }) + } + + const handleJoinSuccess = (data: JoinTableSuccess) => { + if (data.tableId !== tableId) return + retries = 0 + if (retryTimer) { + clearTimeout(retryTimer) + retryTimer = null + } + setPresenceUsers(data.presenceUsers ?? []) + // Re-send our current selection now that the room is joined (earlier emits were + // dropped server-side), so peers see it without waiting for the next change. + if (currentCellRef.current) { + socket.emit(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { cell: currentCellRef.current }) + } + } + const handleJoinError = (data: JoinTableError) => { + if (data.tableId !== tableId) return + logger.warn('Failed to join table room', { code: data.code, error: data.error }) + if (data.retryable && retries < MAX_JOIN_RETRIES) { + retries += 1 + retryTimer = setTimeout(join, JOIN_RETRY_BASE_MS * retries) + } + } + const handlePresence = (users: TablePresenceUser[]) => { + // Take membership from the roster snapshot but keep the `cell` we already hold for + // a known socket. Trade-off: a snapshot can lag the lower-latency CELL_SELECTION + // deltas, so a blind replace could revert a fresher selection (the common case). + // The cost is that a *dropped* delta is no longer healed by the next snapshot, only + // by the peer's next delta — fine, since deltas flow continuously during selection + // and a cleared selection also sends `null` via delta. + setPresenceUsers((prev) => { + const cellBySocket = new Map(prev.map((user) => [user.socketId, user.cell])) + return (users ?? []).map((user) => + cellBySocket.has(user.socketId) + ? { ...user, cell: cellBySocket.get(user.socketId) } + : user + ) + }) + } + const handleCellSelection = (data: TableCellSelectionBroadcast) => { + // Patch the matching roster entry's selection. The peer is always already in + // the roster: the server broadcasts their join (→ presence-update) before they + // can select, and Socket.IO preserves that order — so a delta for an unknown + // socket only means a dropped broadcast, which the next presence-update heals. + setPresenceUsers((prev) => + prev.map((user) => (user.socketId === data.socketId ? { ...user, cell: data.cell } : user)) + ) + } + + // Join now if the socket is already connected; `connect` covers (re)connects. + if (socket.connected) join() + socket.on('connect', join) + socket.on(TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, handleJoinSuccess) + socket.on(TABLE_PRESENCE_EVENTS.JOIN_ERROR, handleJoinError) + socket.on(TABLE_PRESENCE_UPDATE_EVENT, handlePresence) + socket.on(TABLE_PRESENCE_EVENTS.CELL_SELECTION, handleCellSelection) + + return () => { + if (retryTimer) clearTimeout(retryTimer) + socket.off('connect', join) + socket.off(TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, handleJoinSuccess) + socket.off(TABLE_PRESENCE_EVENTS.JOIN_ERROR, handleJoinError) + socket.off(TABLE_PRESENCE_UPDATE_EVENT, handlePresence) + socket.off(TABLE_PRESENCE_EVENTS.CELL_SELECTION, handleCellSelection) + setPresenceUsers([]) + // Leave scoped to THIS table so a table A→B switch (B joins first, auto-leaving + // A) can't have A's deferred leave evict the fresh B membership. + socket.emit(TABLE_PRESENCE_EVENTS.LEAVE, { tableId }) + } + }, [socket, tableId]) + + const socketRef = useRef(socket) + socketRef.current = socket + const lastEmitRef = useRef(0) + const trailingTimerRef = useRef | null>(null) + const pendingCellRef = useRef(null) + /** Key of the last selection sent, to skip re-emitting an unchanged selection. */ + const lastSentKeyRef = useRef(null) + + // Reset the throttle when the table changes (or on unmount): a pending selection for + // the table we're leaving must not flush into the next table's room after a switch. + useEffect( + () => () => { + if (trailingTimerRef.current) { + clearTimeout(trailingTimerRef.current) + trailingTimerRef.current = null + } + pendingCellRef.current = null + currentCellRef.current = null + lastSentKeyRef.current = null + lastEmitRef.current = 0 + }, + [tableId] + ) + + const emitCellSelection = useCallback((cell: TableCellSelection) => { + // Skip re-emitting an unchanged selection: the caller re-resolves on every data + // refetch (so a peer's row insert re-broadcasts the shifted rowId), but most refetches + // don't move the selection — dedup those, and the no-selection state on table open. + const key = cell === null ? 'null' : JSON.stringify(cell) + if (key === lastSentKeyRef.current) return + lastSentKeyRef.current = key + currentCellRef.current = cell + pendingCellRef.current = cell + const flush = () => { + lastEmitRef.current = Date.now() + trailingTimerRef.current = null + socketRef.current?.emit(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { + cell: pendingCellRef.current, + }) + } + const elapsed = Date.now() - lastEmitRef.current + if (elapsed >= SELECTION_EMIT_THROTTLE_MS) { + flush() + } else if (!trailingTimerRef.current) { + trailingTimerRef.current = setTimeout(flush, SELECTION_EMIT_THROTTLE_MS - elapsed) + } + }, []) + + const otherUsers = useMemo( + () => presenceUsers.filter((user) => user.socketId !== currentSocketId), + [presenceUsers, currentSocketId] + ) + const remoteSelections = useMemo( + () => + otherUsers + .filter( + (user): user is TablePresenceUser & { cell: NonNullable } => + user.cell != null + ) + .map((user) => ({ + socketId: user.socketId, + userId: user.userId, + userName: user.userName, + cell: user.cell, + })), + [otherUsers] + ) + + return { otherUsers, remoteSelections, emitCellSelection } +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index 0f7485636bb..08c18c0ef73 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -24,6 +24,7 @@ import { Resource, type SortConfig, } from '@/app/workspace/[workspaceId]/components' +import { PresenceAvatars } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' import { LogDetails } from '@/app/workspace/[workspaceId]/logs/components' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { ImportCsvDialog } from '@/app/workspace/[workspaceId]/tables/components/import-csv-dialog' @@ -59,7 +60,7 @@ import { } from './components' import { COLUMN_SIDEBAR_WIDTH } from './components/table-grid/constants' import { COLUMN_TYPE_ICONS } from './components/table-grid/headers' -import { useTable, useTableEventStream } from './hooks' +import { useTable, useTableEventStream, useTableRoom } from './hooks' import { DEFAULT_TABLE_DETAIL_SORT_DIRECTION, tableDetailParsers, @@ -153,6 +154,14 @@ export function Table({ } useTableEventStream({ tableId, workspaceId, onUsageLimitReached }) + // Live table presence (avatars + cell-selection highlights). Scoped to the + // dedicated tables page — the embedded surface passes no id, disabling it. + const { + otherUsers: presenceUsers, + remoteSelections, + emitCellSelection, + } = useTableRoom(embedded ? '' : tableId) + const [slideout, dispatch] = useReducer(slideoutReducer, { kind: 'none' }) const [showDeleteTableConfirm, setShowDeleteTableConfirm] = useState(false) const [isImportCsvOpen, setIsImportCsvOpen] = useState(false) @@ -654,6 +663,7 @@ export function Table({ breadcrumbs={breadcrumbs} aside={
+ {presenceUsers.length > 0 && } {selection.totalRunning > 0 || selection.hasActiveDispatch ? ( { +): Promise<{ metadata: TableMetadata; schemaChanged: boolean }> { const merged: TableMetadata = { ...(existingMetadata ?? {}), ...metadata } // When `columnOrder` is in the patch, scrub any workflow-group dependency @@ -635,7 +635,7 @@ export async function updateTableMetadata( .set(nextSchema ? { metadata: merged, schema: nextSchema } : { metadata: merged }) .where(eq(userTableDefinitions.id, tableId)) - return merged + return { metadata: merged, schemaChanged: nextSchema !== null } } /** diff --git a/packages/platform-authz/src/rooms.ts b/packages/platform-authz/src/rooms.ts index f8676cfde45..acb4d2779e5 100644 --- a/packages/platform-authz/src/rooms.ts +++ b/packages/platform-authz/src/rooms.ts @@ -1,4 +1,4 @@ -import { db, workspace, workspaceFiles } from '@sim/db' +import { db, userTableDefinitions, workspace, workspaceFiles } from '@sim/db' import { ROOM_TYPES, type RoomRef, type RoomType } from '@sim/realtime-protocol/rooms' import { and, eq, isNull } from 'drizzle-orm' import { getActiveWorkflowContext } from './workflow' @@ -55,6 +55,23 @@ async function resolveFileDocWorkspace(fileId: string): Promise { + const [table] = await db + .select({ workspaceId: userTableDefinitions.workspaceId }) + .from(userTableDefinitions) + .where(and(eq(userTableDefinitions.id, tableId), isNull(userTableDefinitions.archivedAt))) + .limit(1) + + if (!table?.workspaceId) return null + return resolveWorkspaceRoomWorkspace(table.workspaceId) +} + /** * Single source of truth mapping each room type to its resource→workspace * lookup. Every realtime room is workspace-scoped and authorizes through the @@ -74,6 +91,8 @@ const ROOM_WORKSPACE_RESOLVERS: Record = { [ROOM_TYPES.WORKSPACE_FILES]: resolveWorkspaceRoomWorkspace, // A file-doc room is addressed by file id; resolve it to its workspace. [ROOM_TYPES.WORKSPACE_FILE_DOC]: resolveFileDocWorkspace, + // A table room is addressed by table id; resolve it to its workspace. + [ROOM_TYPES.TABLE]: resolveTableWorkspace, } /** Resolves a room's owning workspace, or `null` if the room resource is gone. */ diff --git a/packages/realtime-protocol/package.json b/packages/realtime-protocol/package.json index 7023ec6a12a..e1995aae687 100644 --- a/packages/realtime-protocol/package.json +++ b/packages/realtime-protocol/package.json @@ -29,6 +29,10 @@ "./file-doc": { "types": "./src/file-doc.ts", "default": "./src/file-doc.ts" + }, + "./table-presence": { + "types": "./src/table-presence.ts", + "default": "./src/table-presence.ts" } }, "scripts": { diff --git a/packages/realtime-protocol/src/rooms.ts b/packages/realtime-protocol/src/rooms.ts index b81b65e72bb..de6a97dd82d 100644 --- a/packages/realtime-protocol/src/rooms.ts +++ b/packages/realtime-protocol/src/rooms.ts @@ -29,6 +29,12 @@ export const ROOM_TYPES = { * workspace-scoped {@link ROOM_TYPES.WORKSPACE_FILES} browser room. */ WORKSPACE_FILE_DOC: 'workspace-file-doc', + /** + * A single table's grid (one room per table). Carries live cell-selection + * presence — which cells each viewer has selected — so its id space is the + * table id. + */ + TABLE: 'table', } as const export type RoomType = (typeof ROOM_TYPES)[keyof typeof ROOM_TYPES] diff --git a/packages/realtime-protocol/src/table-presence.ts b/packages/realtime-protocol/src/table-presence.ts new file mode 100644 index 00000000000..45d06a94dc7 --- /dev/null +++ b/packages/realtime-protocol/src/table-presence.ts @@ -0,0 +1,103 @@ +/** + * Wire protocol for live table presence — which cell(s) each viewer has selected + * in a table grid. Carried over the shared, already-authenticated Socket.IO + * connection (the server relay is `apps/realtime/src/handlers/tables.ts`), + * separate from the one-way durable cell-status stream (`lib/table/events.ts`). + * + * Centralized here so the server emits and the client subscriptions cannot drift. + * This module is pure so both `apps/sim` and `apps/realtime` can import it. + */ + +/** Socket.IO event names for the table presence channel. */ +export const TABLE_PRESENCE_EVENTS = { + JOIN: 'join-table', + JOIN_SUCCESS: 'join-table-success', + JOIN_ERROR: 'join-table-error', + LEAVE: 'leave-table', + /** + * The sender's current cell selection. Sent client→server, then relayed + * server→peers with the sender's identity attached. + */ + CELL_SELECTION: 'table-cell-selection', +} as const + +/** + * A single cell address. Keyed by stable ids (never positional indices): row and + * column order differ per client under their own sort/filter, so an index would + * point at the wrong cell on the receiver. + */ +export interface TableCellRef { + rowId: string + columnId: string +} + +/** + * A viewer's grid selection: the `anchor` and `focus` corners of a rectangular + * range (a single-cell selection has `anchor` equal to `focus`). `null` clears + * the selection (the viewer has nothing selected). + */ +export type TableCellSelection = { + anchor: TableCellRef + focus: TableCellRef + /** + * True while the viewer is actively editing the `focus` cell (they double-clicked + * or started typing). Peers render the cell with a slightly darker fill — the + * Google-Sheets "someone is typing here" signal — on top of the color border. + */ + editing?: boolean +} | null + +/** Client→server join request for a table presence room. */ +export interface JoinTablePayload { + tableId: string + /** Stable per-tab id so a reconnecting tab replaces its own stale presence entry. */ + tabSessionId?: string +} + +/** Server→client rejection of a {@link TABLE_PRESENCE_EVENTS.JOIN}. */ +export interface JoinTableError { + tableId: string + error: string + code: + | 'AUTHENTICATION_REQUIRED' + | 'ROOM_MANAGER_UNAVAILABLE' + | 'INVALID_PAYLOAD' + | 'VERIFY_ACCESS_FAILED' + | 'NOT_FOUND' + | 'ACCESS_DENIED' + | 'JOIN_FAILED' + /** Whether re-attempting the join (e.g. after reconnect) could succeed. */ + retryable: boolean +} + +/** + * A remote viewer of a table, as carried in the join ack and every + * `table:presence-update` broadcast. `cell` is the viewer's current selection at + * broadcast time (absent until they select something). + */ +export interface TablePresenceUser { + socketId: string + userId: string + userName: string + avatarUrl?: string | null + cell?: TableCellSelection +} + +/** Server→client ack of a successful join, carrying the room's current viewers. */ +export interface JoinTableSuccess { + tableId: string + socketId: string + presenceUsers: TablePresenceUser[] +} + +/** + * A single remote viewer's cell-selection delta, relayed to peers on + * {@link TABLE_PRESENCE_EVENTS.CELL_SELECTION}. Lower-latency than a full presence + * broadcast for the frequent case of just moving the selection. Carries only the + * socket id + selection — peers already hold the viewer's identity (name, color) from + * the presence roster, so it is not repeated on this high-frequency channel. + */ +export interface TableCellSelectionBroadcast { + socketId: string + cell: TableCellSelection +} From 0a36d6401b100d2f5d701cbe70727d3c944a4ecd Mon Sep 17 00:00:00 2001 From: Waleed Date: Sun, 26 Jul 2026 11:27:37 -0700 Subject: [PATCH 09/53] feat(realtime): accurate in-file presence + collaborative-caret polish (#5965) Per-session file-doc presence (avatars count other sessions like the canvas), StrictMode-safe stable Y.Doc (fixes blank-doc on join), flush caret cap + restored hover hit-slop, and three join-lifecycle race fixes unifying file-doc + workspace-files on one intent-tracked monotonic generation model. All findings root-caused with regression tests. --- apps/realtime/src/handlers/connection.ts | 24 +- apps/realtime/src/handlers/file-doc.test.ts | 87 +++++- apps/realtime/src/handlers/file-doc.ts | 86 +++++- .../src/handlers/workspace-files.test.ts | 120 ++++++-- apps/realtime/src/handlers/workspace-files.ts | 277 ++++++++---------- apps/realtime/src/index.test.ts | 2 +- .../components/presence/presence-avatars.tsx | 6 +- .../collaboration/caret-presence.test.ts | 55 ++++ .../collaboration/caret-presence.ts | 143 +++++++++ .../collaboration/file-doc-avatars.tsx | 14 + .../collaboration/file-doc-provider.test.ts | 20 ++ .../collaboration/file-doc-provider.ts | 12 + .../collaboration/file-doc-room-context.tsx | 43 +++ .../use-file-doc-collaboration.ts | 90 +++++- .../rich-markdown-editor/editor-extensions.ts | 14 +- .../rich-markdown-editor.css | 105 +++++-- .../workspace/[workspaceId]/files/files.tsx | 76 ++--- .../files/hooks/use-workspace-files-room.ts | 79 +---- .../table-grid/remote-selection-overlay.tsx | 180 ++++++++++-- .../components/table-grid/table-grid.tsx | 19 +- .../[tableId]/components/table-grid/utils.ts | 19 ++ packages/realtime-protocol/src/file-doc.ts | 26 ++ 22 files changed, 1112 insertions(+), 385 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context.tsx diff --git a/apps/realtime/src/handlers/connection.ts b/apps/realtime/src/handlers/connection.ts index aa134d8a771..bd6d143aa36 100644 --- a/apps/realtime/src/handlers/connection.ts +++ b/apps/realtime/src/handlers/connection.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { parseRoomName, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { parseRoomName, ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' import { cleanupFileDocForSocket } from '@/handlers/file-doc' import { cleanupPendingSubblocksForSocket } from '@/handlers/subblocks' import { cleanupPendingVariablesForSocket } from '@/handlers/variables' @@ -8,6 +8,14 @@ import type { IRoomManager } from '@/rooms' const logger = createLogger('ConnectionHandlers') +/** + * Room types whose presence lives in the room manager (Redis-backed), so a disconnect must + * remove the socket + broadcast a correction. The workspace-files and file-doc rooms are + * NOT here: workspace-files carries no presence (native Socket.IO membership only), and + * file-doc broadcasts its own server-authenticated roster via `cleanupFileDocForSocket`. + */ +const PRESENCE_BEARING_TYPES = new Set([ROOM_TYPES.WORKFLOW, ROOM_TYPES.TABLE]) + export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { socket.on('error', (error) => { logger.error(`Socket ${socket.id} error:`, error) @@ -33,8 +41,9 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager cleanupPendingSubblocksForSocket(socket.id) cleanupPendingVariablesForSocket(socket.id) // Clear the socket's collaborative-document awareness (removes its caret for - // everyone else) and drop the room if it was the last editor. - cleanupFileDocForSocket(socket.id, roomManager.io) + // everyone else) and drop the room if it was the last editor. `endOfLife` drops the + // socket's join-generation entry — safe only here, on true disconnect (see cleanup). + cleanupFileDocForSocket(socket.id, roomManager.io, true) // A socket may occupy multiple rooms (one per type). Remove it from every // room the manager knows about. @@ -47,12 +56,13 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager const wasInRooms = new Map() for (const room of removedRooms) wasInRooms.set(roomName(room), room) for (const name of liveRoomNames) { - // `wasInRooms.has(name)` already excludes every room the manager removed - // (same room-name key via the roomName/parseRoomName bijection), so any - // room reaching here was NOT in `removedRooms` and needs a removal attempt. + // `wasInRooms.has(name)` already excludes every room the manager removed (same + // room-name key via the roomName/parseRoomName bijection). Skip room types with no + // manager-tracked presence (workspace-files, file-doc): removing there is a no-op and + // broadcasting a correction would emit a dead presence-update no client listens to. if (name === socket.id || wasInRooms.has(name)) continue const ref = parseRoomName(name) - if (!ref) continue + if (!ref || !PRESENCE_BEARING_TYPES.has(ref.type)) continue wasInRooms.set(name, ref) await roomManager.removeUserFromRoom(ref, socket.id) } diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 48da76f22ac..1fbecde9078 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -54,6 +54,8 @@ function createSocket(id: string, overrides?: Record) { id, userId: 'user-1', userName: 'Test User', + // Set so the server's roster resolves the avatar from the socket (never the DB). + userImage: 'avatar.png', disconnected: false, on: vi.fn((event: string, handler: Handler) => { handlers[event] = handler @@ -133,7 +135,9 @@ describe('setupWorkspaceFileDocHandlers', () => { afterEach(() => { // The room store is module-global; drop every room the test's sockets opened. const { io } = createIo() - for (const id of createdSocketIds) cleanupFileDocForSocket(id, io) + // Simulate a full disconnect between tests (`endOfLife`) so the module-global join-generation + // map is cleared and never bleeds a counter into the next test. + for (const id of createdSocketIds) cleanupFileDocForSocket(id, io, true) createdSocketIds.clear() vi.clearAllTimers() vi.useRealTimers() @@ -435,7 +439,7 @@ describe('setupWorkspaceFileDocHandlers', () => { const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) s.socket.disconnected = true - cleanupFileDocForSocket('socket-a', io) // disconnect cleanup — no-op, nothing registered yet + cleanupFileDocForSocket('socket-a', io, true) // disconnect cleanup — no-op, nothing registered yet resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) await pending @@ -462,6 +466,49 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') }) + it('does not reset the join generation on a leave, so an in-flight join still binds', async () => { + const { io } = createIo() + const s = setup('socket-a', io) + + // file-1 join completes; the socket is registered in file-1. + await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + // file-2 join goes in-flight (authorize deferred). + let resolveAuth: (v: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValueOnce(new Promise((resolve) => (resolveAuth = resolve))) + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) + + // A deferred leave for the prior file-1 lands while file-2's join awaits authorization. Its + // cleanup must NOT reset the monotonic join generation, or file-2's guard would see an emptied + // map (`undefined !== generation`) and abort the join the client actually wants. + s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + + resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + expect(joinSuccessFileId(s.socket)).toBe('file-2') + expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') + }) + + it('cancels an in-flight join when the client leaves that same file (no ghost owner)', async () => { + const { io, sent } = createIo() + let resolveAuth: (v: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValueOnce(new Promise((resolve) => (resolveAuth = resolve))) + const s = setup('socket-a', io) + + // Join file-1 is awaiting authorization when the client leaves file-1 (fast open→close). + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + // The stale join must not register: no success, no room join, and no presence broadcast that + // would leave a ghost collaborator until disconnect. + expect(s.socket.join).not.toHaveBeenCalled() + expect(joinSuccessFileId(s.socket)).toBeUndefined() + expect(sent.some((m) => m.event === FILE_DOC_EVENTS.PRESENCE)).toBe(false) + }) + it('scopes LEAVE to the named file (a leave for a different file is a no-op)', async () => { const { io } = createIo() const a = setup('socket-a', io) @@ -574,4 +621,40 @@ describe('setupWorkspaceFileDocHandlers', () => { // No re-election: the successful seed cancelled the deadline. expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)).toBeUndefined() }) + + it('broadcasts a server-authenticated presence roster on join, one entry per session', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io, { userId: 'user-a', userName: 'Ada', userImage: 'ada.png' }) + const b = setup('socket-b', io, { userId: 'user-b', userName: 'Bob', userImage: 'bob.png' }) + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + const roster = sent.filter((m) => m.event === FILE_DOC_EVENTS.PRESENCE).at(-1)?.payload as { + fileId: string + users: Array<{ socketId: string; userId: string; userName: string; avatarUrl: string | null }> + } + expect(roster.fileId).toBe('file-1') + // Identity is each socket's authenticated session — not any client-supplied value. + expect([...roster.users].sort((x, y) => x.userId.localeCompare(y.userId))).toEqual([ + { socketId: 'socket-a', userId: 'user-a', userName: 'Ada', avatarUrl: 'ada.png' }, + { socketId: 'socket-b', userId: 'user-b', userName: 'Bob', avatarUrl: 'bob.png' }, + ]) + }) + + it('keeps a per-session entry for two sockets of the SAME user (no server-side user dedup)', async () => { + const { io, sent } = createIo() + // Two tabs of one account: the client self-excludes its own socket, so the roster must carry + // BOTH sessions or a client could never see the other tab as present. + const a = setup('socket-a', io, { userId: 'user-a', userName: 'Ada', userImage: 'ada.png' }) + const b = setup('socket-b', io, { userId: 'user-a', userName: 'Ada', userImage: 'ada.png' }) + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + const roster = sent.filter((m) => m.event === FILE_DOC_EVENTS.PRESENCE).at(-1)?.payload as { + users: Array<{ socketId: string; userId: string }> + } + expect([...roster.users].map((u) => u.socketId).sort()).toEqual(['socket-a', 'socket-b']) + }) }) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index a6c43c00cfa..d019a561bd8 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -4,6 +4,7 @@ import { FILE_DOC_EVENTS, FILE_DOC_MESSAGE_TYPE, FILE_DOC_SEED, + type FileDocPresenceUser, type JoinFileDocPayload, type LeaveFileDocPayload, toFileDocBytes, @@ -15,6 +16,7 @@ import type { Server } from 'socket.io' import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' +import { resolveAvatarUrl } from '@/handlers/avatar' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -57,6 +59,10 @@ interface FileDocOwner { /** The owning user — used to tell a reconnect (same user reusing its Yjs client * id) from a spoof (a different user binding a peer's id). */ userId: string + /** Server-authenticated display identity for the presence roster (from the socket's + * session, never the client-set awareness — so a peer cannot spoof it). */ + userName: string + avatarUrl: string | null } interface FileDocRoom { @@ -116,6 +122,28 @@ function broadcast(io: Server, name: string, payload: Uint8Array, exceptSocketId channel.emit(FILE_DOC_EVENTS.MESSAGE, payload) } +/** + * Broadcast the room's collaborator roster to everyone in it, for the avatar stack. One entry + * PER SESSION (socket) — the client excludes its own socket and dedupes the remainder per user + * for display, so a second tab of the same account still registers as present (mirroring the + * canvas presence model). Deduping here instead would drop the current user's other sessions + * asymmetrically (only one socket survives), so each client could never reliably self-exclude. + * Identity comes from each owner's server-authenticated session — never the client-set awareness + * — so a peer cannot spoof or suppress an entry. + */ +function broadcastFileDocPresence(io: Server, name: string, room: FileDocRoom) { + const users: FileDocPresenceUser[] = [] + for (const [socketId, owner] of room.owners) { + users.push({ + socketId, + userId: owner.userId, + userName: owner.userName, + avatarUrl: owner.avatarUrl, + }) + } + io.to(name).emit(FILE_DOC_EVENTS.PRESENCE, { fileId: room.fileId, users }) +} + /** Whether the client has recorded that it seeded the document's initial content. */ function isDocSeeded(doc: Y.Doc): boolean { return doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag) === true @@ -316,10 +344,15 @@ function handleMessage(socket: AuthenticatedSocket, data: unknown) { * seeding, and drop the room's document when the last collaborator leaves. * Exported for the disconnect handler; safe to call for a socket in no room. */ -export function cleanupFileDocForSocket(socketId: string, io: Server): void { - // Drop the join generation so an in-flight JOIN for this socket aborts after - // its authorize resolves, and the map never leaks across the socket's life. - joinGeneration.delete(socketId) +export function cleanupFileDocForSocket(socketId: string, io: Server, endOfLife = false): void { + // The join-generation counter is monotonic for the socket's WHOLE life and must survive a room + // switch/leave: resetting it here would let the next join reuse a low number that a still + // in-flight earlier join also holds, so that stale join passes the generation guard and rebinds + // the socket to the wrong document. Drop it ONLY when the socket is truly gone (disconnect), + // which is also the only place the map would otherwise leak. An in-flight join is already + // aborted on disconnect by the `socket.disconnected` check, and on a switch by a newer join + // bumping the generation — neither needs this delete. + if (endOfLife) joinGeneration.delete(socketId) const name = socketToRoomName.get(socketId) if (!name) return @@ -334,6 +367,8 @@ export function cleanupFileDocForSocket(socketId: string, io: Server): void { // Fires the awareness `update` handler with a non-socket origin → the removal // is broadcast to every remaining client, so the departed caret vanishes. awarenessProtocol.removeAwarenessStates(room.awareness, [owner.clientId], null) + // Refresh the roster for whoever remains (server-authenticated identity). + broadcastFileDocPresence(io, name, room) } // Hand off the seeder role: if the elected seeder left before it seeded, elect @@ -351,12 +386,22 @@ export function cleanupFileDocForSocket(socketId: string, io: Server): void { * file id; joining requires workspace `write` (editing a document). Mirrors the * workspace-files join shape (auth → readiness → validate → authorize → join), * then runs the Yjs sync/awareness handshake. + * + * The avatar roster is derived from this room's own `owners` map and broadcast as + * `FILE_DOC_EVENTS.PRESENCE` — NOT the Redis-backed room-manager presence the workflow / + * table rooms use — because the file-doc room already owns an authoritative in-memory Y.Doc + * pinned to a single replica, so the session identity is right here with no extra store. */ export function setupWorkspaceFileDocHandlers( socket: AuthenticatedSocket, roomManager: IRoomManager ) { const io = roomManager.io + // The file this socket currently intends to edit (set when a join starts). A leave targeting it + // — or an unscoped leave — advances the join generation to cancel an in-flight join, so a join + // awaiting authorization can't complete after the client left and register a ghost owner. A + // leave for a DIFFERENT file must NOT cancel it (a document switch), mirroring workspace-files. + let currentFileId: string | null = null socket.on(FILE_DOC_EVENTS.JOIN, async ({ fileId, clientId }: JoinFileDocPayload) => { try { @@ -376,9 +421,11 @@ export function setupWorkspaceFileDocHandlers( return } - // Claim this JOIN's generation before the async authorize below. + // Claim this JOIN's generation before the async authorize below, and record the file the + // socket now intends to edit so a leave for it can cancel this join if it's still in-flight. const generation = (joinGeneration.get(socket.id) ?? 0) + 1 joinGeneration.set(socket.id, generation) + currentFileId = fileId const room = fileDocRoom(fileId) const name = roomName(room) @@ -408,9 +455,13 @@ export function setupWorkspaceFileDocHandlers( return } - // Abort a JOIN superseded during authorization: the socket disconnected, or - // a newer JOIN (a document switch) bumped the generation. Registering here - // would leak a dead socket's room or bind the socket to the wrong document. + // Server-authenticated identity for the presence roster (never trusts the client-set + // awareness). Resolved here so the generation guard below also covers this await. + const avatarUrl = await resolveAvatarUrl(socket, userId) + + // Abort a JOIN superseded during authorization/identity resolution: the socket + // disconnected, or a newer JOIN (a document switch) bumped the generation. Registering + // here would leak a dead socket's room or bind the socket to the wrong document. if (socket.disconnected || joinGeneration.get(socket.id) !== generation) return // Switched documents on the same socket — leave the previous one first (a @@ -449,11 +500,13 @@ export function setupWorkspaceFileDocHandlers( awarenessProtocol.removeAwarenessStates(entry.awareness, [previous.clientId], null) } - entry.owners.set(socket.id, { clientId, userId }) + entry.owners.set(socket.id, { clientId, userId, userName, avatarUrl }) socketToRoomName.set(socket.id, name) socket.join(name) socket.emit(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId }) + // Server-authenticated roster → everyone in the room, including this joiner. + broadcastFileDocPresence(io, name, entry) // Begin the sync handshake: send the server's state (sync step 1). The // client replies with its updates and requests the server's in return. @@ -496,10 +549,17 @@ export function setupWorkspaceFileDocHandlers( socket.on(FILE_DOC_EVENTS.LEAVE, (payload?: LeaveFileDocPayload) => { try { - // Only affect a REGISTERED room; never touch the join generation here. A - // leave that raced ahead of an in-flight join (no room registered yet) is a - // no-op — bumping the generation would silently abort an unrelated join for - // a different file (a document switch), leaving the socket bound to nothing. + // Cancel an in-flight join whose file the client is now leaving (or an unscoped leave): a + // join still awaiting authorization would otherwise complete after the client left, register + // as an owner, and broadcast a ghost collaborator until disconnect. Guard on the current + // file intent so a stale/deferred leave for a DIFFERENT file can't abort the join the client + // has since switched to (bumping the generation blindly caused that regression in #5941). + if (!payload?.fileId || payload.fileId === currentFileId) { + joinGeneration.set(socket.id, (joinGeneration.get(socket.id) ?? 0) + 1) + currentFileId = null + } + // Tear down membership only for a REGISTERED room; a leave that raced ahead of an in-flight + // join (nothing registered yet) has already cancelled it above. const name = socketToRoomName.get(socket.id) if (!name) return // Scope the leave to the named file when provided: a deferred leave from a diff --git a/apps/realtime/src/handlers/workspace-files.test.ts b/apps/realtime/src/handlers/workspace-files.test.ts index 76f20d9a7e5..350651c94a9 100644 --- a/apps/realtime/src/handlers/workspace-files.test.ts +++ b/apps/realtime/src/handlers/workspace-files.test.ts @@ -1,7 +1,6 @@ /** * @vitest-environment node */ -import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { IRoomManager } from '@/rooms' @@ -20,29 +19,28 @@ vi.mock('@sim/platform-authz/rooms', () => ({ import { setupWorkspaceFilesHandlers } from '@/handlers/workspace-files' -interface JoinPayload { - workspaceId: string - folderId?: string | null - tabSessionId?: string -} +type Payload = { workspaceId?: string } function createSocket(overrides?: Record) { - const handlers: Record Promise | void> = {} + const handlers: Record Promise | void> = {} + // Live Set so the handler's native `socket.rooms` membership tracking works in tests. + const rooms = new Set() const socket = { id: 'socket-1', userId: 'user-1', userName: 'Test User', userImage: 'avatar.png', - on: vi.fn((event: string, handler: (payload: JoinPayload) => Promise | void) => { + rooms, + on: vi.fn((event: string, handler: (payload?: Payload) => Promise | void) => { handlers[event] = handler }), emit: vi.fn(), - join: vi.fn(), - leave: vi.fn(), + join: vi.fn((room: string) => rooms.add(room)), + leave: vi.fn((room: string) => rooms.delete(room)), to: vi.fn().mockReturnValue({ emit: vi.fn() }), ...overrides, } - return { handlers, socket } + return { handlers, socket, rooms } } function createRoomManager(overrides?: Partial): IRoomManager { @@ -136,35 +134,101 @@ describe('setupWorkspaceFilesHandlers', () => { ) }) - it('joins the workspace files room and broadcasts presence on success', async () => { + it('joins the workspace files room on success without any presence bookkeeping', async () => { const { socket, handlers } = createSocket() - const roomManager = createRoomManager({ - getRoomUsers: vi.fn().mockResolvedValue([]), - }) + const roomManager = createRoomManager() setupWorkspaceFilesHandlers( socket as unknown as Parameters[0], roomManager ) - await handlers['join-workspace-files']({ + await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) + + expect(socket.join).toHaveBeenCalledWith('workspace-files:ws-1') + expect(socket.emit).toHaveBeenCalledWith('join-workspace-files-success', { workspaceId: 'ws-1', - folderId: 'folder-1', - tabSessionId: 'tab-1', }) + // The room is live-tree-only: no room-manager presence is tracked or broadcast. + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + expect(roomManager.broadcastPresenceUpdate).not.toHaveBeenCalled() + }) + + it('leaves a previously-joined files room when switching workspaces', async () => { + const { socket, handlers, rooms } = createSocket() + rooms.add('workspace-files:ws-old') + setupWorkspaceFilesHandlers( + socket as unknown as Parameters[0], + createRoomManager() + ) + await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) + + expect(socket.leave).toHaveBeenCalledWith('workspace-files:ws-old') expect(socket.join).toHaveBeenCalledWith('workspace-files:ws-1') - expect(roomManager.addUserToRoom).toHaveBeenCalledWith( - { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-1' }, - 'socket-1', - expect.objectContaining({ userId: 'user-1', folderId: 'folder-1', role: 'admin' }) + }) + + it('leaves the scoped files room on leave', () => { + const { socket, handlers, rooms } = createSocket() + rooms.add('workspace-files:ws-1') + setupWorkspaceFilesHandlers( + socket as unknown as Parameters[0], + createRoomManager() ) - expect(socket.emit).toHaveBeenCalledWith( - 'join-workspace-files-success', - expect.objectContaining({ workspaceId: 'ws-1', socketId: 'socket-1' }) + + handlers['leave-workspace-files']({ workspaceId: 'ws-1' }) + + expect(socket.leave).toHaveBeenCalledWith('workspace-files:ws-1') + }) + + it('cancels an in-flight join when the user leaves that workspace mid-authorize', async () => { + const { socket, handlers } = createSocket() + let resolveAuth: (value: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve + }) ) - expect(roomManager.broadcastPresenceUpdate).toHaveBeenCalledWith({ - type: ROOM_TYPES.WORKSPACE_FILES, - id: 'ws-1', + setupWorkspaceFilesHandlers( + socket as unknown as Parameters[0], + createRoomManager() + ) + + // Join ws-1 is awaiting authorization when the view unmounts and leaves ws-1. + const joinPromise = handlers['join-workspace-files']({ workspaceId: 'ws-1' }) + handlers['leave-workspace-files']({ workspaceId: 'ws-1' }) + resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + // The stale join must NOT join the room the client has since left (no stranded membership). + expect(socket.join).not.toHaveBeenCalled() + expect(socket.emit).not.toHaveBeenCalledWith('join-workspace-files-success', { + workspaceId: 'ws-1', + }) + }) + + it('does not cancel an in-flight join when a deferred leave targets a different workspace', async () => { + const { socket, handlers } = createSocket() + let resolveAuth: (value: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve + }) + ) + setupWorkspaceFilesHandlers( + socket as unknown as Parameters[0], + createRoomManager() + ) + + // The client has switched to ws-2 (join in-flight) when a stale leave for the prior ws-1 lands. + const joinPromise = handlers['join-workspace-files']({ workspaceId: 'ws-2' }) + handlers['leave-workspace-files']({ workspaceId: 'ws-1' }) + resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-2', workspacePermission: 'admin' }) + await joinPromise + + // The deferred leave for ws-1 must not abort the join the client actually wants (ws-2). + expect(socket.join).toHaveBeenCalledWith('workspace-files:ws-2') + expect(socket.emit).toHaveBeenCalledWith('join-workspace-files-success', { + workspaceId: 'ws-2', }) }) }) diff --git a/apps/realtime/src/handlers/workspace-files.ts b/apps/realtime/src/handlers/workspace-files.ts index 0c7d3ef8c9b..fa3b56111fa 100644 --- a/apps/realtime/src/handlers/workspace-files.ts +++ b/apps/realtime/src/handlers/workspace-files.ts @@ -1,10 +1,8 @@ import { createLogger } from '@sim/logger' import { authorizeRoom } from '@sim/platform-authz/rooms' import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' -import { resolveAvatarUrl } from '@/handlers/avatar' import type { AuthenticatedSocket } from '@/middleware/auth' -import type { IRoomManager, UserPresence } from '@/rooms' -import { filterVisiblePresence, sweepStalePresence } from '@/rooms/presence-visibility' +import type { IRoomManager } from '@/rooms' const logger = createLogger('WorkspaceFilesHandlers') @@ -14,188 +12,143 @@ const filesRoom = (workspaceId: string): RoomRef => ({ id: workspaceId, }) +/** Socket.IO room-name prefix shared by every workspace-files room. */ +const FILES_ROOM_PREFIX = `${ROOM_TYPES.WORKSPACE_FILES}:` + interface JoinPayload { workspaceId: string - folderId?: string | null - tabSessionId?: string } /** - * Presence handlers for the workspace file browser. Mirrors the workflow join - * flow but is workspace-scoped (room id = workspaceId) and read-only presence: - * there are no persisted file operations over the socket — file mutations go - * through the HTTP API, which fans out a `workspace-files-changed` event - * separately. The viewer's `folderId` is recorded at join (a future hook for - * folder-scoped presence); there is no cursor channel here yet. + * Keeps the workspace file browser live. The socket joins a workspace-scoped Socket.IO room + * so a `workspace-files-changed` event — fanned out by the HTTP mutation API — reaches every + * viewer, who then refetches. This room carries NO presence: "who's in a file" comes from + * the per-file doc room, and file mutations go over HTTP. Membership is tracked natively by + * Socket.IO (`socket.rooms`), so a workspace switch just leaves the prior files room — no + * room-manager presence bookkeeping to keep in sync. */ export function setupWorkspaceFilesHandlers( socket: AuthenticatedSocket, roomManager: IRoomManager ) { - socket.on( - 'join-workspace-files', - async ({ workspaceId, folderId, tabSessionId }: JoinPayload) => { - try { - const userId = socket.userId - const userName = socket.userName - - if (!userId || !userName) { - socket.emit('join-workspace-files-error', { - workspaceId, - error: 'Authentication required', - code: 'AUTHENTICATION_REQUIRED', - retryable: false, - }) - return - } - - if (!roomManager.isReady()) { - socket.emit('join-workspace-files-error', { - workspaceId, - error: 'Realtime unavailable', - code: 'ROOM_MANAGER_UNAVAILABLE', - retryable: true, - }) - return - } - - // Validate the client-supplied id before it reaches the DB query (matches - // the /api/workspace-files-changed guard; join payloads are otherwise raw - // client input). - if (typeof workspaceId !== 'string' || workspaceId.length === 0) { - socket.emit('join-workspace-files-error', { - workspaceId: typeof workspaceId === 'string' ? workspaceId : '', - error: 'Invalid workspace id', - code: 'INVALID_PAYLOAD', - retryable: false, - }) - return - } - - const room = filesRoom(workspaceId) - - let authorized: Awaited> - try { - authorized = await authorizeRoom({ userId, room, action: 'read' }) - } catch (error) { - logger.warn(`Error authorizing files room for ${userId}:`, error) - socket.emit('join-workspace-files-error', { - workspaceId, - error: 'Failed to verify workspace access', - code: 'VERIFY_ACCESS_FAILED', - retryable: true, - }) - return - } - - if (!authorized.allowed) { - socket.emit('join-workspace-files-error', { - workspaceId, - error: authorized.status === 404 ? 'Workspace not found' : 'Access denied to workspace', - code: authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', - retryable: false, - }) - return - } - - // Leave a previously-joined files room if switching workspaces. - const currentRoom = await roomManager.getRoomForSocket( - socket.id, - ROOM_TYPES.WORKSPACE_FILES - ) - if (currentRoom && currentRoom.id !== workspaceId) { - socket.leave(roomName(currentRoom)) - await roomManager.removeUserFromRoom(currentRoom, socket.id) - await roomManager.broadcastPresenceUpdate(currentRoom) - } - - // Clean up the same user's stale socket from the same tab (e.g. a reconnect - // that raced the old socket's disconnect), so presence shows one entry. - if (tabSessionId) { - const existingUsers = await roomManager.getRoomUsers(room) - for (const existing of existingUsers) { - if ( - existing.socketId !== socket.id && - existing.userId === userId && - existing.tabSessionId === tabSessionId - ) { - await roomManager.removeUserFromRoom(room, existing.socketId) - await roomManager.io.in(existing.socketId).socketsLeave(roomName(room)) - } - } - } - - // Reclaim any presence orphaned by an ungraceful disconnect (pod crash - // fires no `disconnecting` event; the room hashes have no TTL). - await sweepStalePresence(roomManager, room) - - socket.join(roomName(room)) - - const presence: UserPresence = { - userId, - room, - userName, - socketId: socket.id, - tabSessionId, - joinedAt: Date.now(), - lastActivity: Date.now(), - role: authorized.workspacePermission ?? 'read', - folderId: folderId ?? null, - avatarUrl: await resolveAvatarUrl(socket, userId), - } - - await roomManager.addUserToRoom(room, socket.id, presence) + // Monotonic per-socket join counter: each join captures its number and, after the async + // authorize, aborts if a newer intent has superseded it — a fast workspace switch A→B can + // otherwise let A's late completion leave B and strand the socket in A, missing B's + // `workspace-files-changed` invalidations. + let joinGeneration = 0 + // The workspace the socket currently intends to be in (set when a join starts). A leave that + // targets this workspace — or an unscoped "leave all" — advances joinGeneration so an in-flight + // join is cancelled instead of completing after the view has closed. A stale/deferred leave for + // a DIFFERENT workspace must NOT advance it, or it would abort the join the client has since + // switched to (the bug that bit the file-doc room in #5941). + let currentWorkspace: string | null = null + + socket.on('join-workspace-files', async ({ workspaceId }: JoinPayload) => { + const joinAttempt = (joinGeneration += 1) + currentWorkspace = workspaceId + try { + if (!socket.userId || !socket.userName) { + socket.emit('join-workspace-files-error', { + workspaceId, + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + return + } - // Filter the join ack to live members so a new joiner never briefly sees a - // ghost from an entry the sweep hasn't reclaimed yet. - const presenceUsers = await filterVisiblePresence( - roomManager.io, - room, - await roomManager.getRoomUsers(room) - ) - socket.emit('join-workspace-files-success', { + if (!roomManager.isReady()) { + socket.emit('join-workspace-files-error', { workspaceId, - socketId: socket.id, - presenceUsers, + error: 'Realtime unavailable', + code: 'ROOM_MANAGER_UNAVAILABLE', + retryable: true, }) + return + } + + // Validate the client-supplied id before it reaches the DB query (join payloads are + // otherwise raw client input). + if (typeof workspaceId !== 'string' || workspaceId.length === 0) { + socket.emit('join-workspace-files-error', { + workspaceId: typeof workspaceId === 'string' ? workspaceId : '', + error: 'Invalid workspace id', + code: 'INVALID_PAYLOAD', + retryable: false, + }) + return + } - await roomManager.broadcastPresenceUpdate(room) + const room = filesRoom(workspaceId) - logger.info(`User ${userId} (${userName}) joined files room for workspace ${workspaceId}`) + let authorized: Awaited> + try { + authorized = await authorizeRoom({ userId: socket.userId, room, action: 'read' }) } catch (error) { - logger.error('Error joining workspace files room:', error) - // Roll back any partial join so a failed attempt can't leave the socket in - // the Socket.IO room or a stale presence entry behind (mirrors the workflow - // join's rollback), before signalling a retryable failure. - try { - const room = filesRoom(workspaceId) - socket.leave(roomName(room)) - await roomManager.removeUserFromRoom(room, socket.id) - } catch {} + logger.warn(`Error authorizing files room for ${socket.userId}:`, error) socket.emit('join-workspace-files-error', { workspaceId, - error: 'Failed to join workspace files', - code: 'JOIN_FAILED', + error: 'Failed to verify workspace access', + code: 'VERIFY_ACCESS_FAILED', retryable: true, }) + return } - } - ) - socket.on('leave-workspace-files', async (payload?: { workspaceId?: string }) => { - try { - if (!roomManager.isReady()) return - const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKSPACE_FILES) - if (!room) return - // Scope the leave to a specific workspace when the client provides one: a - // deferred leave from a prior page must not evict the socket from a room it - // has since switched into (workspace A→B leaves A's leave targeting B). - if (payload?.workspaceId && payload.workspaceId !== room.id) return - socket.leave(roomName(room)) - await roomManager.removeUserFromRoom(room, socket.id) - await roomManager.broadcastPresenceUpdate(room, socket.id) + if (!authorized.allowed) { + socket.emit('join-workspace-files-error', { + workspaceId, + error: authorized.status === 404 ? 'Workspace not found' : 'Access denied to workspace', + code: authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', + retryable: false, + }) + return + } + + // A newer join started on this socket during authorize (or it dropped): abort so a + // stale join can't leave the room the client has since switched to. + if (joinGeneration !== joinAttempt || socket.disconnected) return + + // Leave any previously-joined files room (workspace switch), read straight from the + // socket's native room membership so there's no presence store to keep in sync. + const target = roomName(room) + for (const joined of socket.rooms) { + if (joined !== target && joined.startsWith(FILES_ROOM_PREFIX)) socket.leave(joined) + } + + socket.join(target) + socket.emit('join-workspace-files-success', { workspaceId }) } catch (error) { - logger.error('Error leaving workspace files room:', error) + logger.error('Error joining workspace files room:', error) + try { + socket.leave(roomName(filesRoom(workspaceId))) + } catch {} + socket.emit('join-workspace-files-error', { + workspaceId, + error: 'Failed to join workspace files', + code: 'JOIN_FAILED', + retryable: true, + }) + } + }) + + socket.on('leave-workspace-files', (payload?: { workspaceId?: string }) => { + // Cancel an in-flight join whose target the client is now leaving: a join awaiting + // authorization when the view unmounts would otherwise complete afterwards and strand the + // socket in a room it has left. Only when the leave targets the current join intent (or is + // unscoped) — a deferred leave for a different workspace must not abort the join the client + // has since switched to. + if (!payload?.workspaceId || payload.workspaceId === currentWorkspace) { + joinGeneration += 1 + currentWorkspace = null + } + // Scope the leave to a specific workspace when the client provides one: a deferred leave + // from a prior page must not evict a files room the socket has since switched into. + const target = payload?.workspaceId ? roomName(filesRoom(payload.workspaceId)) : null + for (const joined of socket.rooms) { + if (!joined.startsWith(FILES_ROOM_PREFIX)) continue + if (target && joined !== target) continue + socket.leave(joined) } }) } diff --git a/apps/realtime/src/index.test.ts b/apps/realtime/src/index.test.ts index 5015b58d9bf..eb7ee030937 100644 --- a/apps/realtime/src/index.test.ts +++ b/apps/realtime/src/index.test.ts @@ -4,10 +4,10 @@ * @vitest-environment node */ import { createServer, request as httpRequest } from 'http' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { createMockLogger } from '@sim/testing' import { randomInt } from '@sim/utils/random' import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { createSocketIOServer } from '@/config/socket' import { MemoryRoomManager, workflowRoom } from '@/rooms' import { createHttpHandler } from '@/routes/http' diff --git a/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx index 3698578f8af..70c75e416b1 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx @@ -6,7 +6,9 @@ import { getUserColor } from '@/lib/workspaces/colors' /** Minimal presence shape the avatar stack renders — shared by workflow and files. */ export interface PresenceAvatarUser { - socketId: string + /** Unique id per presence entry, used as the render key: a socket id where presence is + * per-connection (workflow), absent where entries are deduped per user (file docs). */ + socketId?: string userId: string userName?: string avatarUrl?: string | null @@ -107,7 +109,7 @@ export function PresenceAvatars({ users, maxVisible = DEFAULT_MAX_VISIBLE }: Pre )} {visibleUsers.map((user, index) => ( - + ))}
) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.test.ts new file mode 100644 index 00000000000..523335ad8e5 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.test.ts @@ -0,0 +1,55 @@ +/** + * @vitest-environment jsdom + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { activateCaretLabel, CARET_LABEL_HOLD_MS, renderCaret } from './caret-presence' + +const ACTIVE = 'collaboration-carets__caret--active' +const FLIP = 'collaboration-carets__caret--flip' + +describe('caret-presence', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + it('builds a tagged caret with a name label, shown on appearance', () => { + const caret = renderCaret({ name: 'Ada', color: '#f783ac', clientId: 4242 }) + expect(caret.classList.contains('collaboration-carets__caret')).toBe(true) + expect(caret.dataset.caretClientId).toBe('4242') + expect(caret.style.getPropertyValue('--caret-color')).toBeTruthy() + const label = caret.querySelector('.collaboration-carets__label') + expect(label?.textContent).toBe('Ada') + expect(caret.classList.contains(ACTIVE)).toBe(true) + }) + + it('falls back to a default name for a bare user state', () => { + const caret = renderCaret({ clientId: 1 }) + expect(caret.querySelector('.collaboration-carets__label')?.textContent).toBe('Collaborator') + }) + + it('hides the label after the inactivity hold, and re-activation restarts it', () => { + const caret = renderCaret({ name: 'Ada', color: '#f783ac', clientId: 4242 }) + vi.advanceTimersByTime(CARET_LABEL_HOLD_MS - 1) + expect(caret.classList.contains(ACTIVE)).toBe(true) + vi.advanceTimersByTime(1) + expect(caret.classList.contains(ACTIVE)).toBe(false) + + activateCaretLabel(caret) + expect(caret.classList.contains(ACTIVE)).toBe(true) + vi.advanceTimersByTime(CARET_LABEL_HOLD_MS) + expect(caret.classList.contains(ACTIVE)).toBe(false) + }) + + it('flips the label left only when it would overflow the editor right edge', () => { + const caret = renderCaret({ name: 'Ada', color: '#f783ac', clientId: 4242 }) + const label = caret.querySelector('.collaboration-carets__label') + if (!label) throw new Error('label missing') + // double-cast-allowed: jsdom has no layout; stub the label's right edge for the measure + label.getBoundingClientRect = () => ({ right: 500 }) as unknown as DOMRect + + activateCaretLabel(caret, 600) // editor edge past the label → no flip + expect(caret.classList.contains(FLIP)).toBe(false) + + activateCaretLabel(caret, 400) // editor edge before the label's right → flip + expect(caret.classList.contains(FLIP)).toBe(true) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts new file mode 100644 index 00000000000..eff0ce70534 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts @@ -0,0 +1,143 @@ +import { Extension } from '@tiptap/core' +import { Plugin, PluginKey } from '@tiptap/pm/state' +import type { Awareness } from 'y-protocols/awareness' + +/** + * Remote-collaborator caret presence for the file editor: the name label's + * show-then-fade behavior and the edge-aware flip, plus the ProseMirror plugin + * that drives them off awareness activity. + * + * Matches Google Docs: the name flag shows only while the peer has typed in the last + * few seconds or on local hover, then hides after inactivity, leaving just the caret. + */ + +/** + * How long a peer's name label stays visible after their last activity (cursor + * move / edit) before it fades, leaving just the colored caret bar. + */ +export const CARET_LABEL_HOLD_MS = 2000 + +/** Fallback caret color when a peer's awareness carries no `color`. */ +export const DEFAULT_CARET_COLOR = '#000000' + +/** + * The active-state class {@link activateCaretLabel} toggles on the caret node to reveal the + * name label; CSS transitions it back to hidden when removed. (yCursorPlugin reuses the DOM + * node and never re-runs `render`, which is why this file drives the class itself.) + */ +const CARET_ACTIVE_CLASS = 'collaboration-carets__caret--active' + +/** The class that flips the label to the caret's left near the editor's right edge. */ +const CARET_FLIP_CLASS = 'collaboration-carets__caret--flip' + +/** Per-caret fade timers, keyed by the (reused) caret DOM node. */ +const caretFadeTimers = new WeakMap>() + +/** + * Show a peer's name label, (re)start its fade timer, and flip it to the caret's + * left when it would run off the editor's right edge. + * + * yCursorPlugin renders each caret as a keyed widget decoration, so ProseMirror + * REUSES the same DOM node as the caret moves (verified: the render function is + * not re-invoked on a position change) — a CSS animation therefore cannot restart + * on activity. Instead the activity signal is the awareness `change` event, which + * (unlike `update`) fires only on a real state change, not the 15s heartbeat, so + * an idle peer's label correctly stays hidden. + */ +export function activateCaretLabel(caret: HTMLElement, editorRight?: number) { + caret.classList.add(CARET_ACTIVE_CLASS) + const existing = caretFadeTimers.get(caret) + if (existing) clearTimeout(existing) + caretFadeTimers.set( + caret, + setTimeout(() => { + caret.classList.remove(CARET_ACTIVE_CLASS) + caretFadeTimers.delete(caret) + }, CARET_LABEL_HOLD_MS) + ) + + if (editorRight === undefined) return + const label = caret.querySelector('.collaboration-carets__label') + if (!label) return + // Measure the default (rightward) position, then flip left only if it overflows. + caret.classList.remove(CARET_FLIP_CLASS) + if (label.getBoundingClientRect().right > editorRight) { + caret.classList.add(CARET_FLIP_CLASS) + } +} + +/** + * Builds a remote peer's caret DOM: a colored bar plus a name label tagged with + * the peer's Yjs client id (so {@link createCaretActivityExtension} can find and + * re-activate the reused node on later awareness changes). Shown immediately on + * (re)appearance; the fade timer hides it after inactivity. Passed to + * CollaborationCaret as its `render` option, which only supplies `user` — so the + * client id rides along in the awareness `user` payload (each client stamps its own + * `doc.clientID`; see `use-file-doc-collaboration.ts`). + */ +export function renderCaret(user: Record): HTMLElement { + const color = typeof user.color === 'string' ? user.color : DEFAULT_CARET_COLOR + const name = typeof user.name === 'string' && user.name ? user.name : 'Collaborator' + const clientId = typeof user.clientId === 'number' ? user.clientId : undefined + const caret = document.createElement('span') + caret.className = 'collaboration-carets__caret' + // One inline var drives the caret bar, the dormant cap, and the name tag (all in CSS). + caret.style.setProperty('--caret-color', color) + if (clientId !== undefined) caret.dataset.caretClientId = String(clientId) + const label = document.createElement('div') + label.className = 'collaboration-carets__label' + label.textContent = name + caret.appendChild(label) + activateCaretLabel(caret) + return caret +} + +/** + * Drives the caret name-label show-then-fade off awareness activity. Because the + * caret DOM node is reused across moves (see {@link activateCaretLabel}), the + * `render` function alone can't reveal the label when a peer moves — this listens + * for awareness `change` events and re-activates the matching caret node. Deferred + * to the next frame so the node exists and is laid out (for the edge-flip measure). + */ +export function createCaretActivityExtension(awareness: Awareness): Extension { + return Extension.create({ + name: 'collaborationCaretActivity', + addProseMirrorPlugins() { + return [ + new Plugin({ + key: new PluginKey('collaborationCaretActivity'), + view: (editorView) => { + // Coalesce bursts of awareness changes into a single rAF: at most one forced + // layout read + one flush per frame, however many peers moved. Accumulate the + // changed client ids, then re-activate each matching (reused) caret node. + let raf = 0 + const pending = new Set() + const flush = () => { + raf = 0 + const editorRight = editorView.dom.getBoundingClientRect().right + for (const id of pending) { + const caret = editorView.dom.querySelector( + `.collaboration-carets__caret[data-caret-client-id="${id}"]` + ) + if (caret) activateCaretLabel(caret, editorRight) + } + pending.clear() + } + const onChange = ({ added, updated }: { added: number[]; updated: number[] }) => { + for (const id of added) pending.add(id) + for (const id of updated) pending.add(id) + if (pending.size > 0 && !raf) raf = requestAnimationFrame(flush) + } + awareness.on('change', onChange) + return { + destroy: () => { + awareness.off('change', onChange) + if (raf) cancelAnimationFrame(raf) + }, + } + }, + }), + ] + }, + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx new file mode 100644 index 00000000000..13464e720e1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx @@ -0,0 +1,14 @@ +'use client' + +import { PresenceAvatars } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' +import { useFileDocOthers } from './file-doc-room-context' + +/** + * Avatar stack of the collaborators currently in the open file — the `useOthers` avatar + * stack, reading the room roster from {@link useFileDocOthers}. Renders nothing until + * someone else joins. Must sit inside a `FileDocRoomProvider`. + */ +export function FileDocAvatars() { + const others = useFileDocOthers() + return +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index 97fd5e13996..ee9ee61923a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -165,6 +165,26 @@ describe('FileDocProvider', () => { expect(messages.some((m) => m[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS)).toBe(true) }) + it('reseeds a cleared awareness so a reused instance can publish again', () => { + const { socket, emit } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + // Simulate a prior provider teardown having cleared the local state — after + // this, y-protocols' setLocalStateField is a permanent no-op, so the caret + // extension could never publish the local user/cursor on a reused instance. + awarenessProtocol.removeAwarenessStates(awareness, [doc.clientID], 'prior-destroy') + expect(awareness.getLocalState()).toBeNull() + + // Constructing a provider on the reused, cleared awareness must restore it. + new FileDocProvider(socket, 'file-1', doc, awareness) + expect(awareness.getLocalState()).not.toBeNull() + + emit.mockClear() + // The caret extension setting the user field must now actually publish. + awareness.setLocalStateField('user', { name: 'Ada', color: '#f783ac' }) + expect(emittedMessages(emit).some((m) => m[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS)).toBe(true) + }) + it('does not forward awareness it applied from the server', () => { const { emit, fire } = createProvider(true) emit.mockClear() diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index 5472ab31668..f0d24f9c23a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -69,6 +69,18 @@ export class FileDocProvider extends ObservableV2 { ) { super() + // Restore an empty local awareness state if it has been cleared. A fresh + // Awareness starts with `{}`, but a *reused* one whose local state was removed + // (a prior provider's `destroy()` clears it, and so does `Awareness.destroy()`) + // returns `null` here — and y-protocols' `setLocalStateField` is a no-op while + // the local state is `null`. The editor binds CollaborationCaret to this exact + // awareness for its whole life, so without this reseed a remount (e.g. React + // StrictMode's mount→unmount→mount, which re-runs the provider effect on the + // same instance) would leave the caret extension unable to ever publish the + // local user/cursor — remote peers would see no caret or selection, even though + // document sync (which does not depend on local awareness) keeps working. + if (awareness.getLocalState() === null) awareness.setLocalState({}) + socket.on(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) socket.on(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) socket.on(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context.tsx new file mode 100644 index 00000000000..2afc31ae7f1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context.tsx @@ -0,0 +1,43 @@ +'use client' + +import { createContext, type ReactNode, useContext, useState } from 'react' +import type { PresenceAvatarUser } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' + +const EMPTY_OTHERS: PresenceAvatarUser[] = [] +const noop = () => {} + +// Split into two contexts on purpose: the roster (`others`) changes on every join/leave, +// but the setter is stable. The editor (which owns the awareness) only ever *reports* the +// roster, so it subscribes to the setter context — which never changes identity — and never +// re-renders when the roster does; only the header avatar stack subscribes to `others`. +const FileDocOthersContext = createContext(EMPTY_OTHERS) +const FileDocSetOthersContext = createContext<(users: PresenceAvatarUser[]) => void>(noop) + +/** + * Scopes "who's in this file" presence to the open document — the `RoomProvider` + + * `useOthers` pattern (Liveblocks / y-presence) adapted to our component tree. The editor + * owns the Yjs awareness but sits *below* the file-detail header that renders the avatar + * stack, so it publishes the SERVER-AUTHENTICATED roster into this context + * ({@link useReportFileDocOthers}) and the header reads it ({@link useFileDocOthers}). + * Presence is ephemeral and room-scoped, so it lives in this provider, not a global store. + */ +export function FileDocRoomProvider({ children }: { children: ReactNode }) { + const [others, setOthers] = useState(EMPTY_OTHERS) + return ( + + {children} + + ) +} + +/** The roster of collaborators currently in the open file, for an avatar stack. Empty + * outside a {@link FileDocRoomProvider}. */ +export function useFileDocOthers(): PresenceAvatarUser[] { + return useContext(FileDocOthersContext) +} + +/** Publishes the server roster into the room context (editor side). Returns a stable no-op + * outside a {@link FileDocRoomProvider}. */ +export function useReportFileDocOthers(): (users: PresenceAvatarUser[]) => void { + return useContext(FileDocSetOthersContext) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts index 9fe069b56d0..475ea92fb2e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts @@ -1,11 +1,13 @@ 'use client' import { useEffect, useMemo, useRef, useState } from 'react' +import { FILE_DOC_EVENTS, type FileDocPresence } from '@sim/realtime-protocol/file-doc' import { Awareness } from 'y-protocols/awareness' import * as Y from 'yjs' import { getUserColor } from '@/lib/workspaces/colors' import { useSocket } from '@/app/workspace/providers/socket-provider' import { FileDocProvider } from './file-doc-provider' +import { useReportFileDocOthers } from './file-doc-room-context' /** The live collaboration binding the editor wires into TipTap's Collaboration * (the {@link Y.Doc}) and CollaborationCaret (the awareness). */ @@ -20,8 +22,13 @@ export interface FileDocCollaboration { * provider is consumed for seeding events (`synced` / `seed-request`). */ provider: FileDocProvider | null - /** Local caret identity for CollaborationCaret: display name + assigned color. */ - user: { name: string; color: string } + /** + * The local caret identity published to awareness: `name`/`color` for CollaborationCaret, + * and `clientId` so the caret activity extension can tag each caret node (see + * caret-presence.ts). The avatar roster does NOT come from here — it's server-authenticated + * (see the PRESENCE subscription below) so a peer can't spoof identity via awareness. + */ + user: { name: string; color: string; clientId: number | undefined } } interface UseFileDocCollaborationParams { @@ -57,26 +64,42 @@ export function useFileDocCollaboration({ // round-trip-unsafe views never build a Yjs document they won't use. const docRef = useRef(null) const awarenessRef = useRef(null) + // Created ONCE and kept stable for the whole editor lifetime. The editor freezes these into + // its extension set at mount (`useEditor` fixes extensions at creation), so the instances the + // provider binds MUST stay byte-identical to what the editor holds — never destroyed-and- + // recreated mid-life. If a StrictMode dev remount destroyed them, the still-mounted editor + // would keep the dead doc while the provider synced a fresh one, and a joining peer would see a + // blank document. Teardown is therefore deferred to a REAL unmount only (see below). if (enabled && docRef.current === null) { docRef.current = new Y.Doc() awarenessRef.current = new Awareness(docRef.current) } - const [provider, setProvider] = useState(null) - - // Declared BEFORE the provider effect so, on unmount, React runs this cleanup - // AFTER the provider effect's cleanup (cleanups run in reverse declaration - // order) — the provider detaches from the doc/awareness before they're destroyed. + // Destroy the doc + awareness on a REAL unmount only. `Awareness` runs a setInterval to expire + // stale peers, so it MUST be destroyed or that timer leaks for every file ever opened. But a + // StrictMode dev remount fires this cleanup and then re-runs the setup synchronously after — so + // we SCHEDULE the teardown and the remount cancels it, keeping the frozen instances alive. A + // genuine unmount has no remount, so the scheduled teardown runs on the next tick. + const pendingTeardownRef = useRef | null>(null) useEffect(() => { + if (pendingTeardownRef.current !== null) { + clearTimeout(pendingTeardownRef.current) + pendingTeardownRef.current = null + } return () => { - awarenessRef.current?.destroy() - docRef.current?.destroy() + pendingTeardownRef.current = setTimeout(() => { + awarenessRef.current?.destroy() + docRef.current?.destroy() + }, 0) } }, []) + const [provider, setProvider] = useState(null) + useEffect(() => { if (!enabled || !socket) return - // Non-null: both refs are lazily set during render, before any effect runs. + // Non-null: both refs are set during render before any effect runs, and are never destroyed + // (see above), so this always binds the same doc/awareness the editor froze at mount. const doc = docRef.current as Y.Doc const awareness = awarenessRef.current as Awareness const fileProvider = new FileDocProvider(socket, fileId, doc, awareness) @@ -87,7 +110,52 @@ export function useFileDocCollaboration({ } }, [enabled, socket, fileId]) - const user = useMemo(() => ({ name: userName, color: getUserColor(userId) }), [userName, userId]) + const reportOthers = useReportFileDocOthers() + const reportOthersRef = useRef(reportOthers) + reportOthersRef.current = reportOthers + + // "Who's in this file" roster (the useOthers side of the pattern). The server broadcasts a + // roster of SERVER-AUTHENTICATED identities (see FILE_DOC_EVENTS.PRESENCE) — trusted, + // unlike the client-set awareness `user` field a peer could spoof. Publish it (minus self) + // to the room context for the file-detail avatar stack; cleared on unmount so a file switch + // never shows the previous file's occupants. + useEffect(() => { + if (!enabled || !socket) return + const handlePresence = (data: FileDocPresence) => { + if (data.fileId !== fileId) return + // Exclude only our OWN socket (this session), NOT every session that shares our userId — + // so a second tab of the same account still counts as present, matching the canvas avatars + // (avatars.tsx filters by socketId). Then dedupe per user for the display stack, so + // multiple tabs of one person collapse to a single avatar. + const byUser = new Map< + string, + { userId: string; userName: string; avatarUrl: string | null } + >() + for (const peer of data.users) { + if (peer.socketId === socket.id || byUser.has(peer.userId)) continue + byUser.set(peer.userId, { + userId: peer.userId, + userName: peer.userName, + avatarUrl: peer.avatarUrl, + }) + } + reportOthersRef.current([...byUser.values()]) + } + socket.on(FILE_DOC_EVENTS.PRESENCE, handlePresence) + return () => { + socket.off(FILE_DOC_EVENTS.PRESENCE, handlePresence) + reportOthersRef.current([]) + } + }, [enabled, socket, fileId]) + + // The client id rides in the awareness `user` payload so the caret `render` (which only + // receives `user`) can tag each caret node for the activity-driven name label (see + // caret-presence.ts). `doc.clientID` is stable for the doc's life, so reading it from the + // ref needs no memo dep. + const user = useMemo( + () => ({ name: userName, color: getUserColor(userId), clientId: docRef.current?.clientID }), + [userName, userId] + ) return useMemo( () => diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts index 12190e62718..0e75964c427 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts @@ -8,6 +8,11 @@ import { withAlpha } from '@/lib/workspaces/colors' import { BlockMover } from './block-mover' import { CodeBlockWithLanguage } from './code-block' import { CodeBlockHighlight } from './code-highlight' +import { + createCaretActivityExtension, + DEFAULT_CARET_COLOR, + renderCaret, +} from './collaboration/caret-presence' import { LinkEmbed } from './embed/link-embed' import { createMarkdownContentExtensions } from './extensions' import { ResizableImage } from './image' @@ -64,19 +69,22 @@ export function createMarkdownEditorExtensions({ ? [ Collaboration.configure({ document: collaboration.doc }), // CollaborationCaret reads only `provider.awareness` (created synchronously, - // relayed by the socket provider once connected). The default caret + label - // color from `user.color`; only the selection tint needs an explicit override. + // relayed by the socket provider once connected). `render` tags each caret + // with the peer's client id and shows its name label; the selection tint is + // a translucent fill of the peer's identity color. CollaborationCaret.configure({ provider: { awareness: collaboration.awareness }, user: collaboration.user, + render: renderCaret, selectionRender: (user) => { - const hex = typeof user.color === 'string' ? user.color : '#000000' + const hex = typeof user.color === 'string' ? user.color : DEFAULT_CARET_COLOR return { class: 'collaboration-carets__selection', style: `background-color: ${withAlpha(hex, 0.2)};`, } }, }), + createCaretActivityExtension(collaboration.awareness), ] : []), CodeBlockHighlight, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css index 2ec9890d8be..ec33493934b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css @@ -355,12 +355,18 @@ border-radius: 4px; } +/* `overflow: visible` (not `hidden`) so a collaborator's caret name label, which + * pops above the caret, can escape the table box instead of being clipped by it. + * With `table-layout: fixed; width: 100%` columns can't exceed the table, so there + * is nothing to clip here anyway; the only absolutely-positioned descendant is the + * decorative column-resize handle, whose 2px right sliver is clipped by the editor's + * own scroll container. */ .rich-markdown-prose table { width: 100%; border-collapse: collapse; table-layout: fixed; margin: 1rem 0; - overflow: hidden; + overflow: visible; } .rich-markdown-prose th, @@ -456,11 +462,14 @@ } /* - * Collaborative carets (TipTap CollaborationCaret). The caret border and the name - * label's background are colored inline from each user's assigned color; the - * selection is a translucent tint of that color (set via selectionRender). The - * name label is hidden until the caret is hovered — a quiet, Google-Docs-style - * presence that doesn't clutter the text. + * Collaborative carets (TipTap CollaborationCaret). The caret bar and the name + * label's background are colored inline from each collaborator's identity color + * (the same `getUserColor` mechanism the canvas cursors use); the selection is a + * translucent tint of that color (set via selectionRender). The name label shows + * while the peer is active (JS toggles `--active` on each awareness change) or on + * hover, then fades after inactivity — matching Google Docs. `z-index` lifts the + * caret (and its label) above table cell backgrounds so a caret inside a table + * cell is not hidden behind adjacent cells. */ .rich-markdown-prose .collaboration-carets__caret { position: relative; @@ -468,42 +477,98 @@ margin-right: -1px; border-left-width: 1px; border-left-style: solid; + border-left-color: var(--caret-color); border-right-width: 1px; border-right-style: solid; + border-right-color: var(--caret-color); word-break: normal; + z-index: 20; +} + +/* Dormant affordance: a small cap at the top of the caret in the collaborator's color, + * shaped like a collapsed presence name tag (same `rounded-xs` + notch corner as the + * tables/canvas tags) so the whole presence system reads as one language. It signals + * "someone's here — hover for who", and fades out when the full name tag takes over + * (peer active or on hover). */ +.rich-markdown-prose .collaboration-carets__caret::before { + content: ""; + position: absolute; + /* Seat the cap's square bottom-left corner on the pole's top-left, flush like a flag on its + * pole. `left: -1px` backs out the caret's 1px left border (the abs-positioning origin is the + * padding box, inside that border) so the cap's left edge lines up with the pole's left edge; + * `top` overlaps the pole's top a hair so the notch reads as continuous, no gap. */ + top: -2px; + left: -1px; + width: 8px; + height: 5px; + border-radius: 2px 2px 2px 0; + background-color: var(--caret-color); + transition: opacity 0.2s ease; +} + +.rich-markdown-prose .collaboration-carets__caret--active::before, +.rich-markdown-prose .collaboration-carets__caret:hover::before { + opacity: 0; +} + +/* Transparent hover hit-slop. The visible caret bar is only ~2px wide and the dormant cap is + * 8×5px, so "hover for who" (revealing the name label on a dormant caret) would be near + * impossible to trigger. This invisible strip widens the hover target across the caret's full + * height and over the cap — nothing visible changes, only the pointer area. `pointer-events: + * auto` is required so it catches the hover; kept narrow so it barely intrudes on selecting + * text next to a remote caret. */ +.rich-markdown-prose .collaboration-carets__caret::after { + content: ""; + position: absolute; + top: -4px; + bottom: 0; + left: -4px; + right: -4px; + pointer-events: auto; } +/* Matches the canvas cursor name tag (cursors.tsx): identity-color background with + * `--surface-1` text at `text-xs`/`font-medium`, so both presence surfaces read as + * one system. `--surface-1` is the base surface token (readable on every assigned + * identity color in both themes), not a hardcoded value. Hidden by default; the + * show/fade is driven by the `--active` class (see caret-presence.ts) and hover. */ .rich-markdown-prose .collaboration-carets__label { position: absolute; top: -1.4em; left: -1px; + max-width: 10rem; + overflow: hidden; padding: 0.1rem 0.35rem; - border-radius: 3px 3px 3px 0; + border-radius: 2px 2px 2px 0; + /* 11px = the `text-xs` the canvas/tables presence tags use, for pixel parity. */ font-size: 11px; - font-weight: 600; + font-weight: 500; line-height: 1.2; white-space: nowrap; - /* Fixed dark text: the label background is always a light user color (assigned - * inline, theme-independent), so a theme token would go unreadable in one mode. */ - color: #1a1a1a; + text-overflow: ellipsis; + background-color: var(--caret-color); + color: var(--surface-1); user-select: none; pointer-events: none; opacity: 0; - transition: opacity 0.12s ease; -} - -/* Widen the caret's hover target beyond its 1px width so the name label is easy to - * reveal; the label itself stays pointer-events:none and never intercepts clicks. */ -.rich-markdown-prose .collaboration-carets__caret::before { - content: ""; - position: absolute; - inset: -0.1em -2px; + transition: opacity 0.2s ease; } +.rich-markdown-prose .collaboration-carets__caret--active .collaboration-carets__label, .rich-markdown-prose .collaboration-carets__caret:hover .collaboration-carets__label { opacity: 1; } +/* Near the editor's right edge the label is flipped to the caret's left so it never + * runs off (JS toggles `--flip` after measuring); mirror the tag's notch corner. */ +.rich-markdown-prose .collaboration-carets__caret--flip .collaboration-carets__label { + left: auto; + right: -1px; + border-radius: 2px 2px 0 2px; +} + +/* Remote text selection: a rounded translucent tint of the collaborator's identity + * color (the alpha fill is set inline by selectionRender). */ .rich-markdown-prose .collaboration-carets__selection { border-radius: 2px; pointer-events: none; diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 29531265471..8094080b93d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -61,7 +61,6 @@ import { Resource, timeCell, } from '@/app/workspace/[workspaceId]/components' -import { PresenceAvatars } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' import { FilesActionBar } from '@/app/workspace/[workspaceId]/files/components/action-bar' import { DeleteConfirmModal } from '@/app/workspace/[workspaceId]/files/components/delete-confirm-modal' import { FileRowContextMenu } from '@/app/workspace/[workspaceId]/files/components/file-row-context-menu' @@ -73,6 +72,8 @@ import { isPreviewable, isTextEditable, } from '@/app/workspace/[workspaceId]/files/components/file-viewer' +import { FileDocAvatars } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars' +import { FileDocRoomProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context' import { FilesListContextMenu } from '@/app/workspace/[workspaceId]/files/components/files-list-context-menu' import { ShareModal } from '@/app/workspace/[workspaceId]/files/components/share-modal' import { useWorkspaceFilesRoom } from '@/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room' @@ -205,10 +206,10 @@ export function Files() { const canEdit = userPermissions.canEdit === true const { config: permissionConfig } = usePermissionConfig() - const { otherUsers: filesPresenceUsers } = useWorkspaceFilesRoom( - workspaceId, - currentFolderId ?? null - ) + // Joined for the live file tree: a `workspace-files-changed` broadcast invalidates the + // browser. "Who's in this file" comes from the file-doc room (see FileDocRoomProvider), + // not from who's browsing the Files section. + useWorkspaceFilesRoom(workspaceId) useEffect(() => { if (permissionConfig.hideFilesTab) { @@ -1907,36 +1908,42 @@ export function Files() { if (selectedFile) { return ( <> - - - + {/* The room provider scopes "who's in this file" presence to the open document: the + editor (inside FileViewer) publishes the server-authenticated roster and the + header's FileDocAvatars reads it — both must be descendants. */} + + + } + /> + - - + + + } /> ([]) - - const tabSessionIdRef = useRef('') - if (!tabSessionIdRef.current) tabSessionIdRef.current = generateShortId() - const folderIdRef = useRef(folderId) - - useEffect(() => { - folderIdRef.current = folderId - }, [folderId]) - useEffect(() => { if (!socket || !workspaceId) return let retries = 0 let retryTimer: ReturnType | null = null - const join = () => { - socket.emit('join-workspace-files', { - workspaceId, - folderId: folderIdRef.current, - tabSessionId: tabSessionIdRef.current, - }) - } + const join = () => socket.emit('join-workspace-files', { workspaceId }) - const handleJoinSuccess = (data: { - workspaceId: string - presenceUsers: PresenceUpdatePayload[] - }) => { + const handleJoinSuccess = (data: { workspaceId: string }) => { if (data.workspaceId !== workspaceId) return retries = 0 - // Cancel any retry scheduled by a prior retryable error so it can't fire an - // extra join after we're already in. + // Cancel any retry scheduled by a prior retryable error so it can't fire an extra + // join after we're already in. if (retryTimer) { clearTimeout(retryTimer) retryTimer = null } - setPresenceUsers(data.presenceUsers ?? []) } const handleJoinError = (data: JoinErrorPayload) => { if (data.workspaceId !== workspaceId) return @@ -93,7 +58,6 @@ export function useWorkspaceFilesRoom( retryTimer = setTimeout(join, JOIN_RETRY_BASE_MS * retries) } } - const handlePresence = (users: PresenceUpdatePayload[]) => setPresenceUsers(users ?? []) const handleChanged = (data: { workspaceId: string }) => { if (data.workspaceId === workspaceId) invalidateWorkspaceFileBrowsers(queryClient, workspaceId) @@ -104,7 +68,6 @@ export function useWorkspaceFilesRoom( socket.on('connect', join) socket.on('join-workspace-files-success', handleJoinSuccess) socket.on('join-workspace-files-error', handleJoinError) - socket.on('workspace-files:presence-update', handlePresence) socket.on('workspace-files-changed', handleChanged) return () => { @@ -112,22 +75,12 @@ export function useWorkspaceFilesRoom( socket.off('connect', join) socket.off('join-workspace-files-success', handleJoinSuccess) socket.off('join-workspace-files-error', handleJoinError) - socket.off('workspace-files:presence-update', handlePresence) socket.off('workspace-files-changed', handleChanged) - setPresenceUsers([]) - // Leave the room, scoped to THIS workspace: the server no-ops if the socket - // has already switched to another workspace's files room (so a workspace - // A→B switch, where B's join runs first and auto-leaves A, can't have A's - // leave evict the fresh B membership). + // Leave the room, scoped to THIS workspace: the server no-ops if the socket has + // already switched to another workspace's files room (so a workspace A→B switch, + // where B's join runs first and auto-leaves A, can't have A's leave evict B). socket.emit('leave-workspace-files', { workspaceId }) } }, [socket, workspaceId, queryClient]) - - const otherUsers = useMemo( - () => presenceUsers.filter((user) => user.socketId !== currentSocketId), - [presenceUsers, currentSocketId] - ) - - return { otherUsers } } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx index 74ed697f76a..5461082f3de 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx @@ -1,7 +1,12 @@ 'use client' import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' +import { createPortal } from 'react-dom' import { getUserColor, withAlpha } from '@/lib/workspaces/colors' +import { + isCellInSelection, + type NormalizedSelection, +} from '@/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/utils' import type { RemoteTableSelection } from '@/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room' /** A measured remote selection, positioned in the grid content wrapper's space. */ @@ -14,12 +19,26 @@ interface SelectionBox { left: number width: number height: number + /** Viewport-space top/left of the selection, for the body-portaled name label. */ + viewportTop: number + viewportLeft: number + /** Resolved anchor/focus cell indices (undefined when off-window). Coverage by the local + * selection is derived from these in render (see {@link isSelectionCovered}) — no DOM + * re-measure when only the local caret moves. */ + anchorRow: number | undefined + anchorCol: number | undefined + focusRow: number | undefined + focusCol: number | undefined } interface RemoteSelectionOverlayProps { remoteSelections: RemoteTableSelection[] /** Column id → its rendered column index (matches the cells' `data-col`). */ columnIndexById: Map + /** Row id → its index in the current row list, to test local-selection coverage. */ + rowIndexById: Map + /** The local user's own normalized selection, so a co-selected remote cell defers to it. */ + localSelection: NormalizedSelection | null /** The grid's scroll container (`data-table-scroll`), queried for cell rects. */ scrollElement: HTMLElement | null } @@ -39,6 +58,24 @@ function cellRect( return cell?.getBoundingClientRect() } +/** + * Whether a remote selection defers to the local one: true only when the local selection + * fully contains it (both corners inside), so a partial overlap still shows. + */ +function isSelectionCovered( + anchorRow: number | undefined, + anchorCol: number | undefined, + focusRow: number | undefined, + focusCol: number | undefined, + bounds: NormalizedSelection | null +): boolean { + return ( + bounds !== null && + isCellInSelection(anchorRow, anchorCol, bounds) && + isCellInSelection(focusRow, focusCol, bounds) + ) +} + /** * Renders remote collaborators' cell selections over the table grid — a colored * border per user (Google-Sheets style), a darker fill while they are editing, and @@ -54,6 +91,8 @@ function cellRect( export function RemoteSelectionOverlay({ remoteSelections, columnIndexById, + rowIndexById, + localSelection, scrollElement, }: RemoteSelectionOverlayProps) { const rootRef = useRef(null) @@ -68,6 +107,11 @@ export function RemoteSelectionOverlay({ remoteSelectionsRef.current = remoteSelections const columnIndexByIdRef = useRef(columnIndexById) columnIndexByIdRef.current = columnIndexById + const rowIndexByIdRef = useRef(rowIndexById) + rowIndexByIdRef.current = rowIndexById + // Read only by the pointer hit-test (never in render) to skip a locally-covered box. + const localSelectionRef = useRef(localSelection) + localSelectionRef.current = localSelection // Cached content-wrapper origin, refreshed on each measure (scroll/resize/data change), // so the pointer hit-test never forces a layout read per mouse move. const originRef = useRef({ top: 0, left: 0 }) @@ -81,14 +125,20 @@ export function RemoteSelectionOverlay({ const next: SelectionBox[] = [] for (const selection of remoteSelectionsRef.current) { const { anchor, focus, editing } = selection.cell + const anchorCol = columnIndexByIdRef.current.get(anchor.columnId) + const focusCol = columnIndexByIdRef.current.get(focus.columnId) + const anchorRow = rowIndexByIdRef.current.get(anchor.rowId) + const focusRow = rowIndexByIdRef.current.get(focus.rowId) const rects = [ - cellRect(scrollEl, anchor.rowId, columnIndexByIdRef.current.get(anchor.columnId)), - cellRect(scrollEl, focus.rowId, columnIndexByIdRef.current.get(focus.columnId)), + cellRect(scrollEl, anchor.rowId, anchorCol), + cellRect(scrollEl, focus.rowId, focusCol), ].filter((rect): rect is DOMRect => rect !== undefined) if (rects.length === 0) continue - const top = Math.min(...rects.map((r) => r.top)) - origin.top - const left = Math.min(...rects.map((r) => r.left)) - origin.left + const viewportTop = Math.min(...rects.map((r) => r.top)) + const viewportLeft = Math.min(...rects.map((r) => r.left)) + const top = viewportTop - origin.top + const left = viewportLeft - origin.left const bottom = Math.max(...rects.map((r) => r.bottom)) - origin.top const right = Math.max(...rects.map((r) => r.right)) - origin.left next.push({ @@ -100,6 +150,12 @@ export function RemoteSelectionOverlay({ left, width: right - left, height: bottom - top, + viewportTop, + viewportLeft, + anchorRow, + anchorCol, + focusRow, + focusCol, }) } setBoxes(next) @@ -124,7 +180,20 @@ export function RemoteSelectionOverlay({ const x = event.clientX - left const y = event.clientY - top const hit = boxesRef.current.find( - (b) => x >= b.left && x <= b.left + b.width && y >= b.top && y <= b.top + b.height + (b) => + x >= b.left && + x <= b.left + b.width && + y >= b.top && + y <= b.top + b.height && + // Skip a box the local selection covers — it isn't drawn, so hovering it must not + // pop a name tag over a cell with no visible remote selection. + !isSelectionCovered( + b.anchorRow, + b.anchorCol, + b.focusRow, + b.focusCol, + localSelectionRef.current + ) ) setHoveredSocketId((prev) => prev === (hit?.socketId ?? null) ? prev : (hit?.socketId ?? null) @@ -160,37 +229,84 @@ export function RemoteSelectionOverlay({ } }, [scrollElement, measure]) - // Re-measure when the selections or column layout change (listeners stay subscribed). - // Layout effect so positions update before paint — no one-frame lag as a peer moves. + // Re-measure when the remote selections or column layout change (listeners stay + // subscribed). Layout effect so positions update before paint — no one-frame lag as a + // peer moves. NOT keyed on `localSelection`: moving the local caret changes only which + // boxes are `covered`, which the cheap in-memory pass below handles without a reflow. useLayoutEffect(() => { measure() }, [remoteSelections, columnIndexById, measure]) + // Re-derived in render so it reacts to `localSelection`: when the local selection grows to + // cover the hovered box (its outline is no longer drawn) without another pointer move, the + // floating name tag must drop rather than linger over cells with no visible remote selection. + const hoveredBox = hoveredSocketId + ? boxes.find( + (box) => + box.socketId === hoveredSocketId && + !isSelectionCovered( + box.anchorRow, + box.anchorCol, + box.focusRow, + box.focusCol, + localSelection + ) + ) + : undefined + return ( -
- {boxes.map((box) => ( -
- {hoveredSocketId === box.socketId && ( - - {box.userName} - - )} -
- ))} -
+ <> +
+ {boxes.map((box) => + // A cell the local user also has selected shows only the local selection — the + // remote box isn't drawn (its `boxes` entry still drives the hover name). The + // border is an inset box-shadow (no layout width, so it never stacks with an + // adjacent cell's border) plus a subtle fill, darker while the peer is editing. + isSelectionCovered( + box.anchorRow, + box.anchorCol, + box.focusRow, + box.focusCol, + localSelection + ) ? null : ( +
+ ) + )} +
+ {/* The name label portals to the body so it floats on top of the grid (and its + sticky header) instead of being clipped by the overlay's overflow-hidden; it's + placed in viewport space, its bottom-left tabbed onto the selection's top-left. */} + {hoveredBox && + createPortal( + // Same chrome as the workflow-canvas presence label (see cursors.tsx): the + // identity color is the only per-user value; text color, font, radius, and + // padding all reuse the canvas tokens so tables + canvas presence look identical. + // `rounded-bl-none` tabs the label's bottom-left corner onto the selection's + // top-left, the one deviation from the free-floating canvas cursor tag. +
+ {hoveredBox.userName} +
, + document.body + )} + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index 8ae1abd1df7..821018b6692 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -62,6 +62,7 @@ import { computeNormalizedSelection, type ExecStatusMix, expandToDisplayColumns, + isCellInSelection, moveCell, ROW_SELECTION_ALL, ROW_SELECTION_NONE, @@ -663,6 +664,15 @@ export function TableGrid({ return map }, [displayColumns]) + /** Row id → its index in the current row list, for testing local-selection coverage. + * Only built when collaborators are present (the overlay is gated on that too), so + * solo editing never pays the O(n) map build on a refetch. */ + const rowIndexById = useMemo(() => { + const map = new Map() + if (remoteSelections.length > 0) rows.forEach((row, index) => map.set(row.id, index)) + return map + }, [rows, remoteSelections.length]) + const workflowGroupById = useMemo( () => new Map(tableWorkflowGroups.map((g) => [g.id, g])), [tableWorkflowGroups] @@ -1211,12 +1221,7 @@ export function TableGrid({ selectionAnchorRef.current, selectionFocusRef.current ) - const isWithinSelection = - sel !== null && - rowIndex >= sel.startRow && - rowIndex <= sel.endRow && - colIndex >= sel.startCol && - colIndex <= sel.endCol + const isWithinSelection = sel !== null && isCellInSelection(rowIndex, colIndex, sel) if (!isWithinSelection) { setSelectionAnchor({ rowIndex, colIndex }) @@ -3943,6 +3948,8 @@ export function TableGrid({ )} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/utils.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/utils.ts index da0e5674751..788d11f6579 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/utils.ts @@ -83,6 +83,25 @@ export interface NormalizedSelection { anchorCol: number } +/** + * Whether a (row, col) index pair falls inside a normalized selection rectangle. + * Row/col may be `undefined` (e.g. an id that didn't resolve to an index) → `false`. + */ +export function isCellInSelection( + row: number | undefined, + col: number | undefined, + sel: NormalizedSelection +): boolean { + return ( + row !== undefined && + col !== undefined && + row >= sel.startRow && + row <= sel.endRow && + col >= sel.startCol && + col <= sel.endCol + ) +} + /** A run of consecutive `displayColumns` rendered together in the meta header row. */ export type HeaderGroup = | { kind: 'plain'; size: 1; startColIndex: number } diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index 738af5bc123..9d2fbe3a5db 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -32,6 +32,13 @@ export const FILE_DOC_EVENTS = { LEAVE: 'leave-file-doc', /** Both directions: a framed Yjs message (binary), tagged by {@link FILE_DOC_MESSAGE_TYPE}. */ MESSAGE: 'file-doc-message', + /** + * Server → client: the roster of collaborators currently in the document + * ({@link FileDocPresence}), for the avatar stack. Identity is server-authenticated (from + * each socket's session), so — unlike the client-set awareness `user` field — a peer + * cannot spoof or suppress another user's entry. Re-sent on every join and leave. + */ + PRESENCE: 'file-doc-presence', } as const /** @@ -121,6 +128,25 @@ export interface LeaveFileDocPayload { fileId: string } +/** One collaborator session in a {@link FileDocPresence} roster — server-authenticated identity. + * Keyed per socket (session), not per user: the client excludes its OWN `socketId` and then + * dedupes the rest per user for the avatar stack, so a second tab of the same account still + * registers as present (mirroring the canvas presence model). */ +export interface FileDocPresenceUser { + /** The collaborator's socket id, so a client can exclude its own session (not every session + * that happens to share its userId). */ + socketId: string + userId: string + userName: string + avatarUrl: string | null +} + +/** Server → client roster of who is in the document ({@link FILE_DOC_EVENTS.PRESENCE}). */ +export interface FileDocPresence { + fileId: string + users: FileDocPresenceUser[] +} + /** * Coerce a Socket.IO binary payload to a `Uint8Array`, or `null` if it is * neither. Shared by the server relay and the client provider so the two agree From cddcb736b80a386341b8699adb2a34985c8487ab Mon Sep 17 00:00:00 2001 From: mzxchandra <129460234+mzxchandra@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:35:01 -0700 Subject: [PATCH 10/53] feat(files): smarter bullet delete/indent and untitled-file title sync (#5971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * improvement(files): smarter bullet delete/indent, fix empty-nested-bullet heading corruption Backspace at the start of a list item now outdents a nested item or clears a top-level item to a paragraph in place instead of deleting the row and jumping the caret to the previous block; Enter on an empty nested item outdents. Empty non-trailing top-level items still collapse cleanly since they cannot round-trip as a lifted paragraph. Also strips nested empty list-item marker lines on serialize: a nested empty bullet re-parsed as a Setext heading underline, silently turning its parent line into an H2 and dropping the bullet. Top-level empty items are preserved. * feat(files): sync an untitled file's name with its leading heading While a file is still named untitled(.md), typing a leading heading auto-renames the file after it (debounced), and renaming the file first seeds a leading H1 from the new name. One-shot: coupling stops once the file has a real name, and the heading seed always prepends so existing content is never clobbered. * fix(files): count inline atoms in list-item emptiness, keep multi-block items on Backspace Addresses review findings on the list Backspace logic: - Emptiness now uses the caret block's content.size (counts inline images/mentions), not textContent, so a bullet holding only a non-text atom is no longer treated as empty and deleted. - An empty first block whose item has sibling blocks removes only that block instead of lifting the whole item out of the list. * fix(files): preserve the untitled to named heading seed across a rename during editor load The parent captures the file name at mount (before content/session finish loading) and passes it as the transition baseline, so a rename that lands in the loading window is still seen as an untitled to named transition and the leading heading seed is not skipped. * fix(files): drop the name-to-heading seed, keep title sync one-way Removes the effect that inserted a leading H1 when an untitled file was renamed. On the collaborative Files page every open client observed the untitled-to-named transition and inserted into the shared doc, producing duplicate headings; it could also re-insert a heading a user had just deleted while a rename was in flight. Seeding document content from an async rename transition is the wrong model on a shared editor. The primary direction — typing a leading heading renames a still-untitled file — is unaffected (it never mutates the doc). * fix(files): keep empty lines between paragraphs on reload The chunked markdown parser (parseMarkdownToDoc) parses each block stripped of the blank lines between them, so it dropped the empty paragraphs @tiptap/markdown builds from runs of blank lines — a saved visual blank line silently vanished on the next load (the settle/reopen re-seed goes through the chunker). The whole-document parser preserves them, but whether a gap yields an empty paragraph is a global, block-type- dependent decision (kept between two paragraphs, dropped after a heading), so it can't be reconstructed block-locally. Route documents with empty-paragraph blank-line spacing to the whole-document parser for exact fidelity — the same tradeoff NON_CHUNKABLE makes; ordinary single-blank-line separation still takes the fast chunked path. Adds a suite asserting chunked output matches the whole-document parser for leading/trailing/between gaps and around lists/headings. * fix(files): only auto-name an untitled file when the user can edit The debounced untitled→filename hook ran on every onUpdate — including the mount-time seed and for view-only viewers — without checking edit permission, so a read-only user could schedule a rename they have no permission to make (a spurious, server-rejected write). Gate the derive-title on editor.isEditable (canEdit + settled + collab-ready, the same signal the autosave path uses), at both schedule and fire time. * fix(files): normalize line endings before the empty-paragraph guard; Enter/Backspace symmetry - markdown-parse: EMPTY_PARAGRAPH_SPACING/NON_CHUNKABLE tested the raw body, but a classic \r-only file (blank lines are \r) would miss the \n-anchored guard and still be chunked, dropping empties. Normalize line endings once up front so the routing guards, the chunker, and the parser all see the same \n. +CRLF/CR test cases. - keymap: Enter on an empty first block of a multi-block item now removes only that block (removeEmptyWrappedBlock) instead of exiting the list, mirroring the Backspace hasSiblingBlocks case — the trailing check no longer swallows multi-block items. +test. * fix(files): editor audit follow-ups (trailing-blank read-only, collab rename, over-strip) A 4-agent independent audit (UX vs inkeep + SOTA, cleanliness, adversarial correctness) surfaced these: - HIGH regression: files ending in a blank line opened READ-ONLY. The empty-paragraph routing preserved a TRAILING empty paragraph, but postProcess collapses trailing newlines → serialize/parse non-idempotent → isRoundTripSafe flipped the file read-only. A trailing empty paragraph can't be serialized stably, so parseMarkdownToDoc now strips trailing empty paragraphs and the guard no longer routes on trailing blanks. Interior/leading empties are unaffected. +regression tests. - Medium: the debounced untitled→filename rename fired on remote Yjs edits too, so every peer renamed and could rename from a not-yet-synced heading. Gate on isChangeOrigin (local edits only; false for non-collab surfaces). - Medium: stripEmptyListItemLines over-stripped a nested empty item that follows a same-indent sibling (a real placeholder the parser keeps). Narrowed to the actual Setext hazard — an empty item DIRECTLY under a shallower parent line — matching the function's own docstring intent. Probe-verified. +test. - Low: corrected untitled-title.ts docstring that described a reverse name→heading coupling removed during review. * fix(files): a remote edit must not cancel the local rename debounce The isChangeOrigin gate cleared the debounce timer BEFORE bailing on a remote update, so a peer's edit arriving within the 600ms window cancelled the local user's pending rename. Bail on isChangeOrigin first, before touching the timer; only local edits clear/reschedule it. * docs(files): correct EMPTY_PARAGRAPH_SPACING rationale after trailing-strip The stacked trailing-empty-paragraph strip made the older comment overstate a correctness necessity it no longer owns, mislabel trailing runs of 2+ blanks, and advertise dead CRLF handling. Reword to match what the code actually does. --------- Co-authored-by: Waleed Latif --- .../components/file-viewer/file-viewer.tsx | 7 + .../rich-markdown-editor/keymap.test.ts | 276 ++++++++++++++++-- .../rich-markdown-editor/keymap.ts | 117 ++++++-- .../markdown-fidelity.test.ts | 64 ++++ .../rich-markdown-editor/markdown-fidelity.ts | 80 ++++- .../markdown-parse.test.ts | 73 ++++- .../rich-markdown-editor/markdown-parse.ts | 69 ++++- .../rich-markdown-editor.tsx | 61 +++- .../title-heading.test.ts | 63 ++++ .../rich-markdown-editor/title-heading.ts | 9 + .../workspace/[workspaceId]/files/files.tsx | 42 ++- .../files/untitled-title.test.ts | 77 +++++ .../[workspaceId]/files/untitled-title.ts | 57 ++++ 13 files changed, 930 insertions(+), 65 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/untitled-title.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index b9bc4b4a302..bba37374240 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -116,6 +116,11 @@ interface FileViewerProps { * never target one editor. See {@link RichMarkdownEditorProps.collaborative}. */ collaborative?: boolean + /** + * Called (debounced) with the markdown document's leading-heading text while the file is still + * untitled, so the caller can name the file after it. Only wired for the editable markdown editor. + */ + onDeriveTitleFromHeading?: (headingText: string) => void } export function FileViewer(props: FileViewerProps) { @@ -148,6 +153,7 @@ function FileViewerContent({ disableStreamingAutoScroll = false, previewContextKey, collaborative, + onDeriveTitleFromHeading, }: FileViewerProps) { const category = resolveFileCategory(file.type, file.name) @@ -193,6 +199,7 @@ function FileViewerContent({ disableStreamingAutoScroll={disableStreamingAutoScroll} previewContextKey={previewContextKey} collaborative={collaborative} + onDeriveTitleFromHeading={onDeriveTitleFromHeading} /> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.test.ts index 586f2bc2ff2..9c2ba8f3b70 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.test.ts @@ -11,6 +11,7 @@ import { GapCursor } from '@tiptap/pm/gapcursor' import { AllSelection, NodeSelection } from '@tiptap/pm/state' import { beforeEach, describe, expect, it, vi } from 'vitest' import { createMarkdownEditorExtensions } from './editor-extensions' +import { postProcessSerializedMarkdown } from './markdown-fidelity' import { MENTION_PLUGIN_KEY } from './mention' import { SLASH_COMMAND_PLUGIN_KEY } from './slash-command/slash-command' @@ -192,13 +193,11 @@ describe('empty wrapped-block Backspace', () => { it.each([ ['bullet middle', '- one\n- two\n- three', 'two', '- one\n- three'], ['bullet first', '- one\n- two\n- three', 'one', '- two\n- three'], - ['bullet last', '- one\n- two', 'two', '- one'], ['ordered middle', '1. one\n2. two\n3. three', 'two', '1. one\n2. three'], ['task middle', '- [ ] one\n- [ ] two\n- [ ] three', 'two', '- [ ] one\n- [ ] three'], ['blockquote middle', '> one\n>\n> two\n>\n> three', 'two', '> one\n>\n> three'], - ['nested item', '- one\n - two\n- three', 'two', '- one\n- three'], ])( - 'removes the emptied %s cleanly — one container, no stray paragraph, round-trips', + 'removes an emptied non-trailing %s cleanly — one container, no stray paragraph, round-trips', (_label, markdown, word, expected) => { const editor = editorWith('') editor.commands.setContent(markdown, { contentType: 'markdown' }) @@ -213,10 +212,10 @@ describe('empty wrapped-block Backspace', () => { } ) - it('leaves a gap cursor — never a NodeSelection — when the removed bullet was followed by an image at doc start', () => { - // Regression: `Selection.near` after the delete silently NodeSelected the following image, so a - // second Backspace while "clearing the bullet" deleted the image (and typing would have replaced - // it). The selection left behind must never make the next keystroke destructive. + it('clears a lone empty bullet before an image to a paragraph and never destroys the image', () => { + // Regression: an earlier delete-and-jump path silently NodeSelected the following image, so a + // second Backspace while "clearing the bullet" deleted it. The lone empty bullet now lifts into an + // empty paragraph in place, the image is untouched, and no repeated Backspace destroys it. const editor = editorWith({ type: 'doc', content: [ @@ -227,11 +226,11 @@ describe('empty wrapped-block Backspace', () => { editor.commands.setTextSelection(3) pressBackspace(editor) - expect(blockShape(editor)).toEqual(['image', 'paragraph']) + expect(blockShape(editor)).toEqual(['paragraph', 'image', 'paragraph']) expect(editor.state.selection).not.toBeInstanceOf(NodeSelection) pressBackspace(editor) - expect(blockShape(editor)).toEqual(['image', 'paragraph']) + expect(blockShape(editor)).toContain('image') editor.destroy() }) @@ -252,11 +251,10 @@ describe('empty wrapped-block Backspace', () => { editor.destroy() }) - it('never NodeSelects a leaf BEFORE the removed bullet either (findFrom textOnly skips atoms)', () => { - // `Selection.findFrom($gap, -1, true)` cannot return a NodeSelection: with textOnly, - // prosemirror-state's findSelectionIn skips atoms entirely (`!text && isSelectable`). With an - // image directly before the emptied bullet and no textblock behind it, the backward search - // returns null and the gap-cursor branch takes over — the image is never silently selected. + it('clears a lone empty bullet after an image to a paragraph without selecting the image', () => { + // With an image directly before the emptied bullet, clearing the bullet must not silently select + // (and so endanger) the image. The bullet lifts into an empty paragraph in place; the image is + // untouched and no repeated Backspace destroys it. const editor = editorWith({ type: 'doc', content: [ @@ -268,11 +266,11 @@ describe('empty wrapped-block Backspace', () => { expect(editor.state.selection.$from.parent.type.name).toBe('paragraph') pressBackspace(editor) - expect(blockShape(editor)).toEqual(['image', 'paragraph']) + expect(blockShape(editor)).toEqual(['image', 'paragraph', 'paragraph']) expect(editor.state.selection).not.toBeInstanceOf(NodeSelection) pressBackspace(editor) - expect(blockShape(editor)).toEqual(['image', 'paragraph']) + expect(blockShape(editor)).toContain('image') editor.destroy() }) @@ -291,7 +289,7 @@ describe('empty wrapped-block Backspace', () => { editor.destroy() }) - it('still prefers the previous textblock caret when one exists (image after the bullet untouched)', () => { + it('clears a lone empty bullet between a paragraph and an image to a paragraph, leaving the image', () => { const editor = editorWith({ type: 'doc', content: [ @@ -303,10 +301,204 @@ describe('empty wrapped-block Backspace', () => { editor.commands.setTextSelection(10) pressBackspace(editor) - expect(blockShape(editor)).toEqual(['paragraph', 'image', 'paragraph']) + expect(blockShape(editor)).toEqual(['paragraph', 'paragraph', 'image', 'paragraph']) expect(editor.state.selection.empty).toBe(true) expect(editor.state.selection).not.toBeInstanceOf(NodeSelection) - expect(editor.state.selection.$from.parent.textContent).toBe('hello') + // The cleared bullet is now an empty paragraph, and 'hello' is untouched above it. + expect(editor.state.doc.firstChild?.textContent).toBe('hello') + editor.destroy() + }) +}) + +describe('list Backspace (clear / outdent)', () => { + beforeEach(() => { + Element.prototype.scrollIntoView = vi.fn() + }) + + /** Puts the caret at the very start of the item text `word`. */ + function caretAtStartOf(editor: Editor, word: string): void { + editor.state.doc.descendants((node, pos) => { + if (node.isText && node.text === word) editor.commands.setTextSelection(pos) + }) + } + + it('lifts a top-level bullet WITH TEXT into a paragraph, keeping the text (round-trips)', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n- two', { contentType: 'markdown' }) + editor.commands.focus() + caretAtStartOf(editor, 'two') + pressBackspace(editor) + + // A bulletList (just 'one') followed by the lifted 'two' paragraph (+ TipTap's trailing filler). + expect(blockShape(editor).slice(0, 2)).toEqual(['bulletList', 'paragraph']) + const { md, reparsed } = markdownRoundTrip(editor) + expect(md.trim()).toBe('- one\n\ntwo') + expect(reparsed).toBe(md) + editor.destroy() + }) + + it('clears an empty TRAILING bullet to a paragraph in place — no delete, caret stays on the line', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n- two', { contentType: 'markdown' }) + editor.commands.focus() + emptyItem(editor, 'two') + pressBackspace(editor) + + // The bullet becomes a paragraph; the list keeps only 'one'. The caret sits in the new empty + // paragraph rather than jumping back into the 'one' bullet. + const list = editor.getJSON().content?.find((node) => node.type === 'bulletList') + expect(list?.content).toHaveLength(1) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.$from.parent.type.name).toBe('paragraph') + expect(editor.state.selection.$from.parent.textContent).toBe('') + expect(editor.getMarkdown().trim()).toBe('- one') + editor.destroy() + }) + + it('clears a lone empty bullet to an empty paragraph (whole doc)', () => { + const editor = editorWith('') + editor.commands.setContent('- one', { contentType: 'markdown' }) + editor.commands.focus() + emptyItem(editor, 'one') + pressBackspace(editor) + + expect(editor.getJSON().content?.some((n) => n.type === 'bulletList')).toBe(false) + expect(editor.state.selection.$from.parent.type.name).toBe('paragraph') + editor.destroy() + }) + + it('outdents a nested bullet WITH TEXT one level instead of merging it (round-trips)', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n - two', { contentType: 'markdown' }) + editor.commands.focus() + caretAtStartOf(editor, 'two') + pressBackspace(editor) + + const { md, reparsed } = markdownRoundTrip(editor) + expect(md.trim()).toBe('- one\n- two') + expect(reparsed).toBe(md) + editor.destroy() + }) + + it('outdents an empty nested bullet one level (round-trips)', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n - two\n- three', { contentType: 'markdown' }) + editor.commands.focus() + emptyItem(editor, 'two') + pressBackspace(editor) + + const { md, reparsed } = markdownRoundTrip(editor) + expect(md.trim()).toBe('- one\n- \n- three') + expect(reparsed).toBe(md) + editor.destroy() + }) + + it('clears a checklist item the same way (task item → paragraph)', () => { + const editor = editorWith('') + editor.commands.setContent('- [ ] one\n- [ ] two', { contentType: 'markdown' }) + editor.commands.focus() + caretAtStartOf(editor, 'two') + pressBackspace(editor) + + expect(blockShape(editor).slice(0, 2)).toEqual(['taskList', 'paragraph']) + expect(editor.getMarkdown().trim()).toBe('- [ ] one\n\ntwo') + editor.destroy() + }) + + it('does not delete a non-trailing item whose block holds only a non-text atom', () => { + // Emptiness is the caret block's content.size, not its text: a bullet holding only an inline atom + // (image/mention — here a hardBreak stand-in) is NOT block-empty, so Backspace clears it to a + // paragraph (content preserved) instead of removeEmptyWrappedBlock deleting the whole row. + const editor = editorWith('') + editor.commands.setContent({ + type: 'doc', + content: [ + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [{ type: 'paragraph', content: [{ type: 'hardBreak' }] }], + }, + { + type: 'listItem', + content: [{ type: 'paragraph', content: [{ type: 'text', text: 'two' }] }], + }, + ], + }, + ], + }) + const atomPos = firstPosOf(editor, 'hardBreak') + editor.commands.setTextSelection(atomPos) + pressBackspace(editor) + + // The atom survives (not deleted) and 'two' is untouched. + expect(firstPosOf(editor, 'hardBreak')).toBeGreaterThanOrEqual(0) + expect(editor.state.doc.textContent).toContain('two') + editor.destroy() + }) + + it('removes only the empty first block of a multi-block item, not the whole item', () => { + // An empty first block whose item has sibling blocks must not lift the whole item out of the list; + // only that empty block is removed, the rest of the item (and the list) stays intact. + const editor = editorWith('') + editor.commands.setContent({ + type: 'doc', + content: [ + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [ + { type: 'paragraph' }, + { type: 'paragraph', content: [{ type: 'text', text: 'more' }] }, + ], + }, + ], + }, + ], + }) + // Caret at the start of the empty first paragraph (position 3: doc>bulletList>listItem>paragraph). + editor.commands.setTextSelection(3) + pressBackspace(editor) + + // Still a list (item was NOT lifted out to a top-level paragraph), and 'more' survives. + expect(blockShape(editor)[0]).toBe('bulletList') + expect(editor.state.doc.textContent).toBe('more') + const list = editor.getJSON().content?.find((n) => n.type === 'bulletList') + expect(list?.content).toHaveLength(1) + editor.destroy() + }) +}) + +describe('empty nested bullet does not corrupt its parent (Enter → Tab)', () => { + beforeEach(() => { + Element.prototype.scrollIntoView = vi.fn() + }) + + it('serializes a stranded empty sub-bullet away instead of turning the parent into a heading', () => { + // Repro: type a bullet, Enter for a new bullet, Tab to indent it into an empty sub-bullet, then + // leave it. The serialized `- one\n - ` would re-parse as `- ## one` (Setext underline). The + // serialize step must strip the empty sub-bullet so the parent stays a bullet and round-trips. + const editor = editorWith('') + editor.commands.setContent('- one', { contentType: 'markdown' }) + editor.commands.focus() + let end = -1 + editor.state.doc.descendants((node, pos) => { + if (node.isText && node.text === 'one') end = pos + 3 + }) + editor.commands.setTextSelection(end) + pressKey(editor, 'Enter') + pressKey(editor, 'Tab') + + const saved = postProcessSerializedMarkdown(editor.getMarkdown()) + expect(saved).toBe('- one\n') + + // Reloading the saved markdown keeps a bullet — never a heading. + editor.commands.setContent(saved, { contentType: 'markdown' }) + expect(blockShape(editor)).not.toContain('heading') + expect(blockShape(editor)).toContain('bulletList') editor.destroy() }) }) @@ -341,6 +533,52 @@ describe('empty list-item Enter', () => { expect(editor.getJSON().content?.some((node) => node.type === 'paragraph')).toBe(true) editor.destroy() }) + + it('outdents an empty NESTED item one level instead of removing it', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n - two', { contentType: 'markdown' }) + editor.commands.focus() + emptyItem(editor, 'two') + pressKey(editor, 'Enter') + + // The emptied nested item outdents to a second top-level bullet rather than being deleted. + const list = editor.getJSON().content?.find((node) => node.type === 'bulletList') + expect(list?.content).toHaveLength(2) + expect(list?.content?.every((item) => item.type === 'listItem')).toBe(true) + editor.destroy() + }) + + it('removes only the empty first block of a multi-block item, matching Backspace (keeps the list)', () => { + // Symmetry with the Backspace multi-block case: an empty first block whose item has sibling blocks + // is removed in place — the continuation and the list stay intact — rather than exiting the list. + const editor = editorWith('') + editor.commands.setContent({ + type: 'doc', + content: [ + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [ + { type: 'paragraph' }, + { type: 'paragraph', content: [{ type: 'text', text: 'more' }] }, + ], + }, + ], + }, + ], + }) + // Caret at the start of the empty first paragraph (doc>bulletList>listItem>paragraph). + editor.commands.setTextSelection(3) + pressKey(editor, 'Enter') + + expect(blockShape(editor)[0]).toBe('bulletList') + expect(editor.state.doc.textContent).toBe('more') + const list = editor.getJSON().content?.find((n) => n.type === 'bulletList') + expect(list?.content).toHaveLength(1) + editor.destroy() + }) }) describe('verbatim block boundary (isolating)', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.ts index bdbac57bfc6..524e35eee21 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.ts @@ -20,6 +20,50 @@ const WRAPPER_TYPES = new Set(['listItem', 'taskItem', 'blockquote']) /** Item node types a list is built from, used to detect an empty item's position within its list. */ const LIST_ITEM_TYPES = new Set(['listItem', 'taskItem']) +/** The enclosing list/task item at a caret position, with the facts the boundary keys branch on. */ +interface ListItemContext { + /** `'listItem'` or `'taskItem'` — the type name `liftListItem` must be called with. */ + itemType: string + /** The item's list is itself inside another list item, i.e. the item is indented. */ + isNested: boolean + /** + * The caret's own block (the row the boundary key acts on) has no content. Uses `content.size`, so an + * inline image or mention atom counts as content — a bullet holding only an image is NOT block-empty. + */ + blockEmpty: boolean + /** The item has more than one child block (continuation paragraph, nested list, block image, …). */ + hasSiblingBlocks: boolean + /** The item is the last child of its immediate list. */ + isTrailing: boolean + /** The caret sits in the item's first child block (the row a boundary key should act on). */ + isFirstBlock: boolean +} + +/** + * Resolves the nearest enclosing list/task item at `$from` and the facts the Backspace/Enter handlers + * branch on (nesting, emptiness, trailing position, whether the caret is in the item's first block), or + * null when the caret is not inside a list item. Walking up from `$from` finds the item regardless of + * how deeply the caret's block is nested inside it. + */ +function getListItemContext($from: ResolvedPos): ListItemContext | null { + for (let depth = $from.depth; depth >= 1; depth--) { + const item = $from.node(depth) + if (!LIST_ITEM_TYPES.has(item.type.name)) continue + const listDepth = depth - 1 + const isNested = listDepth >= 1 && LIST_ITEM_TYPES.has($from.node(listDepth - 1).type.name) + const list = $from.node(listDepth) + return { + itemType: item.type.name, + isNested, + blockEmpty: $from.parent.content.size === 0, + hasSiblingBlocks: item.childCount > 1, + isTrailing: $from.index(listDepth) === list.childCount - 1, + isFirstBlock: $from.index(depth) === 0, + } + } + return null +} + const RICH_LEAF_SELECTION_FOCUS_KEY = new PluginKey('richLeafSelectionFocus') /** True when the resolved position sits anywhere inside a {@link WRAPPER_TYPES} ancestor. */ @@ -136,21 +180,26 @@ function selectAdjacentSelectedLeaf(editor: Editor, direction: 'up' | 'down'): b * Editor-specific keyboard behavior layered on top of StarterKit's defaults: * * - **Backspace** at the start of a heading reverts it to a paragraph (ProseMirror's default joins or - * no-ops, stranding the heading style; a second Backspace then merges as usual). At the start of an - * *empty block inside a list item, task item, or blockquote* it removes that whole emptied wrapper via - * {@link removeEmptyWrappedBlock} instead of ProseMirror's default lift — lifting an empty item out of - * the middle of a list/quote splits the container in two and strands an empty paragraph (a visible gap - * that also re-parses to a different markdown document), while the default `joinBackward` alternately - * no-ops on nested items (leaving them stuck) or merges an empty continuation paragraph into the - * previous item. At the start of a block whose previous sibling is a divider or image, where - * ProseMirror's `joinBackward` can't cross the leaf and no-ops: an *empty* block is deleted (clearing - * the blank line between/below dividers without touching the divider itself), while a *non-empty* - * block selects the leaf — so a first Backspace highlights what a second deletes, the same - * highlight-before-delete affordance as clicking it and parity with the arrow-key leaf selection. - * - **Enter** on an *empty, non-trailing list/task item* removes the empty item ({@link - * removeEmptyWrappedBlock}) rather than letting the default split the list into two around a stranded - * empty paragraph (which does not round-trip). A *trailing* empty item still falls through to the - * default, which exits the list — the standard "press Enter on a blank bullet to leave the list". + * no-ops, stranding the heading style; a second Backspace then merges as usual). At the start of a + * *list or task item* it outdents or clears in place via {@link getListItemContext}: a nested item outdents one + * level, a top-level item with text lifts out of the list into a paragraph (keeping the text), and a + * top-level *empty trailing* (or sole) item lifts into an empty paragraph in place — so the blank + * bullet made by pressing Enter can be cleared back to normal text on the same line instead of being + * deleted with the caret jumping to the previous block. The one case lift can't take is a top-level + * *empty, non-trailing* item: lifting it strands an empty paragraph between the two list halves, which + * re-parses to a different markdown document (an empty line between list items is a loose list, not a + * break); that item is removed via {@link removeEmptyWrappedBlock} instead, keeping the list whole. An + * empty block inside a *blockquote* is likewise removed via {@link removeEmptyWrappedBlock}. At the + * start of a block whose previous sibling is a divider or image, where ProseMirror's `joinBackward` + * can't cross the leaf and no-ops: an *empty* block is deleted (clearing the blank line between/below + * dividers without touching the divider itself), while a *non-empty* block selects the leaf — so a + * first Backspace highlights what a second deletes, the same highlight-before-delete affordance as + * clicking it and parity with the arrow-key leaf selection. + * - **Enter** on an empty *nested* list/task item outdents it one level, on an empty + * *non-trailing top-level* item removes it ({@link removeEmptyWrappedBlock}) rather than splitting the + * list around a stranded empty paragraph (which does not round-trip), and on an empty *trailing* item + * falls through to the default, which exits the list — the standard "press Enter on a blank bullet to + * leave the list". * - **Mod-A** inside a code block selects only that block's contents; pressing it again (when the * block is already fully selected) falls through to the default whole-document select-all, the * same scoped behavior as a code editor. @@ -183,6 +232,28 @@ export const RichMarkdownKeymap = Extension.create({ if ($from.parent.type.name === 'heading') { return editor.commands.setParagraph() } + const listCtx = getListItemContext($from) + if (listCtx?.isFirstBlock) { + const { itemType, isNested, blockEmpty, hasSiblingBlocks, isTrailing } = listCtx + // Backspace at the start of a bullet outdents or clears it in place rather than + // deleting the row and jumping the caret to the previous block. + // - Nested item → outdent one level (empty or not). + // - Top-level item whose first line has content (text OR an inline image/mention) → lift out of + // the list into a paragraph, keeping that content. + // - Top-level item whose empty first block has *sibling* blocks (a continuation paragraph, a + // block image, a nested list) → remove only that empty first block via {@link + // removeEmptyWrappedBlock}, leaving the rest of the item intact (never lift the whole item). + // - Top-level empty single-block item that is trailing (or the sole item) → lift into an empty + // paragraph in place, so a fresh bullet made with Enter can be cleared to normal text in place. + // A top-level *empty, non-trailing* single-block item is the one case lift can't take: it strands + // an empty paragraph between the two list halves, which re-parses to a different markdown document + // (an empty line between list items is a loose list, not a break). That case removes the row via + // {@link removeEmptyWrappedBlock} instead, which keeps the list whole and round-trips. + if (isNested || !blockEmpty) return editor.commands.liftListItem(itemType) + if (hasSiblingBlocks) return removeEmptyWrappedBlock(editor, $from) + if (isTrailing) return editor.commands.liftListItem(itemType) + return removeEmptyWrappedBlock(editor, $from) + } if ($from.parent.content.size === 0 && isInsideWrapper($from)) { return removeEmptyWrappedBlock(editor, $from) } @@ -207,11 +278,17 @@ export const RichMarkdownKeymap = Extension.create({ if (!selection.empty || selection.$from.parentOffset !== 0) return false const { $from } = selection if ($from.parent.content.size !== 0) return false - const itemDepth = $from.depth - 1 - if (itemDepth < 1 || !LIST_ITEM_TYPES.has($from.node(itemDepth).type.name)) return false - const listDepth = itemDepth - 1 - const isTrailingItem = $from.index(listDepth) === $from.node(listDepth).childCount - 1 - if (isTrailingItem) return false + const listCtx = getListItemContext($from) + if (!listCtx?.isFirstBlock) return false + // Enter on an empty item, mirroring the Backspace cases above: a nested item outdents one level; + // an empty first block that has *sibling* blocks (continuation paragraph, block image, nested + // list) removes only that empty block in place, keeping the rest of the item — never exiting the + // list or splitting it; a trailing single-block item falls through to the default (exits the + // list); and a non-trailing single-block item is removed rather than splitting the list around a + // stranded empty paragraph (which does not round-trip). + if (listCtx.isNested) return editor.commands.liftListItem(listCtx.itemType) + if (listCtx.hasSiblingBlocks) return removeEmptyWrappedBlock(editor, $from) + if (listCtx.isTrailing) return false return removeEmptyWrappedBlock(editor, $from) }, 'Mod-a': ({ editor }) => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.test.ts new file mode 100644 index 00000000000..721b91dc305 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { postProcessSerializedMarkdown } from './markdown-fidelity' + +describe('postProcessSerializedMarkdown — empty list-item stripping', () => { + it('drops a nested empty bullet that would re-parse as a Setext heading', () => { + // `- one\n - ` re-parses as `- ## one` (the ` - ` acts as a Setext underline). Stripping the + // empty bullet on serialize keeps the parent a bullet and makes the round-trip stable. + expect(postProcessSerializedMarkdown('- one\n - \n\n')).toBe('- one\n') + }) + + it('drops a nested empty ordered item', () => { + expect(postProcessSerializedMarkdown('1. one\n 2. \n')).toBe('1. one\n') + }) + + it('preserves a top-level empty bullet (placeholder / imported blank — round-trips faithfully)', () => { + // A top-level empty item is not Setext-hazardous and round-trips as an empty item, so it must be + // kept: it may be a placeholder row the user is about to fill, or an intentionally-blank imported item. + expect(postProcessSerializedMarkdown('- one\n- \n')).toBe('- one\n- \n') + expect(postProcessSerializedMarkdown('- one\n- \n- three\n')).toBe('- one\n- \n- three\n') + expect(postProcessSerializedMarkdown('1. one\n2. \n')).toBe('1. one\n2. \n') + }) + + it('keeps bullets that have content', () => { + expect(postProcessSerializedMarkdown('- a\n- b\n')).toBe('- a\n- b\n') + expect(postProcessSerializedMarkdown('- a\n - b\n')).toBe('- a\n - b\n') + }) + + it('keeps a nested empty parent whose next line is an indented child (no orphaning)', () => { + expect(postProcessSerializedMarkdown('- top\n - \n - child\n')).toBe( + '- top\n - \n - child\n' + ) + }) + + it('keeps a nested empty item that follows a same-indent sibling (real placeholder, no Setext hazard)', () => { + // ` - ` after ` - two` (same indent) is a real empty item the parser keeps — it does NOT underline + // a shallower parent's text, so it must not be stripped. (Only ` - ` directly under `- one` does.) + expect(postProcessSerializedMarkdown('- one\n - two\n - \n - three\n')).toBe( + '- one\n - two\n - \n - three\n' + ) + // The hazard case — empty item directly under the shallower parent — is still stripped. + expect(postProcessSerializedMarkdown('- one\n - \n - three\n')).toBe('- one\n - three\n') + }) + + it('keeps a thematic break and empty checklist items (not Setext-hazardous)', () => { + expect(postProcessSerializedMarkdown('text\n\n---\n\nmore\n')).toBe('text\n\n---\n\nmore\n') + expect(postProcessSerializedMarkdown('- [ ] a\n- [ ] \n')).toBe('- [ ] a\n- [ ] \n') + }) + + it('leaves marker-only lines inside a fenced code block untouched', () => { + const code = '```\n- \n-\n1. \n```\n' + expect(postProcessSerializedMarkdown(code)).toBe(code) + }) + + it('leaves marker-only lines inside a tilde (~~~) fence untouched', () => { + const code = '~~~\n- \n1. \n~~~\n' + expect(postProcessSerializedMarkdown(code)).toBe(code) + }) + + it('does not strip inside an unterminated fence (fence stays open to EOF)', () => { + // A fence with no closing delimiter must keep every interior line, including marker-only ones. + const code = '```\n- \n-\n' + expect(postProcessSerializedMarkdown(code)).toBe(code) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts index f416482d491..94511d5ebad 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts @@ -106,16 +106,78 @@ export function normalizeLinkHref(href: string): string { return `https://${trimmed}` } +/** A line that is a bullet/ordered list marker with no content (`-`, ` - `, `1. `). Task items (`- [ ]`) don't match. */ +const EMPTY_LIST_ITEM_LINE = /^([ \t]*)(?:[-*+]|\d+[.)])[ \t]*$/ +/** A fenced code-block delimiter (``` or ~~~), used to leave code interiors untouched. */ +const FENCE_DELIMITER = /^[ \t]*(`{3,}|~{3,})/ +/** Leading indentation of a line, used to detect whether an empty list item has indented children. */ +const LEADING_INDENT = /^[ \t]*/ + +/** + * Removes only the *nested* empty list-item marker lines that re-parse as a Setext heading underline: + * a nested empty bullet (` - `) sitting DIRECTLY under a shallower parent line silently turns that + * parent's text into an `## heading` and drops the bullet on the next load (a data-corrupting + * round-trip). The strip is therefore scoped by three conditions, all required: + * - *indented* (`indent > 0`): a top-level empty bullet (`- ` / `1. `) round-trips faithfully as an + * empty item, never a heading, so a placeholder/blank imported row is preserved. + * - the immediately-preceding line is *shallower* (the parent whose text the underline would consume): + * an empty item after a *same-indent sibling* (` - two` then ` - `) does NOT corrupt — the parser + * keeps it as a real empty item — so it is preserved. A blank line above also breaks the hazard. + * - no more-indented children on the next non-blank line, so its children are never orphaned. + * + * Operates only on the editor's own serialized output, which uses fenced (never 4-space-indented) code + * blocks and `\n` newlines — so tracking fences is sufficient and a bare `-` inside an indented code + * block or a `-\r` line is not a case that can occur here. + */ +function stripEmptyListItemLines(markdown: string): string { + const lines = markdown.split('\n') + const kept: string[] = [] + let fence: string | null = null + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + const delimiter = line.match(FENCE_DELIMITER)?.[1] + if (fence) { + kept.push(line) + if (delimiter && delimiter[0] === fence[0] && delimiter.length >= fence.length) fence = null + continue + } + if (delimiter) { + fence = delimiter + kept.push(line) + continue + } + const empty = line.match(EMPTY_LIST_ITEM_LINE) + if (empty) { + const indent = empty[1].length + let next = i + 1 + while (next < lines.length && lines[next].trim() === '') next++ + const hasChildren = + next < lines.length && (lines[next].match(LEADING_INDENT)?.[0].length ?? 0) > indent + // The Setext-underline hazard exists only when the empty item follows a SHALLOWER parent line + // (whose text the underline would consume). An empty item after a same/deeper-indent sibling + // (` - two` then ` - `) is a real empty item the parser keeps — a nested placeholder between + // siblings must not be lost. Uses the preceding non-blank line's indent; a lone empty item with + // nothing above it (`prevIndent = -1`) has no parent text to corrupt but stays stripped as before. + let prevIdx = i - 1 + while (prevIdx >= 0 && lines[prevIdx].trim() === '') prevIdx-- + const prevIndent = prevIdx >= 0 ? (lines[prevIdx].match(LEADING_INDENT)?.[0].length ?? 0) : -1 + if (indent > 0 && !hasChildren && prevIndent < indent) continue + } + kept.push(line) + } + return kept.join('\n') +} + /** - * Cleans up serializer output: restores callout markers the serializer backslash-escapes - * (`> \[!NOTE\]` → `> [!NOTE]`) and collapses trailing blank lines to a single newline. The - * table serializer's spurious surrounding blank lines are trimmed at the source (PipeSafeTable), - * so no global leading-newline strip is needed here — avoiding clobbering content that legitimately - * begins with whitespace. + * Cleans up serializer output: drops empty list-item marker lines that would otherwise corrupt on + * round-trip ({@link stripEmptyListItemLines}), restores callout markers the serializer + * backslash-escapes (`> \[!NOTE\]` → `> [!NOTE]`), and collapses trailing blank lines to a single + * newline. The table serializer's spurious surrounding blank lines are trimmed at the source + * (PipeSafeTable), so no global leading-newline strip is needed here — avoiding clobbering content + * that legitimately begins with whitespace. */ export function postProcessSerializedMarkdown(markdown: string): string { - return collapseAutolinkedUrls(markdown.replace(ESCAPED_CALLOUT_REGEX, '$1[!$2]')).replace( - /\n+$/, - '\n' - ) + return collapseAutolinkedUrls( + stripEmptyListItemLines(markdown).replace(ESCAPED_CALLOUT_REGEX, '$1[!$2]') + ).replace(/\n+$/, '\n') } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.test.ts index 48cb5718ab8..d59ef6afe6e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.test.ts @@ -59,6 +59,12 @@ const CASES: Array<[string, string]> = [ '1. First\n - sub bullet\n - another\n 1. deep ordered\n 2. item\n2. Second', ], ['heading-separated sections', '# A\n\nalpha\n\n## B\n\nbeta\n\n## C\n\ngamma'], + // Blank-line spacing: `@tiptap/markdown` reconstructs empty paragraphs from runs of blank lines, so + // the chunker must reinsert them or a saved blank line vanishes on reload. See the dedicated + // "empty paragraphs" suite below for the exact whole-document-parser parity. + ['one empty paragraph between paragraphs', 'first\n\n\n\nsecond'], + ['two empty paragraphs between paragraphs', 'first\n\n\n\n\n\nsecond'], + ['empty paragraphs between headings and text', '# A\n\n\n\nalpha\n\n\n\n## B'], ] describe('parseMarkdownToDoc (chunked)', () => { @@ -87,6 +93,66 @@ describe('parseMarkdownToDoc (chunked)', () => { expect(splitMarkdownBlocks('\n\n \n')).toEqual([]) }) + // The chunker used to drop empty paragraphs (visual blank lines between blocks) that the whole-document + // parser preserves, so a saved blank line silently vanished on the next load. These assert the chunked + // parse reconstructs the SAME empty-paragraph structure the whole-document parser does — at document + // edges and between blocks, for one or many blank lines, and around lists. + describe('empty paragraphs (blank-line spacing) match the whole-document parser', () => { + /** Block-type shape of a doc, `∅` for an empty paragraph, normalized through the editor. */ + function shapeOf(md: string, parse: 'chunked' | 'whole'): string { + editor = new Editor({ extensions: createMarkdownContentExtensions() }) + if (parse === 'whole') editor.commands.setContent(md, { contentType: 'markdown' }) + else editor.commands.setContent(parseMarkdownToDoc(md), { contentType: 'json' }) + const shape = (editor.getJSON().content ?? []) + .map((n) => (n.type === 'paragraph' && !n.content?.length ? '∅' : n.type)) + .join(',') + editor.destroy() + editor = null + return shape + } + + it.each([ + ['one empty between paragraphs', 'a\n\n\n\nb'], + ['two empties between paragraphs', 'a\n\n\n\n\n\nb'], + ['three empties between paragraphs', 'a\n\n\n\n\n\n\n\nb'], + ['even blank-line gap (rounds down)', 'a\n\n\n\n\nb'], + ['leading empties', '\n\n\n\na'], + ['leading + between', '\n\n\na\n\n\n\nb'], + ['empties between a heading and text', '# H\n\n\n\ntext'], + ['empties after a tight list', '- a\n- b\n\n\n\ntext'], + ['empties before a tight list', 'text\n\n\n\n- a\n- b'], + // Line-ending variants: the whole-vs-chunked routing must normalize first, or a `\r`-only body + // skips the empty-paragraph guard and is chunked (dropping the empties this fix restores). + ['CRLF between empties', 'a\r\n\r\n\r\n\r\nb'], + ['CR-only (classic Mac) between empties', 'a\r\r\r\rb'], + ])('chunked matches whole-doc: %s', (_label, md) => { + expect(shapeOf(md, 'chunked')).toBe(shapeOf(md, 'whole')) + }) + }) + + // Regression: a file ending in a blank line (a trailing empty paragraph) must stay EDITABLE. Such an + // empty paragraph can't be serialized stably (postProcess collapses trailing newlines), so the parser + // strips it — keeping the doc round-trip-safe/idempotent instead of flipping the file read-only. + describe('trailing blank lines stay editable (regression)', () => { + it.each([ + ['plain paragraph', 'abc\n\n'], + ['heading + text', '# Title\n\nSome text\n\n'], + ['three trailing newlines', 'hello\n\n\n'], + ['two paragraphs', 'para one\n\npara two\n\n'], + ['interior empties + trailing', 'a\n\n\n\nb\n\n'], + ])('a file ending in a blank line is round-trip-safe: %s', (_label, md) => { + expect(isRoundTripSafe(md)).toBe(true) + }) + + it('strips the trailing empty paragraph but keeps interior ones', () => { + const trailing = parseMarkdownToDoc('abc\n\n').content ?? [] + expect(trailing.at(-1)?.type).toBe('paragraph') + expect(trailing.at(-1)?.content?.length ?? 0).toBeGreaterThan(0) + const interior = parseMarkdownToDoc('a\n\n\n\nb').content ?? [] + expect(interior.some((n) => n.type === 'paragraph' && !n.content?.length)).toBe(true) + }) + }) + it('parses reference-style links whole (non-chunkable) without dropping the definition', () => { const body = 'See [the docs][ref] for details.\n\n[ref]: https://example.com/docs' expect(serializeMarkdownBody(body)).toBe(oneShot(body)) @@ -195,13 +261,18 @@ function buildFuzzDoc(seed: number): string { describe('chunked parse — property test over randomized documents', () => { it('chunked === one-shot for every document, and idempotent for every editable one', () => { const failures: Array<{ seed: number; kind: string }> = [] + // Compare modulo trailing whitespace: `parseMarkdownToDoc` strips trailing empty paragraphs (they + // can't be serialized stably — postProcess collapses trailing newlines — so keeping them would flip + // the file read-only), whereas the raw one-shot parse keeps them. That trailing-only divergence is + // intended and invisible after save; interior/leading fidelity is still compared exactly. + const trimEnd = (md: string) => md.replace(/\n+$/, '') for (let seed = 1; seed <= 400; seed++) { const body = buildFuzzDoc(seed) const chunked = serializeMarkdownBody(body) // Fidelity is the load-bearing invariant — chunked must never diverge from the whole-document // parse, for ANY input; idempotency only needs to hold where the doc is editable (raw HTML is // non-idempotent in the underlying editor regardless of chunking, which is why it opens read-only). - if (chunked !== oneShot(body)) failures.push({ seed, kind: 'fidelity' }) + if (trimEnd(chunked) !== trimEnd(oneShot(body))) failures.push({ seed, kind: 'fidelity' }) else if (isRoundTripSafe(body) && serializeMarkdownBody(chunked) !== chunked) { failures.push({ seed, kind: 'idempotency' }) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts index 5417c0ee047..c38fac63008 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts @@ -47,6 +47,20 @@ const FENCE_CLOSE = /^ {0,3}(`{3,}|~{3,})[ \t]*$/ const LIST_MARKER = /^[ ]{0,3}(?:[-*+]|\d+[.)])\s/ const BLOCKQUOTE = /^[ ]{0,3}>/ +/** + * Blank-line spacing that `@tiptap/markdown` reconstructs as *interior* or *leading* empty paragraphs — + * a run of two or more blank lines somewhere, or blank line(s) at the document's leading edge. `[^\S\n]` + * matches horizontal whitespace, so a "blank" line may carry spaces/tabs. This is only ever tested + * against the `\r`-normalized body ({@link parseMarkdownToDoc}), so no CRLF handling is needed here. + * + * A *single* trailing blank line is deliberately not matched — purely to avoid routing an otherwise-plain + * file to the slower whole-document parser. Correctness does not depend on it: {@link parseMarkdownToDoc} + * strips trailing empty paragraphs on *both* parse paths ({@link stripTrailingEmptyParagraphs}), so + * serialize→parse stays idempotent regardless of which parser ran. (A trailing run of two or more blanks + * still matches the interior alternative — harmless, since the strip cleans it either way.) + */ +const EMPTY_PARAGRAPH_SPACING = /\n[^\S\n]*\n[^\S\n]*\n|^[^\S\n]*\n[^\S\n]*\n/ + /** * Split a markdown body into top-level blocks that can each be parsed independently and reassembled * without changing meaning. Blank lines separate candidate groups (fenced code blocks stay atomic), @@ -120,20 +134,57 @@ export function splitMarkdownBlocks(body: string): string[] { * vs ~1270ms at 61KB — and byte-identical, because each block is parsed with the same tokenizers. * Documents whose constructs span blocks ({@link NON_CHUNKABLE}) parse whole, and any failure falls * back to a single whole-document parse, so correctness never depends on the splitter. + * + * Blank-line spacing ({@link EMPTY_PARAGRAPH_SPACING}) also parses whole: the chunker parses each block + * stripped of the blank lines between them, so it drops the empty paragraphs `@tiptap/markdown` builds + * from runs of blank lines — a saved visual blank line would silently vanish on reload. Whether a gap + * yields an empty paragraph is a global, block-type-dependent decision (kept between two paragraphs, + * dropped after a heading), so it can't be reconstructed block-locally; these documents parse whole for + * exact fidelity. Ordinary single-blank-line separation still takes the fast chunked path. */ export function parseMarkdownToDoc(body: string): JSONContent { const manager = markdownManager() - if (NON_CHUNKABLE.test(body)) return manager.parse(body) - try { - const content: JSONContent[] = [] - for (const block of splitMarkdownBlocks(body)) { - // `MarkdownManager.parse` always returns a doc node with a `content` array; spread its blocks. - content.push(...(manager.parse(block).content ?? [])) + // Normalize line endings up front so the routing guards see the same `\n` the chunker and parser + // do — the guards' `\n`-anchored tests would otherwise miss a classic `\r`-only body (its blank + // lines are `\r`), routing it to the chunker that then drops its empty paragraphs. + const normalized = body.replace(/\r\n?/g, '\n') + let doc: JSONContent + if (NON_CHUNKABLE.test(normalized) || EMPTY_PARAGRAPH_SPACING.test(normalized)) { + doc = manager.parse(normalized) + } else { + try { + const content: JSONContent[] = [] + for (const block of splitMarkdownBlocks(normalized)) { + // `MarkdownManager.parse` always returns a doc node with a `content` array; spread its blocks. + content.push(...(manager.parse(block).content ?? [])) + } + doc = { type: 'doc', content } + } catch { + doc = manager.parse(normalized) } - return { type: 'doc', content } - } catch { - return manager.parse(body) } + return stripTrailingEmptyParagraphs(doc) +} + +/** An empty paragraph node — the shape a blank line reconstructs to (no content, or `content: []`). */ +function isEmptyParagraph(node: JSONContent): boolean { + return node.type === 'paragraph' && !node.content?.length +} + +/** + * Drop trailing empty paragraphs from a parsed doc. {@link postProcessSerializedMarkdown} collapses + * trailing blank lines to a single newline, so a trailing empty paragraph can never round-trip — the + * whole-document parser reconstructs one from a file ending in a blank line, but keeping it makes + * serialize→parse non-idempotent, which flips the file read-only via the round-trip-safety probe. + * Leading/interior empty paragraphs are untouched (postProcess never strips those). TipTap re-adds its + * own trailing filler paragraph on `setContent`, so the editor still has a place to type. + */ +function stripTrailingEmptyParagraphs(doc: JSONContent): JSONContent { + const content = doc.content + if (!content || content.length === 0) return doc + let end = content.length + while (end > 0 && isEmptyParagraph(content[end - 1])) end-- + return end === content.length ? doc : { ...doc, content: content.slice(0, end) } } /** diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 2b9514b4f6a..c7bd689c7d7 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -4,6 +4,7 @@ import { memo, useEffect, useRef, useState } from 'react' import { cn, toast } from '@sim/emcn' import type { JoinFileDocError } from '@sim/realtime-protocol/file-doc' import type { Extensions, JSONContent } from '@tiptap/core' +import { isChangeOrigin } from '@tiptap/extension-collaboration' import { Fragment, Slice } from '@tiptap/pm/model' import { NodeSelection } from '@tiptap/pm/state' import { dropPoint } from '@tiptap/pm/transform' @@ -13,6 +14,7 @@ import { useRouter } from 'next/navigation' import { useSession } from '@/lib/auth/auth-client' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { extractEmbeddedFileRef } from '@/lib/uploads/utils/embedded-image-ref' +import { isUntitledName } from '@/app/workspace/[workspaceId]/files/untitled-title' import { useUploadWorkspaceFile } from '@/hooks/queries/workspace-files' import type { SaveStatus } from '@/hooks/use-autosave' import { useFileContentSource } from '@/hooks/use-file-content-source' @@ -41,6 +43,7 @@ import { LinkHoverCard } from './menus/link-hover-card' import { TableBubbleMenu } from './menus/table-menu' import { normalizeMarkdownContent } from './normalize-content' import { isRoundTripSafe } from './round-trip-safety' +import { firstHeadingTitle } from './title-heading' import '@sim/emcn/components/code/code.css' import './rich-markdown-editor.css' @@ -55,6 +58,9 @@ const EXTENSIONS = createMarkdownEditorExtensions({ const STREAM_REPARSE_THROTTLE_THRESHOLD = 40_000 const STREAM_REPARSE_THROTTLE_MS = 120 +/** Debounce before naming a still-untitled file after its leading heading, so it fires once typing settles. */ +const DERIVE_TITLE_DEBOUNCE_MS = 600 + interface RichMarkdownEditorProps { file: WorkspaceFileRecord workspaceId: string @@ -83,6 +89,12 @@ interface RichMarkdownEditorProps { * construction (they cannot both drive one editor and corrupt the shared doc). */ collaborative?: boolean + /** + * Called (debounced) with the document's leading-heading text while the file is still untitled, so the + * caller can name the file after it. Omitted on read-only/non-editable surfaces. See + * {@link isUntitledName}. + */ + onDeriveTitleFromHeading?: (headingText: string) => void } /** Inline WYSIWYG markdown editor: agent output streams in read-only, then the same instance becomes editable on settle. */ @@ -102,6 +114,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ previewContextKey, disableTagging, collaborative = false, + onDeriveTitleFromHeading, }: RichMarkdownEditorProps) { const { data: session, isPending: isSessionPending } = useSession() const userId = session?.user?.id ?? '' @@ -168,6 +181,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ onChange={setDraftContent} onSaveShortcut={saveImmediately} onCollabReadyChange={setCollabReady} + onDeriveTitleFromHeading={onDeriveTitleFromHeading} /> ) }) @@ -194,6 +208,8 @@ interface LoadedRichMarkdownEditorProps { onSaveShortcut: () => Promise /** Reports whether the collaborative document is synced+seeded (autosave gate). */ onCollabReadyChange: (ready: boolean) => void + /** See {@link RichMarkdownEditorProps.onDeriveTitleFromHeading}. */ + onDeriveTitleFromHeading?: (headingText: string) => void } interface SettledContent { @@ -223,6 +239,7 @@ export function LoadedRichMarkdownEditor({ onChange, onSaveShortcut, onCollabReadyChange, + onDeriveTitleFromHeading, }: LoadedRichMarkdownEditorProps) { /** Whether this editor mounted mid-stream — if so it starts empty and syncs streamed chunks until settle. */ const streamingAtMountRef = useRef(isStreaming) @@ -288,6 +305,17 @@ export function LoadedRichMarkdownEditor({ onChangeRef.current = onChange const onSaveShortcutRef = useRef(onSaveShortcut) onSaveShortcutRef.current = onSaveShortcut + + /** + * While the file is still unnamed, name it after its leading heading: `onDeriveTitleFromHeading` is + * called (debounced) so the caller can rename the file, and `fileNameRef` lets the onUpdate handler + * read the current name without re-subscribing. See {@link isUntitledName}. + */ + const onDeriveTitleFromHeadingRef = useRef(onDeriveTitleFromHeading) + onDeriveTitleFromHeadingRef.current = onDeriveTitleFromHeading + const fileNameRef = useRef(file.name) + fileNameRef.current = file.name + const deriveTitleTimerRef = useRef | null>(null) /** * Read in the RAF tick so an already-scheduled tick still sees the latest edit kind (it can change * between sessions within one turn, e.g. an append followed by a rewrite). @@ -542,14 +570,45 @@ export function LoadedRichMarkdownEditor({ return false }, }, - onUpdate: ({ editor }) => { + onUpdate: ({ editor, transaction }) => { const md = postProcessSerializedMarkdown(editor.getMarkdown()) lastSyncedBodyRef.current = md onChangeRef.current(applyFrontmatter(settledRef.current?.frontmatter ?? '', md)) + // While the file is still untitled, name it after its leading heading once typing settles — but + // only for the LOCAL user's own edits. `isChangeOrigin` is true for a remote Yjs change (a peer + // typing); bail BEFORE touching the timer so a remote edit never cancels or reschedules the local + // user's pending rename (and every client doesn't schedule the same rename from a peer's + // not-yet-synced heading). It is false for local edits and non-collaborative surfaces. + if (isChangeOrigin(transaction)) return + // Local edit: restart the debounce. Clearing first cancels a stale rename if the heading was + // removed/changed before it fired; the timer re-derives the title from the live doc rather than a + // value captured now, so it can never name the file after a heading the user has since changed. + // `editor.isEditable` is the autosave gate (canEdit + settled + collab-ready), so a view-only + // viewer or the not-yet-editable mount seed never schedules a rename. + if (deriveTitleTimerRef.current) clearTimeout(deriveTitleTimerRef.current) + if ( + !editor.isEditable || + !isUntitledName(fileNameRef.current) || + firstHeadingTitle(editor.state.doc) === null + ) + return + deriveTitleTimerRef.current = setTimeout(() => { + const liveEditor = editorInstanceRef.current + if (!liveEditor || !liveEditor.isEditable || !isUntitledName(fileNameRef.current)) return + const title = firstHeadingTitle(liveEditor.state.doc) + if (title) onDeriveTitleFromHeadingRef.current?.(title) + }, DERIVE_TITLE_DEBOUNCE_MS) }, }) editorInstanceRef.current = editor + useEffect( + () => () => { + if (deriveTitleTimerRef.current) clearTimeout(deriveTitleTimerRef.current) + }, + [] + ) + /** * The loaded markdown to seed the shared doc from, held by pointer so the parse * runs once at seed time rather than every render. diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.test.ts new file mode 100644 index 00000000000..f6c46bd027f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.test.ts @@ -0,0 +1,63 @@ +/** + * @vitest-environment jsdom + */ +import { Editor } from '@tiptap/core' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createMarkdownEditorExtensions } from './editor-extensions' +import { firstHeadingTitle } from './title-heading' + +function editorWith(markdown: string): Editor { + const editor = new Editor({ extensions: createMarkdownEditorExtensions({ placeholder: '' }) }) + if (markdown) editor.commands.setContent(markdown, { contentType: 'markdown' }) + return editor +} + +describe('firstHeadingTitle', () => { + beforeEach(() => { + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ) + Element.prototype.scrollIntoView = vi.fn() + document.elementFromPoint = vi.fn(() => null) + }) + + it('returns the leading H1 text', () => { + const editor = editorWith('# Q3 Planning\n\nbody') + expect(firstHeadingTitle(editor.state.doc)).toBe('Q3 Planning') + editor.destroy() + }) + + it('returns the text of any leading heading level', () => { + const editor = editorWith('## Sub title') + expect(firstHeadingTitle(editor.state.doc)).toBe('Sub title') + editor.destroy() + }) + + it('returns null when the first block is a paragraph, not a heading', () => { + const editor = editorWith('just text\n\n# later heading') + expect(firstHeadingTitle(editor.state.doc)).toBeNull() + editor.destroy() + }) + + it('returns null for an empty leading heading', () => { + const editor = editorWith('') + editor.commands.setContent({ type: 'doc', content: [{ type: 'heading', attrs: { level: 1 } }] }) + expect(firstHeadingTitle(editor.state.doc)).toBeNull() + editor.destroy() + }) + + it('returns null for a whitespace-only leading heading (trim boundary)', () => { + const editor = editorWith('') + editor.commands.setContent({ + type: 'doc', + content: [{ type: 'heading', attrs: { level: 1 }, content: [{ type: 'text', text: ' ' }] }], + }) + expect(firstHeadingTitle(editor.state.doc)).toBeNull() + editor.destroy() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.ts new file mode 100644 index 00000000000..3877519ca97 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.ts @@ -0,0 +1,9 @@ +import type { Node as ProseMirrorNode } from '@tiptap/pm/model' + +/** The text of the document's leading heading (any level), or null when the first block isn't a heading. */ +export function firstHeadingTitle(doc: ProseMirrorNode): string | null { + const first = doc.firstChild + if (!first || first.type.name !== 'heading') return null + const text = first.textContent.trim() + return text.length > 0 ? text : null +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 8094080b93d..807c8cc3ad9 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -85,6 +85,12 @@ import { filesSortParams, filesUrlKeys, } from '@/app/workspace/[workspaceId]/files/search-params' +import { + DEFAULT_UNTITLED_NAME, + deriveMarkdownFileName, + isUntitledName, + uniqueMarkdownName, +} from '@/app/workspace/[workspaceId]/files/untitled-title' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks' import { useWorkspaceMembersQuery, type WorkspaceMember } from '@/hooks/queries/workspace' @@ -356,6 +362,34 @@ export function Files() { const selectedFileRef = useRef(selectedFile) selectedFileRef.current = selectedFile + /** + * While a file is still untitled, name it after the leading heading the user types in its editor. The + * editor reports the heading text (debounced); here we re-check the file is still untitled, derive a + * unique `.md` name among its folder siblings, and rename. A no-op once the file has a real name. + */ + const handleDeriveTitleFromHeading = useCallback( + (headingText: string) => { + const currentFile = selectedFileRef.current + if (!currentFile || !isUntitledName(currentFile.name)) return + const derived = deriveMarkdownFileName(headingText) + if (!derived) return + const siblingNames = new Set( + filesRef.current + .filter( + (f) => + (f.folderId ?? null) === (currentFile.folderId ?? null) && f.id !== currentFile.id + ) + .map((f) => f.name) + ) + const name = uniqueMarkdownName(derived, siblingNames) + if (name === currentFile.name) return + renameFile + .mutateAsync({ workspaceId, fileId: currentFile.id, name }) + .catch((err) => logger.error('Failed to auto-name file from heading:', err)) + }, + [workspaceId] + ) + const shareFile = shareFileId ? (files.find((f) => f.id === shareFileId) ?? null) : null const shareModal = shareFile ? ( (f.folderId ?? null) === currentFolderId).map((f) => f.name) ) - let name = 'untitled.md' - let counter = 1 - while (existingNames.has(name)) { - name = `untitled (${counter}).md` - counter++ - } + const name = uniqueMarkdownName(DEFAULT_UNTITLED_NAME, existingNames) const mimeType = getMimeTypeFromExtension('md') const blob = new Blob([''], { type: mimeType }) @@ -1931,6 +1960,7 @@ export function Files() { saveRef={saveRef} discardRef={discardRef} collaborative + onDeriveTitleFromHeading={handleDeriveTitleFromHeading} /> { + // Guards against DEFAULT_UNTITLED_NAME / uniqueMarkdownName drifting from the isUntitledName regex: + // the default name and its deduped siblings must always read back as "untitled". + it('recognizes the default name and its deduped siblings as untitled', () => { + expect(isUntitledName(DEFAULT_UNTITLED_NAME)).toBe(true) + const second = uniqueMarkdownName(DEFAULT_UNTITLED_NAME, new Set([DEFAULT_UNTITLED_NAME])) + expect(second).toBe('untitled (1).md') + expect(isUntitledName(second)).toBe(true) + }) +}) + +describe('isUntitledName', () => { + it.each([ + ['untitled.md', true], + ['untitled (1).md', true], + ['untitled (23).md', true], + ['Untitled.md', false], + ['untitled.txt', false], + ['untitled', false], + ['my notes.md', false], + ['untitled draft.md', false], + ['untitled ().md', false], + ])('%s → %s', (name, expected) => { + expect(isUntitledName(name)).toBe(expected) + }) +}) + +describe('deriveMarkdownFileName', () => { + it('turns heading text into a .md file name', () => { + expect(deriveMarkdownFileName('Q3 Planning')).toBe('Q3 Planning.md') + }) + it('strips filesystem-illegal characters and collapses whitespace', () => { + expect(deriveMarkdownFileName('Roadmap: Q3 / Q4 *draft*')).toBe('Roadmap Q3 Q4 draft.md') + }) + it('keeps hyphens and dots inside the title', () => { + expect(deriveMarkdownFileName('v1.2 - release-notes')).toBe('v1.2 - release-notes.md') + }) + it('returns null when nothing usable remains', () => { + expect(deriveMarkdownFileName(' ')).toBeNull() + expect(deriveMarkdownFileName('///')).toBeNull() + }) + + it('does not double the extension when the heading already ends in .md', () => { + expect(deriveMarkdownFileName('README.md')).toBe('README.md') + expect(deriveMarkdownFileName('notes.MD')).toBe('notes.MD') + }) + it('hard-caps the length (no ellipsis) before the extension', () => { + const result = deriveMarkdownFileName('a'.repeat(200)) + expect(result).toBe(`${'a'.repeat(100)}.md`) + }) + + it('re-trims when the hard cap lands on a space (no "foo .md")', () => { + // 99 non-space chars + space at index 99 → truncate(100) leaves a trailing space to re-trim away. + const result = deriveMarkdownFileName(`${'a'.repeat(99)} bcd`) + expect(result).toBe(`${'a'.repeat(99)}.md`) + }) +}) + +describe('uniqueMarkdownName', () => { + it('returns the name unchanged when free', () => { + expect(uniqueMarkdownName('notes.md', new Set())).toBe('notes.md') + }) + it('appends an incrementing suffix before the extension when taken', () => { + expect(uniqueMarkdownName('notes.md', new Set(['notes.md']))).toBe('notes (1).md') + expect(uniqueMarkdownName('notes.md', new Set(['notes.md', 'notes (1).md']))).toBe( + 'notes (2).md' + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts new file mode 100644 index 00000000000..4800fb13e9a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts @@ -0,0 +1,57 @@ +import { truncate } from '@sim/utils/string' + +/** + * The name a freshly-created markdown file is given in `handleCreateFile`: `untitled.md`, or + * `untitled (n).md` when that is taken. A file keeps this "unnamed" status until it is renamed — + * while unnamed, typing a leading heading names the file (one direction only; the reverse + * name→heading seed was removed as unsafe on the shared editor). See {@link isUntitledName}. + */ +export const DEFAULT_UNTITLED_NAME = 'untitled.md' + +const UNTITLED_NAME_RE = /^untitled(?: \(\d+\))?\.md$/ + +/** Longest title kept when deriving a file name from a heading, before the `.md` extension. */ +const MAX_DERIVED_TITLE_LENGTH = 100 + +/** + * Filename characters disallowed across the common platforms (`\ / : * ? " < > |`) plus C0 control + * characters, replaced with a space when deriving a file name from heading text. + */ +const ILLEGAL_FILENAME_CHARS = /[\\/:*?"<>|\x00-\x1f]/g + +/** True when `name` is still the auto-assigned untitled markdown name (`untitled.md`, `untitled (2).md`). */ +export function isUntitledName(name: string): boolean { + return UNTITLED_NAME_RE.test(name) +} + +/** + * Derives a markdown file name from heading text — illegal filename characters dropped, whitespace + * collapsed, trimmed, hard-capped at {@link MAX_DERIVED_TITLE_LENGTH}, and suffixed with `.md`. + * Returns null when nothing usable remains (e.g. a heading of only slashes), so the caller keeps the + * current name. + */ +export function deriveMarkdownFileName(headingText: string): string | null { + const base = headingText.replace(ILLEGAL_FILENAME_CHARS, ' ').replace(/\s+/g, ' ').trim() + if (!base) return null + // Re-trim after the hard cap: truncation can land mid-word and leave a trailing space (`"foo .md"`). + const capped = truncate(base, MAX_DERIVED_TITLE_LENGTH, '').trim() + if (!capped) return null + // A heading that already ends in `.md` (e.g. `# README.md`) must not become `README.md.md`. + return /\.md$/i.test(capped) ? capped : `${capped}.md` +} + +/** + * Makes `name` unique among `existingNames` by appending ` (n)` before the `.md` extension — the same + * scheme `handleCreateFile` uses for the default untitled name. + */ +export function uniqueMarkdownName(name: string, existingNames: ReadonlySet): string { + if (!existingNames.has(name)) return name + const withoutExt = name.replace(/\.md$/i, '') + let counter = 1 + let candidate = `${withoutExt} (${counter}).md` + while (existingNames.has(candidate)) { + counter++ + candidate = `${withoutExt} (${counter}).md` + } + return candidate +} From bdf47c867fde9372f69d219948bdc4eebccb9b24 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 16:18:51 -0700 Subject: [PATCH 11/53] fix(realtime): access-revalidation multi-room safety + cleanup pass Fix a blocker surfaced by a full cleanup/simplify audit of the branch: the access-revalidation sweep (staging's workflow-only #5917) treated every entry in socket.rooms as a workflow id, but the generalized multi-room model puts namespaced files/tables/file-doc rooms on the same io. It would resolve those as bogus workflows, get null, and evict files/tables collaborators every ~30s. collectScanTargets now decodes each room name with parseRoomName and sweeps only workflow rooms; added a regression test and fixed the now-false TSDoc. Other audit fixes (all behavior-preserving): - workflow.ts reuses resolveAvatarUrl (drops db/user/eq imports duplicated from avatar.ts) - PresenceAvatars: mr-1 was baked into the shared component, silently adding a margin to the workflow sidebar stack; moved to an optional layout className, re-applied on the tables/file-doc header surfaces only - table DELETE routes only signal collaborators when rows were actually removed (matches PUT) - events.ts definition kind: drop the never-emitted reason:'schema', fix its doc - event-log: rename buildMemory -> buildEntry (it builds the entry on the Redis success path too, not just the memory fallback) - remove dead resolveWorkspaceIdForRoom export; parallelize per-socket removals in handleWorkflowDeletion; gate the table columnIndexById map on remote selections; move file-doc module TSDoc off the FileDocOwner interface; fix a stale @returns --- apps/realtime/src/access-revalidation.test.ts | 29 ++++++++++++-- apps/realtime/src/access-revalidation.ts | 17 ++++++--- apps/realtime/src/handlers/file-doc.ts | 38 ++++++++++--------- apps/realtime/src/handlers/workflow.ts | 19 +--------- .../src/rooms/workflow-room-service.ts | 7 ++-- .../sim/app/api/table/[tableId]/rows/route.ts | 4 +- .../components/presence/presence-avatars.tsx | 12 ++++-- .../collaboration/file-doc-avatars.tsx | 2 +- .../components/table-grid/table-grid.tsx | 8 ++-- .../[workspaceId]/tables/[tableId]/table.tsx | 4 +- apps/sim/lib/realtime/event-log.test.ts | 2 +- apps/sim/lib/realtime/event-log.ts | 15 +++++--- apps/sim/lib/table/events.ts | 8 ++-- apps/sim/lib/table/service.ts | 2 +- packages/platform-authz/src/rooms.ts | 5 --- 15 files changed, 99 insertions(+), 73 deletions(-) diff --git a/apps/realtime/src/access-revalidation.test.ts b/apps/realtime/src/access-revalidation.test.ts index 358c03188a0..8d9d37cf349 100644 --- a/apps/realtime/src/access-revalidation.test.ts +++ b/apps/realtime/src/access-revalidation.test.ts @@ -50,13 +50,13 @@ function makeManager(sockets: FakeSocket[], presence: Partial[] = const manager = { io: { sockets: { sockets: socketMap } }, isReady: () => true, - getWorkflowUsers: vi.fn().mockResolvedValue(presence), + getRoomUsers: vi.fn().mockResolvedValue(presence), getRoomForSocket: vi.fn().mockResolvedValue(null), removeUserFromRoom: vi.fn().mockResolvedValue(true), broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), } return manager as unknown as IRoomManager & { - getWorkflowUsers: ReturnType + getRoomUsers: ReturnType getRoomForSocket: ReturnType removeUserFromRoom: ReturnType broadcastPresenceUpdate: ReturnType @@ -142,7 +142,30 @@ describe('access-revalidation sweep', () => { expect(mockResolveRole).toHaveBeenCalledWith('user-1', 'wf-1', 'read') // The security scan must stay Redis-free — presence is never consulted. - expect(manager.getWorkflowUsers).not.toHaveBeenCalled() + expect(manager.getRoomUsers).not.toHaveBeenCalled() + }) + + it('never evicts a socket joined only to a non-workflow room (files/tables/file-doc)', async () => { + // The sweep shares one io with the files/tables/file-doc handlers. Those rooms are + // namespaced (`workspace-files:ws-1`, `table:t-1`), so treating every socket.rooms + // entry as a workflow id would resolve a bogus permission → null → evict the socket + // from its files/table room every pass. Non-workflow rooms must be filtered out. + const filesSocket = makeSocket('sock-1', 'user-1', 'workspace-files:ws-1') + const tableSocket = makeSocket('sock-2', 'user-2', 'table:t-1') + const manager = makeManager([filesSocket, tableSocket]) + // Even if the role resolver would say "no access", these must never be swept. + mockResolveRole.mockResolvedValue(null) + + const sweep = startAccessRevalidationSweep(manager) + await sweep.runOnce() + sweep.stop() + + expect(mockResolveRole).not.toHaveBeenCalled() + expect(filesSocket.leave).not.toHaveBeenCalled() + expect(filesSocket.emit).not.toHaveBeenCalled() + expect(tableSocket.leave).not.toHaveBeenCalled() + expect(tableSocket.emit).not.toHaveBeenCalled() + expect(manager.removeUserFromRoom).not.toHaveBeenCalled() }) it('evicts only the revoked socket, not co-members of the room', async () => { diff --git a/apps/realtime/src/access-revalidation.ts b/apps/realtime/src/access-revalidation.ts index 3337b06e871..971a12de4b5 100644 --- a/apps/realtime/src/access-revalidation.ts +++ b/apps/realtime/src/access-revalidation.ts @@ -1,6 +1,6 @@ import { createLogger } from '@sim/logger' import type { AccessRevokedBroadcast } from '@sim/realtime-protocol/events' -import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { parseRoomName, ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { sleep } from '@sim/utils/helpers' import type { AuthenticatedSocket } from '@/middleware/auth' import { ROLE_REVALIDATION_TTL_MS, resolveCurrentWorkflowRole } from '@/middleware/permissions' @@ -66,9 +66,14 @@ interface ScanTarget { * Collects this pod's authenticated sockets with the workflow room each has * joined, in stable socket order. * - * The workflow room is derived from the socket's own `rooms` set (pod-local, no - * Redis round-trips): a socket joins exactly one workflow room, so its rooms are - * `{ ownSocketId, workflowId }`. Only local sockets are evaluated — sockets are + * Rooms are derived from the socket's own `rooms` set (pod-local, no Redis + * round-trips). A socket may occupy several rooms of different types at once + * (workflow canvas, workspace-files browser, table, file-doc), all on the same + * io — so each name is decoded with {@link parseRoomName} and only **workflow** + * rooms are swept here. Non-workflow room names are namespaced (`type:id`) and + * resolve to a non-workflow type; sweeping them as workflow ids would resolve a + * bogus permission, come back `null`, and spuriously evict the socket from its + * files/table room every pass. Only local sockets are evaluated — sockets are * sticky to a pod, so every socket is swept by exactly one pod using that pod's * warm role cache (mirroring the per-pod reasoning of the write-path cache). */ @@ -79,7 +84,9 @@ function collectScanTargets(io: IRoomManager['io']): ScanTarget[] { if (!authed.userId) continue for (const room of socket.rooms) { if (room === socket.id) continue - targets.push({ workflowId: room, socket: authed, userId: authed.userId }) + const ref = parseRoomName(room) + if (ref?.type !== ROOM_TYPES.WORKFLOW) continue + targets.push({ workflowId: ref.id, socket: authed, userId: authed.userId }) } } return targets diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index d019a561bd8..31c9d562cfc 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -1,3 +1,23 @@ +/** + * Collaborative document editing (live carets + text selection) for a single + * file's rich-text editor. This is the standard Yjs "websocket server" relay — + * an authoritative in-memory {@link Y.Doc} + {@link awarenessProtocol.Awareness} + * per file — carried over the shared, already-authenticated Socket.IO connection + * and the room abstraction, rather than a separate ws server. Clients speak the + * `y-protocols` sync + awareness protocols; the server applies and relays them. + * + * No durable Yjs state is kept yet: the document lives only while at least one + * collaborator is connected, and is re-seeded from the file's stored markdown on + * the next cold open (the markdown, saved by a client through the content API, is + * the durable source of truth). Durable Yjs snapshots are a separate follow-up. + * + * Single-writer assumption: the authoritative {@link Y.Doc} is held in this + * process's memory, so correctness assumes one realtime replica per file (Helm + * pins `realtime.replicaCount: 1`). Horizontal scaling would need a shared Yjs + * backend (y-redis / Hocuspocus) — out of scope here. + * + * @module + */ import { createLogger } from '@sim/logger' import { authorizeRoom } from '@sim/platform-authz/rooms' import { @@ -30,24 +50,6 @@ const logger = createLogger('FileDocHandlers') */ const SEED_DEADLINE_MS = 10_000 -/** - * Collaborative document editing (live carets + text selection) for a single - * file's rich-text editor. This is the standard Yjs "websocket server" relay — - * an authoritative in-memory {@link Y.Doc} + {@link awarenessProtocol.Awareness} - * per file — carried over the shared, already-authenticated Socket.IO connection - * and the room abstraction, rather than a separate ws server. Clients speak the - * `y-protocols` sync + awareness protocols; the server applies and relays them. - * - * No durable Yjs state is kept yet: the document lives only while at least one - * collaborator is connected, and is re-seeded from the file's stored markdown on - * the next cold open (the markdown, saved by a client through the content API, is - * the durable source of truth). Durable Yjs snapshots are a separate follow-up. - * - * Single-writer assumption: the authoritative {@link Y.Doc} is held in this - * process's memory, so correctness assumes one realtime replica per file (Helm - * pins `realtime.replicaCount: 1`). Horizontal scaling would need a shared Yjs - * backend (y-redis / Hocuspocus) — out of scope here. - */ /** A socket's presence ownership within a room. */ interface FileDocOwner { /** diff --git a/apps/realtime/src/handlers/workflow.ts b/apps/realtime/src/handlers/workflow.ts index 4d482a1434a..05b9e30fabf 100644 --- a/apps/realtime/src/handlers/workflow.ts +++ b/apps/realtime/src/handlers/workflow.ts @@ -1,8 +1,7 @@ -import { db, user } from '@sim/db' import { createLogger } from '@sim/logger' import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' -import { eq } from 'drizzle-orm' import { getWorkflowState } from '@/database/operations' +import { resolveAvatarUrl } from '@/handlers/avatar' import type { AuthenticatedSocket } from '@/middleware/auth' import { resolveCurrentWorkflowRole, verifyWorkflowAccess } from '@/middleware/permissions' import { type IRoomManager, type UserPresence, workflowRoom as wf } from '@/rooms' @@ -158,21 +157,7 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: // Join the new room socket.join(workflowId) - // Get avatar URL - let avatarUrl = socket.userImage || null - if (!avatarUrl) { - try { - const [userRecord] = await db - .select({ image: user.image }) - .from(user) - .where(eq(user.id, userId)) - .limit(1) - - avatarUrl = userRecord?.image ?? null - } catch (error) { - logger.warn('Failed to load user avatar for presence', { userId, error }) - } - } + const avatarUrl = await resolveAvatarUrl(socket, userId) // Create presence entry const userPresence: UserPresence = { diff --git a/apps/realtime/src/rooms/workflow-room-service.ts b/apps/realtime/src/rooms/workflow-room-service.ts index f12a6c337d4..165224b08c4 100644 --- a/apps/realtime/src/rooms/workflow-room-service.ts +++ b/apps/realtime/src/rooms/workflow-room-service.ts @@ -49,9 +49,10 @@ export class WorkflowRoomService { // Remove every socket from the Socket.IO room (cross-pod via the Redis adapter). await this.manager.io.in(name).socketsLeave(name) - for (const socketId of socketIds) { - await this.manager.removeUserFromRoom(room, socketId) - } + // Independent per-socket removals — run concurrently. + await Promise.all( + Array.from(socketIds, (socketId) => this.manager.removeUserFromRoom(room, socketId)) + ) // Final unconditional wipe — the workflow is gone, so no room state may linger // even if a per-socket removal failed (matches the pre-refactor managers, which diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index f8570eadb76..3c42f757374 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -441,7 +441,7 @@ export const DELETE = withRouteHandler( { tableId, rowIds: validated.rowIds, workspaceId: validated.workspaceId }, requestId ) - signalTableRowsChanged(tableId) + if (result.deletedCount > 0) signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -467,7 +467,7 @@ export const DELETE = withRouteHandler( }, requestId ) - signalTableRowsChanged(tableId) + if (result.affectedCount > 0) signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx index 70c75e416b1..da5bcc68d68 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx @@ -1,7 +1,7 @@ 'use client' import { useMemo } from 'react' -import { Avatar, AvatarFallback, AvatarImage, Tooltip } from '@sim/emcn' +import { Avatar, AvatarFallback, AvatarImage, cn, Tooltip } from '@sim/emcn' import { getUserColor } from '@/lib/workspaces/colors' /** Minimal presence shape the avatar stack renders — shared by workflow and files. */ @@ -64,6 +64,8 @@ interface PresenceAvatarsProps { users: PresenceAvatarUser[] /** Max avatars before collapsing the remainder into a "+N" chip. */ maxVisible?: number + /** Layout-only classes for the outer stack (e.g. surrounding margin); chrome is owned here. */ + className?: string } const DEFAULT_MAX_VISIBLE = 5 @@ -73,7 +75,11 @@ const DEFAULT_MAX_VISIBLE = 5 * the caller owns fetching/filtering presence (workflow sidebar item, files * header, etc.), so the stack looks identical everywhere it appears. */ -export function PresenceAvatars({ users, maxVisible = DEFAULT_MAX_VISIBLE }: PresenceAvatarsProps) { +export function PresenceAvatars({ + users, + maxVisible = DEFAULT_MAX_VISIBLE, + className, +}: PresenceAvatarsProps) { const { visibleUsers, overflowCount } = useMemo(() => { if (users.length === 0) { return { visibleUsers: [] as PresenceAvatarUser[], overflowCount: 0 } @@ -89,7 +95,7 @@ export function PresenceAvatars({ users, maxVisible = DEFAULT_MAX_VISIBLE }: Pre } return ( -
+
{overflowCount > 0 && ( diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx index 13464e720e1..1d093b21fa2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx @@ -10,5 +10,5 @@ import { useFileDocOthers } from './file-doc-room-context' */ export function FileDocAvatars() { const others = useFileDocOthers() - return + return } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index 547243d51b9..68133c595c1 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -748,12 +748,14 @@ export function TableGrid({ return expandToDisplayColumns(ordered, tableWorkflowGroups) }, [columns, columnOrder, tableWorkflowGroups]) - /** Column id → its rendered index (matches the cells' `data-col`), for placing overlays. */ + /** Column id → its rendered index (matches the cells' `data-col`), for placing overlays. + * Only built when collaborators are present (the overlay it feeds is gated on that too), + * so solo editing never pays the map build. */ const columnIndexById = useMemo(() => { const map = new Map() - displayColumns.forEach((col, index) => map.set(col.key, index)) + if (remoteSelections.length > 0) displayColumns.forEach((col, index) => map.set(col.key, index)) return map - }, [displayColumns]) + }, [displayColumns, remoteSelections.length]) /** Row id → its index in the current row list, for testing local-selection coverage. * Only built when collaborators are present (the overlay is gated on that too), so diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index 37e24e0b46b..b2ae6d7fe1c 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -769,7 +769,9 @@ export function Table({ breadcrumbs={breadcrumbs} aside={
- {presenceUsers.length > 0 && } + {presenceUsers.length > 0 && ( + + )} {selection.totalRunning > 0 || selection.hasActiveDispatch ? ( ({ eventId, streamId, value }), + buildEntry: (eventId: number): TestEntry => ({ eventId, streamId, value }), } } diff --git a/apps/sim/lib/realtime/event-log.ts b/apps/sim/lib/realtime/event-log.ts index fd107a774a2..f7b470c33a2 100644 --- a/apps/sim/lib/realtime/event-log.ts +++ b/apps/sim/lib/realtime/event-log.ts @@ -68,14 +68,17 @@ export interface EventLogEntry { /** * How an adapter serializes one entry. `entryPrefix`/`entrySuffix` are spliced - * around the minted `eventId` in Lua (`prefix + eventId + suffix`); `buildMemory` - * must produce the byte-identical object for the in-memory fallback. The two MUST - * agree — a divergence makes dev/no-Redis behave differently from prod. + * around the minted `eventId` in Lua (`prefix + eventId + suffix`) for the Redis + * write; `buildEntry` returns the equivalent object. It is the canonical entry + * builder for BOTH paths — the in-memory fallback AND the Redis success path, + * which returns `buildEntry(eventId)` rather than re-parsing the stored string — + * so it MUST produce the byte-identical shape to the spliced JSON, or dev/no-Redis + * behaves differently from prod. */ export interface EntrySerializer { entryPrefix: string entrySuffix: string - buildMemory: (eventId: number) => E + buildEntry: (eventId: number) => E } export type EventLogReadResult = @@ -142,7 +145,7 @@ export async function appendEvent( if (canUseMemoryBuffer()) { try { const stream = getMemoryStream(config, streamId) - const entry = serializer.buildMemory(stream.nextEventId++) + const entry = serializer.buildEntry(stream.nextEventId++) stream.events.push(entry) if (stream.events.length > config.cap) { stream.events = stream.events.slice(-config.cap) @@ -175,7 +178,7 @@ export async function appendEvent( ) const eventId = typeof result === 'number' ? result : Number(result) if (!Number.isFinite(eventId)) return null - return serializer.buildMemory(eventId) + return serializer.buildEntry(eventId) } catch (error) { logger.warn('appendEvent: Redis append failed', { streamId, error: toError(error).message }) return null diff --git a/apps/sim/lib/table/events.ts b/apps/sim/lib/table/events.ts index 44fd3eebde2..0fda87f6d1f 100644 --- a/apps/sim/lib/table/events.ts +++ b/apps/sim/lib/table/events.ts @@ -141,11 +141,11 @@ export type TableEvent = * lock toggle. Carries no payload; the client just invalidates the * table-detail query so every open viewer re-reads the fresh locks * (otherwise an idle grid stays stale and writes 423). Emitted only by - * {@link signalTableRowsChanged}'s sibling lock path in the service; schema - * and metadata changes use the dedicated `schema`/`metadata` kinds above. */ + * `updateTableLocks` in the service; schema and metadata changes use the + * dedicated `schema`/`metadata` kinds above. */ kind: 'definition' tableId: string - reason: 'locks' | 'schema' + reason: 'locks' } export interface TableEventEntry { @@ -165,7 +165,7 @@ export async function appendTableEvent(event: TableEvent): Promise(TABLE_EVENT_LOG, event.tableId, { entryPrefix: '{"eventId":', entrySuffix: `,"tableId":${JSON.stringify(event.tableId)},"event":${JSON.stringify(event)}}`, - buildMemory: (eventId) => ({ eventId, tableId: event.tableId, event }), + buildEntry: (eventId) => ({ eventId, tableId: event.tableId, event }), }) } diff --git a/apps/sim/lib/table/service.ts b/apps/sim/lib/table/service.ts index 0e17b53be17..5bc24e8f8f6 100644 --- a/apps/sim/lib/table/service.ts +++ b/apps/sim/lib/table/service.ts @@ -684,7 +684,7 @@ export async function updateTableLocks( * @param tableId - Table ID to update * @param metadata - Partial metadata object (merged with existing) * @param existingMetadata - Existing metadata from a prior fetch (avoids redundant DB read) - * @returns Updated metadata + * @returns The merged metadata plus whether the reorder also mutated the schema */ export async function updateTableMetadata( tableId: string, diff --git a/packages/platform-authz/src/rooms.ts b/packages/platform-authz/src/rooms.ts index acb4d2779e5..61b920d9142 100644 --- a/packages/platform-authz/src/rooms.ts +++ b/packages/platform-authz/src/rooms.ts @@ -95,11 +95,6 @@ const ROOM_WORKSPACE_RESOLVERS: Record = { [ROOM_TYPES.TABLE]: resolveTableWorkspace, } -/** Resolves a room's owning workspace, or `null` if the room resource is gone. */ -export function resolveWorkspaceIdForRoom(room: RoomRef): Promise { - return ROOM_WORKSPACE_RESOLVERS[room.type](room.id) -} - export interface RoomAuthorizationResult { allowed: boolean status: number From aeda9b008a6d60bedfab6820dfc96d10dad12a02 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 16:43:57 -0700 Subject: [PATCH 12/53] fix(realtime,tables): close table-presence race + v1/copilot live-collab gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated each issue with subagents before implementing the cleanest fix. - tables LEAVE in-flight-join race (B8): the table handler tracked no current-table intent, so an unscoped/same-table leave during an in-flight authorize left the socket stranded in the room (present in the roster, broadcasting a ghost until disconnect). Mirror workspace-files: a closure-local currentTableId + a leave that advances joinGeneration to cancel the racing join. + 3 regression tests. - v1 + copilot live-collab signal gap (D1): tables edited via the v1 public API or Sim/copilot emitted no edit/schema signal, so open collaborators didn't live-update. Add the signals at those call sites (add-only, matching the existing route seam) — never in the service, so execution writes can't double-emit. Sync-only for copilot bulk ops, guarded on affected/deleted count; async job branches stay covered by their kind:'job' events; create/delete/get untouched. - table join read consolidation (B5): sweepStalePresence returns its roster so the same-tab dedup reuses it instead of a second getRoomUsers. - shared authorize slice (B6): extract only the guard-safe authorize->allowed branch into resolveRoomJoinAuth, shared by the three room handlers (the full preamble stays inline — file-doc's generation capture sits mid-ladder and must not move). - resize-revert flicker (E3): a peer's value-less metadata event forces a refetch that could momentarily revert a just-finished local resize; a pendingWidthWriteRef keeps local widths leading until the width PUT settles. - embedded-mode stray emit (E5): gate emitCellSelection on a bound table id so the embedded surface stops broadcasting cell selections the server drops. --- apps/realtime/src/handlers/file-doc.ts | 41 ++++------ apps/realtime/src/handlers/room-join-auth.ts | 55 ++++++++++++++ apps/realtime/src/handlers/tables.test.ts | 66 ++++++++++++++++ apps/realtime/src/handlers/tables.ts | 75 +++++++++++-------- apps/realtime/src/handlers/workspace-files.ts | 40 ++++------ .../realtime/src/rooms/presence-visibility.ts | 12 ++- .../api/v1/tables/[tableId]/columns/route.ts | 4 + .../v1/tables/[tableId]/rows/[rowId]/route.ts | 3 + .../app/api/v1/tables/[tableId]/rows/route.ts | 6 ++ .../v1/tables/[tableId]/rows/upsert/route.ts | 2 + .../components/table-grid/table-grid.tsx | 29 +++++-- .../tables/[tableId]/hooks/use-table-room.ts | 6 ++ .../copilot/tools/server/table/user-table.ts | 21 ++++++ 13 files changed, 269 insertions(+), 91 deletions(-) create mode 100644 apps/realtime/src/handlers/room-join-auth.ts diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 31c9d562cfc..1ca39540ceb 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -19,7 +19,6 @@ * @module */ import { createLogger } from '@sim/logger' -import { authorizeRoom } from '@sim/platform-authz/rooms' import { FILE_DOC_EVENTS, FILE_DOC_MESSAGE_TYPE, @@ -37,6 +36,7 @@ import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' import { resolveAvatarUrl } from '@/handlers/avatar' +import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -432,30 +432,21 @@ export function setupWorkspaceFileDocHandlers( const room = fileDocRoom(fileId) const name = roomName(room) - let authorized: Awaited> - try { - authorized = await authorizeRoom({ userId, room, action: 'write' }) - } catch (error) { - logger.warn(`Error authorizing file-doc room for ${userId}:`, error) - emitJoinError( - socket, - fileId, - 'Failed to verify workspace access', - 'VERIFY_ACCESS_FAILED', - true - ) - return - } - if (!authorized.allowed) { - emitJoinError( - socket, - fileId, - authorized.status === 404 ? 'File not found' : 'Access denied to file', - authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', - false - ) - return - } + const authorized = await resolveRoomJoinAuth({ + userId, + room, + action: 'write', + logger, + logLabel: `file-doc room for ${userId}`, + messages: { + verifyFailed: 'Failed to verify workspace access', + notFound: 'File not found', + accessDenied: 'Access denied to file', + }, + emitError: ({ error, code, retryable }) => + emitJoinError(socket, fileId, error, code, retryable), + }) + if (!authorized) return // Server-authenticated identity for the presence roster (never trusts the client-set // awareness). Resolved here so the generation guard below also covers this await. diff --git a/apps/realtime/src/handlers/room-join-auth.ts b/apps/realtime/src/handlers/room-join-auth.ts new file mode 100644 index 00000000000..a7c2a9aa24c --- /dev/null +++ b/apps/realtime/src/handlers/room-join-auth.ts @@ -0,0 +1,55 @@ +import type { createLogger } from '@sim/logger' +import { authorizeRoom } from '@sim/platform-authz/rooms' +import type { RoomRef } from '@sim/realtime-protocol/rooms' + +type Authorized = Awaited> + +interface ResolveRoomJoinAuthParams { + userId: string + room: RoomRef + action: 'read' | 'write' + logger: ReturnType + /** Included in the warn log on an authorize throw, e.g. `table room for ${userId}`. */ + logLabel: string + messages: { verifyFailed: string; notFound: string; accessDenied: string } + /** Emits the handler's own JOIN_ERROR shape (event name + id key differ per handler). */ + emitError: (args: { error: string; code: string; retryable: boolean }) => void +} + +/** + * Runs the shared authorize→allowed slice of a room join: authorizes the room and checks + * the result, emitting the handler-specific JOIN_ERROR on failure. Returns the authorized + * result on success, or `null` when it has already emitted an error and the caller must return. + * + * Deliberately excludes the auth/readiness/id-validation preamble and the join-generation + * capture/recheck — those differ per handler, and for file-doc the generation capture sits + * mid-preamble. This helper is always invoked strictly between a handler's generation capture + * and its post-authorize recheck; it contains exactly the one `await authorizeRoom` that the + * recheck was designed to cover and returns before any state mutation, so it never straddles + * that seam. Pass each handler's own `logger` so the log namespace/request-id context is kept. + */ +export async function resolveRoomJoinAuth( + params: ResolveRoomJoinAuthParams +): Promise { + const { userId, room, action, logger, logLabel, messages, emitError } = params + + let authorized: Authorized + try { + authorized = await authorizeRoom({ userId, room, action }) + } catch (error) { + logger.warn(`Error authorizing ${logLabel}:`, error) + emitError({ error: messages.verifyFailed, code: 'VERIFY_ACCESS_FAILED', retryable: true }) + return null + } + + if (!authorized.allowed) { + emitError({ + error: authorized.status === 404 ? messages.notFound : messages.accessDenied, + code: authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', + retryable: false, + }) + return null + } + + return authorized +} diff --git a/apps/realtime/src/handlers/tables.test.ts b/apps/realtime/src/handlers/tables.test.ts index 2e47099f09a..1b09298b1c9 100644 --- a/apps/realtime/src/handlers/tables.test.ts +++ b/apps/realtime/src/handlers/tables.test.ts @@ -218,6 +218,72 @@ describe('setupTablesHandlers', () => { ) }) + it('aborts an in-flight join when a leave for that table arrives during authorize', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + let releaseAuth: (value: unknown) => void = () => {} + const pending = new Promise((resolve) => { + releaseAuth = resolve + }) + mockAuthorizeRoom.mockReturnValueOnce(pending) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const joinPromise = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + // Client navigates away while the join is still awaiting authorization. + await handlers[TABLE_PRESENCE_EVENTS.LEAVE]({ tableId: 'table-1' }) + releaseAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + // The cancelled join must touch no room state — the socket is not stranded. + expect(socket.join).not.toHaveBeenCalled() + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + expect(roomManager.broadcastPresenceUpdate).not.toHaveBeenCalledWith(TABLE_ROOM) + }) + + it('aborts an in-flight join when an unscoped leave arrives during authorize', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + let releaseAuth: (value: unknown) => void = () => {} + const pending = new Promise((resolve) => { + releaseAuth = resolve + }) + mockAuthorizeRoom.mockReturnValueOnce(pending) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const joinPromise = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + // A leave with no table id (view teardown) must also cancel the in-flight join. + await handlers[TABLE_PRESENCE_EVENTS.LEAVE](undefined) + releaseAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + expect(socket.join).not.toHaveBeenCalled() + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + }) + + it('does not abort an in-flight join when a leave targets a different table', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + let releaseAuth: (value: unknown) => void = () => {} + const pending = new Promise((resolve) => { + releaseAuth = resolve + }) + mockAuthorizeRoom.mockReturnValueOnce(pending) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const joinPromise = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) + // A stale/deferred leave for a table the client already left must NOT cancel the B join. + await handlers[TABLE_PRESENCE_EVENTS.LEAVE]({ tableId: 'table-A' }) + releaseAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + expect(socket.join).toHaveBeenCalledWith('table:table-B') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-B' }, + 'socket-1', + expect.anything() + ) + }) + it('leaves the table room on leave', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager({ diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index d1fcdd632c2..efbe3f1dea1 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -1,5 +1,4 @@ import { createLogger } from '@sim/logger' -import { authorizeRoom } from '@sim/platform-authz/rooms' import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' import { type JoinTablePayload, @@ -8,6 +7,7 @@ import { type TableCellSelection, } from '@sim/realtime-protocol/table-presence' import { resolveAvatarUrl } from '@/handlers/avatar' +import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager, UserPresence } from '@/rooms' import { filterVisiblePresence, sweepStalePresence } from '@/rooms/presence-visibility' @@ -66,9 +66,16 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // authorize, aborts if a newer JOIN has started — a fast table switch A→B can otherwise // let A's late handler leave B and strand the socket in room A while the client views B. let joinGeneration = 0 + // The table the socket currently intends to be in (set when a join starts). A leave + // targeting it — or an unscoped leave — advances joinGeneration to cancel an in-flight + // join, so a join still awaiting authorization can't complete after the client left and + // strand the socket in the room (present in presence + still receiving broadcasts). A + // leave for a DIFFERENT table must NOT cancel it (a table switch), mirroring workspace-files. + let currentTableId: string | null = null socket.on(TABLE_PRESENCE_EVENTS.JOIN, async ({ tableId, tabSessionId }: JoinTablePayload) => { const joinAttempt = (joinGeneration += 1) + currentTableId = tableId try { const userId = socket.userId const userName = socket.userName @@ -106,29 +113,21 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR const room = tableRoom(tableId) - let authorized: Awaited> - try { - authorized = await authorizeRoom({ userId, room, action: 'read' }) - } catch (error) { - logger.warn(`Error authorizing table room for ${userId}:`, error) - socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { - tableId, - error: 'Failed to verify table access', - code: 'VERIFY_ACCESS_FAILED', - retryable: true, - }) - return - } - - if (!authorized.allowed) { - socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { - tableId, - error: authorized.status === 404 ? 'Table not found' : 'Access denied to table', - code: authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', - retryable: false, - }) - return - } + const authorized = await resolveRoomJoinAuth({ + userId, + room, + action: 'read', + logger, + logLabel: `table room for ${userId}`, + messages: { + verifyFailed: 'Failed to verify table access', + notFound: 'Table not found', + accessDenied: 'Access denied to table', + }, + emitError: ({ error, code, retryable }) => + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { tableId, error, code, retryable }), + }) + if (!authorized) return // A newer JOIN started on this socket during authorize (or the socket dropped): // abort so a stale join can't leave the room the client has since moved to. @@ -142,11 +141,16 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR await roomManager.broadcastPresenceUpdate(currentRoom) } - // Clean up the same user's stale socket from the same tab (a reconnect that - // raced the old socket's disconnect), so presence shows one entry. + // Reclaim presence orphaned by an ungraceful disconnect (no `disconnecting` + // event fires on a pod crash; the room hashes have no TTL). Returns the roster it + // read so the same-tab dedup below reuses it instead of issuing a second read. + const roster = await sweepStalePresence(roomManager, room) + + // Clean up the same user's stale socket from the same tab (a reconnect that raced + // the old socket's disconnect), so presence shows one entry. Reuses the sweep's + // roster snapshot; re-removing an already-swept entry is a harmless no-op. if (tabSessionId) { - const existingUsers = await roomManager.getRoomUsers(room) - for (const existing of existingUsers) { + for (const existing of roster) { if ( existing.socketId !== socket.id && existing.userId === userId && @@ -158,10 +162,6 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR } } - // Reclaim presence orphaned by an ungraceful disconnect (no `disconnecting` - // event fires on a pod crash; the room hashes have no TTL). - await sweepStalePresence(roomManager, room) - socket.join(roomName(room)) const presence: UserPresence = { @@ -214,6 +214,17 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR socket.on(TABLE_PRESENCE_EVENTS.LEAVE, async (payload?: { tableId?: string }) => { try { + // Cancel an in-flight join whose table the client is now leaving (or an unscoped + // leave): a join still awaiting authorization would otherwise complete after the + // client left — joining the room, registering presence, and broadcasting a ghost + // until disconnect. Guard on the current table intent so a stale/deferred leave for + // a DIFFERENT table can't abort the join the client has since switched to. Runs + // before the teardown below because the racing join has registered nothing yet + // (getRoomForSocket returns null), so only this generation bump can stop it. + if (!payload?.tableId || payload.tableId === currentTableId) { + joinGeneration += 1 + currentTableId = null + } if (!roomManager.isReady()) return const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) if (!room) return diff --git a/apps/realtime/src/handlers/workspace-files.ts b/apps/realtime/src/handlers/workspace-files.ts index fa3b56111fa..a4b0604dc9f 100644 --- a/apps/realtime/src/handlers/workspace-files.ts +++ b/apps/realtime/src/handlers/workspace-files.ts @@ -1,6 +1,6 @@ import { createLogger } from '@sim/logger' -import { authorizeRoom } from '@sim/platform-authz/rooms' import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -81,29 +81,21 @@ export function setupWorkspaceFilesHandlers( const room = filesRoom(workspaceId) - let authorized: Awaited> - try { - authorized = await authorizeRoom({ userId: socket.userId, room, action: 'read' }) - } catch (error) { - logger.warn(`Error authorizing files room for ${socket.userId}:`, error) - socket.emit('join-workspace-files-error', { - workspaceId, - error: 'Failed to verify workspace access', - code: 'VERIFY_ACCESS_FAILED', - retryable: true, - }) - return - } - - if (!authorized.allowed) { - socket.emit('join-workspace-files-error', { - workspaceId, - error: authorized.status === 404 ? 'Workspace not found' : 'Access denied to workspace', - code: authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', - retryable: false, - }) - return - } + const authorized = await resolveRoomJoinAuth({ + userId: socket.userId, + room, + action: 'read', + logger, + logLabel: `files room for ${socket.userId}`, + messages: { + verifyFailed: 'Failed to verify workspace access', + notFound: 'Workspace not found', + accessDenied: 'Access denied to workspace', + }, + emitError: ({ error, code, retryable }) => + socket.emit('join-workspace-files-error', { workspaceId, error, code, retryable }), + }) + if (!authorized) return // A newer join started on this socket during authorize (or it dropped): abort so a // stale join can't leave the room the client has since switched to. diff --git a/apps/realtime/src/rooms/presence-visibility.ts b/apps/realtime/src/rooms/presence-visibility.ts index 7e43b9fde29..0d2eb6b91bd 100644 --- a/apps/realtime/src/rooms/presence-visibility.ts +++ b/apps/realtime/src/rooms/presence-visibility.ts @@ -1,6 +1,6 @@ import { type RoomRef, roomName } from '@sim/realtime-protocol/rooms' import type { Server } from 'socket.io' -import type { IRoomManager } from '@/rooms/types' +import type { IRoomManager, UserPresence } from '@/rooms/types' /** * How stale a not-live presence entry must be before a join-time sweep reclaims @@ -53,13 +53,16 @@ export async function filterVisiblePresence( * Run on join, like the workflow room does. No-op when the liveness lookup fails * (so a transient adapter blip can't evict live collaborators). */ -export async function sweepStalePresence(manager: IRoomManager, room: RoomRef): Promise { +export async function sweepStalePresence( + manager: IRoomManager, + room: RoomRef +): Promise { let liveIds: Set try { const liveSockets = await manager.io.in(roomName(room)).fetchSockets() liveIds = new Set(liveSockets.map((socket) => socket.id)) } catch { - return + return [] } const now = Date.now() @@ -71,4 +74,7 @@ export async function sweepStalePresence(manager: IRoomManager, room: RoomRef): await manager.removeUserFromRoom(room, user.socketId) } } + // Return the pre-removal roster so a caller can reuse it (e.g. same-tab dedup) instead + // of re-reading; re-removing an already-swept entry downstream is a harmless no-op. + return users } diff --git a/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts b/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts index 46723538c7c..df249de280d 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts @@ -18,6 +18,7 @@ import { updateColumnType, } from '@/lib/table' import { columnMatchesRef } from '@/lib/table/column-keys' +import { signalTableSchemaChanged } from '@/lib/table/events' import { accessError, checkAccess, @@ -69,6 +70,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Colum } const updatedTable = await addTableColumn(tableId, validated.column, requestId) + signalTableSchemaChanged(tableId) recordAudit({ workspaceId: validated.workspaceId, @@ -218,6 +220,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu if (!updatedTable) { return NextResponse.json({ error: 'No updates specified' }, { status: 400 }) } + signalTableSchemaChanged(tableId) recordAudit({ workspaceId: validated.workspaceId, @@ -301,6 +304,7 @@ export const DELETE = withRouteHandler( { tableId, columnName: validated.columnName }, requestId ) + signalTableSchemaChanged(tableId) recordAudit({ workspaceId: validated.workspaceId, diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts index 2a7ea2fe7a5..b9a47ab5726 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts @@ -16,6 +16,7 @@ import type { RowData, TableSchema } from '@/lib/table' import { deleteRow, updateRow } from '@/lib/table' import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, rowDataNameToId } from '@/lib/table/column-keys' +import { signalTableRowsChanged } from '@/lib/table/events' import { accessError, checkAccess, tableLockErrorResponse } from '@/app/api/table/utils' import { checkRateLimit, @@ -155,6 +156,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: RowR if (!updatedRow) { return NextResponse.json({ error: 'Row not found' }, { status: 404 }) } + signalTableRowsChanged(tableId) // Auto-dispatch for user edits is handled inside `updateRow` (mode: 'new'). // Firing a second mode: 'incomplete' dispatch here would race with it AND // bulk-clear sibling-group outputs. @@ -237,6 +239,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Row // Route through the service (not a raw `db.delete`) so the delete lock is // enforced — the raw path would return 200 on a locked table. await deleteRow(result.table, rowId, requestId) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts index d0e37376cca..26544913472 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts @@ -32,6 +32,7 @@ import { rowDataNameToId, sortNamesToIds, } from '@/lib/table/column-keys' +import { signalTableRowsChanged } from '@/lib/table/events' import { queryRows } from '@/lib/table/rows/service' import { resolveFilterSelectValues } from '@/lib/table/select-values' import { TableQueryValidationError } from '@/lib/table/sql' @@ -91,6 +92,7 @@ async function handleBatchInsert( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -278,6 +280,7 @@ export const POST = withRouteHandler( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -361,6 +364,7 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: TableR }, requestId ) + if (result.affectedCount > 0) signalTableRowsChanged(tableId) if (result.affectedCount === 0) { return NextResponse.json({ @@ -431,6 +435,7 @@ export const DELETE = withRouteHandler( { tableId, rowIds: validated.rowIds, workspaceId: validated.workspaceId }, requestId ) + if (result.deletedCount > 0) signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -459,6 +464,7 @@ export const DELETE = withRouteHandler( }, requestId ) + if (result.affectedCount > 0) signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/upsert/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/upsert/route.ts index 1df6b4b2384..2097073f473 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/upsert/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/upsert/route.ts @@ -9,6 +9,7 @@ import type { RowData, TableSchema } from '@/lib/table' import { upsertRow } from '@/lib/table' import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, rowDataNameToId } from '@/lib/table/column-keys' +import { signalTableRowsChanged } from '@/lib/table/events' import { accessError, checkAccess, tableLockErrorResponse } from '@/app/api/table/utils' import { checkRateLimit, @@ -71,6 +72,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Upser table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index 68133c595c1..11b1ee826ca 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -415,6 +415,11 @@ export function TableGrid({ const [resizingColumn, setResizingColumn] = useState(null) const resizingColumnRef = useRef(resizingColumn) resizingColumnRef.current = resizingColumn + /** True from a committed local width change until its metadata PUT settles. Keeps a + * concurrent peer-triggered definition refetch (the value-less `metadata` event forces a + * refetch that can carry the not-yet-persisted server widths) from momentarily reverting + * the just-written widths. */ + const pendingWidthWriteRef = useRef(false) const [columnOrder, setColumnOrder] = useState(null) const columnOrderRef = useRef(columnOrder) columnOrderRef.current = columnOrder @@ -1551,7 +1556,11 @@ export function TableGrid({ const handleColumnResizeEnd = useCallback(() => { setResizingColumn(null) - updateMetadataRef.current({ columnWidths: columnWidthsRef.current }) + pendingWidthWriteRef.current = true + updateMetadataRef.current( + { columnWidths: columnWidthsRef.current }, + { onSettled: () => (pendingWidthWriteRef.current = false) } + ) }, []) const handleColumnAutoResize = useCallback((columnKey: string) => { @@ -1612,7 +1621,11 @@ export function TableGrid({ setColumnWidths((prev) => ({ ...prev, [columnKey]: newWidth })) const updated = { ...columnWidthsRef.current, [columnKey]: newWidth } columnWidthsRef.current = updated - updateMetadataRef.current({ columnWidths: updated }) + pendingWidthWriteRef.current = true + updateMetadataRef.current( + { columnWidths: updated }, + { onSettled: () => (pendingWidthWriteRef.current = false) } + ) }, []) const handleColumnDragStart = useCallback((columnName: string) => { @@ -1916,11 +1929,13 @@ export function TableGrid({ if (serverWidths && serverWidths !== columnWidthsRef.current) { const resizing = resizingColumnRef.current const localWidth = resizing ? columnWidthsRef.current[resizing] : undefined - setColumnWidths( - resizing && localWidth !== undefined - ? { ...serverWidths, [resizing]: localWidth } - : serverWidths - ) + if (resizing && localWidth !== undefined) { + setColumnWidths({ ...serverWidths, [resizing]: localWidth }) + } else if (!pendingWidthWriteRef.current) { + setColumnWidths(serverWidths) + } + // else: a just-committed local width write is still in flight — local leads until + // its onSettled invalidation brings back the server's committed (merged) widths. } // Pins toggle instantly (no in-progress gesture) — apply on change. const serverPins = tableData.metadata.pinnedColumns diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts index 24d0b99f772..30d8d3fadcd 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts @@ -147,6 +147,9 @@ export function useTableRoom(tableId: string): UseTableRoomResult { const socketRef = useRef(socket) socketRef.current = socket + /** Presence is disabled when no table id is bound (e.g. the embedded/mothership surface). */ + const enabledRef = useRef(false) + enabledRef.current = Boolean(tableId) const lastEmitRef = useRef(0) const trailingTimerRef = useRef | null>(null) const pendingCellRef = useRef(null) @@ -170,6 +173,9 @@ export function useTableRoom(tableId: string): UseTableRoomResult { ) const emitCellSelection = useCallback((cell: TableCellSelection) => { + // No room joined (empty tableId, e.g. embedded mode) — never broadcast; the server + // would drop it anyway. Local selection UI is unaffected (grid-owned state). + if (!enabledRef.current) return // Skip re-emitting an unchanged selection: the caller re-resolves on every data // refetch (so a peer's row insert re-broadcasts the shifted rowId), but most refetches // don't move the selection — dedup those, and the no-selection state on table open. diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index a88e1dee903..913d6048617 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -45,6 +45,7 @@ import { updateColumnType, } from '@/lib/table/columns/service' import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner' +import { signalTableRowsChanged, signalTableSchemaChanged } from '@/lib/table/events' import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { assertRowDelete, assertRowUpdate, patchColumnIds } from '@/lib/table/mutation-locks' @@ -557,6 +558,7 @@ export const userTableServerTool: BaseServerTool table, requestId ) + signalTableRowsChanged(args.tableId) return { success: true, @@ -600,6 +602,7 @@ export const userTableServerTool: BaseServerTool table, requestId ) + signalTableRowsChanged(args.tableId) return { success: true, @@ -743,6 +746,7 @@ export const userTableServerTool: BaseServerTool // doesn't, so the guard never trips here. Defensive narrowing. return { success: false, message: 'Row update was skipped' } } + signalTableRowsChanged(args.tableId) // Auto-dispatch for user edits is handled inside `updateRow` // (mode: 'new' for newly-cleared groups + cancel+rerun for in-flight // downstream groups). Firing a second mode: 'incomplete' dispatch @@ -782,6 +786,7 @@ export const userTableServerTool: BaseServerTool return { success: false, message: `Table ${args.tableId} not found` } } await deleteRow(deleteRowTable, args.rowId, requestId) + signalTableRowsChanged(args.tableId) return { success: true, @@ -885,6 +890,7 @@ export const userTableServerTool: BaseServerTool }, requestId ) + if (result.affectedCount > 0) signalTableRowsChanged(args.tableId) return { success: true, @@ -990,6 +996,7 @@ export const userTableServerTool: BaseServerTool } finally { await releaseJobClaim(table.id, inlineDeleteId).catch(() => {}) } + if (result.affectedCount > 0) signalTableRowsChanged(args.tableId) recordAudit({ workspaceId, @@ -1112,6 +1119,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, rowIds, workspaceId }, requestId ) + if (result.deletedCount > 0) signalTableRowsChanged(args.tableId) recordAudit({ workspaceId, @@ -1420,6 +1428,7 @@ export const userTableServerTool: BaseServerTool table, requestId ) + signalTableRowsChanged(table.id) logger.info('Rows replaced from file', { tableId: table.id, @@ -1511,6 +1520,7 @@ export const userTableServerTool: BaseServerTool ? { ...col, options: normalizeSelectOptionsInput(col.options) } : { ...col, options: undefined } const updated = await addTableColumn(args.tableId, columnToAdd, requestId) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Added column "${col.name}" (${col.type}) to table`, @@ -1540,6 +1550,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, oldName: colName, newName: newColName }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Renamed column "${colName}" to "${newColName}"`, @@ -1571,6 +1582,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, columnName: names[0] }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Deleted column "${names[0]}"`, @@ -1582,6 +1594,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, columnNames: names }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Deleted ${names.length} columns: ${names.join(', ')}`, @@ -1687,6 +1700,7 @@ export const userTableServerTool: BaseServerTool requestId ) } + if (result) signalTableSchemaChanged(args.tableId) return { success: true, message: `Updated column "${colName}"`, @@ -1716,6 +1730,7 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) assertNotAborted() const renamed = await renameTable(args.tableId, newName, requestId, context.userId) + signalTableSchemaChanged(args.tableId) return { success: true, @@ -1839,6 +1854,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, group, outputColumns, autoRun, actorUserId: context.userId }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Added workflow group "${name ?? groupId}" with ${outputs.length} output column(s)`, @@ -1911,6 +1927,7 @@ export const userTableServerTool: BaseServerTool }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Updated workflow group ${groupId}`, @@ -1932,6 +1949,7 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) assertNotAborted() const updated = await deleteWorkflowGroup({ tableId: args.tableId, groupId }, requestId) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Deleted workflow group ${groupId}`, @@ -1969,6 +1987,7 @@ export const userTableServerTool: BaseServerTool }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Added output to workflow group ${groupId}`, @@ -1997,6 +2016,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, groupId, columnName }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Removed output "${columnName}" from workflow group ${groupId}`, @@ -2210,6 +2230,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, group, outputColumns, autoRun, actorUserId: context.userId }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Added enrichment "${name}" with ${outputs.length} output column(s)${ From a7aa317b4d1cbc23beb3e60f9662c267ac0cd4ec Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 16:53:24 -0700 Subject: [PATCH 13/53] fix(tables): close two copilot live-collab signal gaps + harden presence sweep Follow-ups from a comprehensive review of the branch: - copilot batch_update_rows and import_file's inline append branch wrote rows but emitted no live-collab signal, so collaborators didn't see those edits live (the append's sibling replace branch already signalled). Add the guarded signal to both, matching the internal route. - sweepStalePresence now reads the roster before the fetchSockets liveness probe and returns it on a probe failure, so same-tab dedup still runs during a transient fetchSockets outage instead of being skipped. - reword an internal comment off the retired "mothership" term. --- apps/realtime/src/rooms/presence-visibility.ts | 7 +++++-- .../[workspaceId]/tables/[tableId]/hooks/use-table-room.ts | 2 +- apps/sim/lib/copilot/tools/server/table/user-table.ts | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/realtime/src/rooms/presence-visibility.ts b/apps/realtime/src/rooms/presence-visibility.ts index 0d2eb6b91bd..407278850b6 100644 --- a/apps/realtime/src/rooms/presence-visibility.ts +++ b/apps/realtime/src/rooms/presence-visibility.ts @@ -57,16 +57,19 @@ export async function sweepStalePresence( manager: IRoomManager, room: RoomRef ): Promise { + // Read the roster first so it is returned to the caller (for same-tab dedup) even when the + // liveness probe below fails — a fetchSockets outage must skip only the stale-removal, never + // the caller's dedup. + const users = await manager.getRoomUsers(room) let liveIds: Set try { const liveSockets = await manager.io.in(roomName(room)).fetchSockets() liveIds = new Set(liveSockets.map((socket) => socket.id)) } catch { - return [] + return users } const now = Date.now() - const users = await manager.getRoomUsers(room) for (const user of users) { if (liveIds.has(user.socketId)) continue const lastSeen = user.lastActivity || user.joinedAt || 0 diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts index 30d8d3fadcd..72e7c7036ce 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts @@ -147,7 +147,7 @@ export function useTableRoom(tableId: string): UseTableRoomResult { const socketRef = useRef(socket) socketRef.current = socket - /** Presence is disabled when no table id is bound (e.g. the embedded/mothership surface). */ + /** Presence is disabled when no table id is bound (e.g. the embedded Chat panel surface). */ const enabledRef = useRef(false) enabledRef.current = Boolean(tableId) const lastEmitRef = useRef(0) diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index 913d6048617..20fd16cc67c 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -1080,6 +1080,7 @@ export const userTableServerTool: BaseServerTool table, requestId ) + if (result.affectedCount > 0) signalTableRowsChanged(args.tableId) return { success: true, @@ -1457,6 +1458,7 @@ export const userTableServerTool: BaseServerTool } const inserted = await batchInsertAll(table.id, coerced, table, workspaceId, context) + if (inserted > 0) signalTableRowsChanged(table.id) logger.info('Rows imported from file', { tableId: table.id, From 869679ff471ad78a08eac804fc91a1234504d13f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:14:14 -0700 Subject: [PATCH 14/53] fix(realtime): guard table join commit + rollback against supersession; drop no-op eviction cleanup Review round on #5991: - Table join re-checked the generation only once after authorize, then awaited leave/sweep/avatar before joining + registering presence. A table switch or leave in that window stranded the socket in the wrong room, and the failure catch could tear down a newer successful join. Resolve the avatar up-front, re-check generation immediately before the membership commit (matching the file-doc join), and skip the rollback/error for a superseded join. + a post-authorize-window regression test. - access-revalidation cleanup treated removeUserFromRoom's no-op false as a transport failure and re-enqueued a still-connected socket forever. Only retry when the socket is still mapped to the room (a healthy null mapping means the entry is already gone). Repurposed the expired-mapping test to lock it. --- apps/realtime/src/access-revalidation.test.ts | 20 ++++---- apps/realtime/src/access-revalidation.ts | 17 +++++-- apps/realtime/src/handlers/tables.test.ts | 47 +++++++++++++++++++ apps/realtime/src/handlers/tables.ts | 18 ++++++- 4 files changed, 86 insertions(+), 16 deletions(-) diff --git a/apps/realtime/src/access-revalidation.test.ts b/apps/realtime/src/access-revalidation.test.ts index 8d9d37cf349..4ea7999c301 100644 --- a/apps/realtime/src/access-revalidation.test.ts +++ b/apps/realtime/src/access-revalidation.test.ts @@ -226,25 +226,25 @@ describe('access-revalidation sweep', () => { expect(manager.broadcastPresenceUpdate).toHaveBeenCalledWith({ type: 'workflow', id: 'wf-1' }) }) - it('defers cleanup when removal fails with expired socket mappings', async () => { + it('drops eviction cleanup when the socket is no longer mapped to the room (no infinite retry)', async () => { const socket = makeSocket('sock-1', 'user-1', 'wf-1') const manager = makeManager([socket], [{ socketId: 'sock-1', role: 'read' }]) - // Mapping keys already expired (lookup resolves null) AND the removal fails - // (the Redis manager swallows the transport error into null) — the failed - // removal must still defer instead of reading as success. - manager.removeUserFromRoom.mockResolvedValueOnce(false) + // A healthy lookup shows the socket is no longer mapped to any workflow room (its presence + // is already gone), and removeUserFromRoom reports a no-op `false`. This is "already clean", + // not a deferrable failure — the cleanup must drop it, never re-enqueue a still-connected + // socket forever. (A genuine failure — still mapped + false — is covered by the next test.) + manager.getRoomForSocket.mockResolvedValue(null) + manager.removeUserFromRoom.mockResolvedValue(false) mockResolveRole.mockResolvedValue(null) const sweep = startAccessRevalidationSweep(manager) await sweep.runOnce() - - expect(manager.broadcastPresenceUpdate).not.toHaveBeenCalled() - await sweep.runOnce() sweep.stop() - expect(manager.removeUserFromRoom).toHaveBeenCalledTimes(2) - expect(manager.broadcastPresenceUpdate).toHaveBeenCalledWith({ type: 'workflow', id: 'wf-1' }) + // Attempted once, then dropped — not re-enqueued across passes, and no broadcast. + expect(manager.removeUserFromRoom).toHaveBeenCalledTimes(1) + expect(manager.broadcastPresenceUpdate).not.toHaveBeenCalled() }) it('defers cleanup when the manager swallows a removal failure into null', async () => { diff --git a/apps/realtime/src/access-revalidation.ts b/apps/realtime/src/access-revalidation.ts index 971a12de4b5..93edaae9a0b 100644 --- a/apps/realtime/src/access-revalidation.ts +++ b/apps/realtime/src/access-revalidation.ts @@ -172,11 +172,18 @@ export function startAccessRevalidationSweep(roomManager: IRoomManager): AccessR // entry from the known target room via the explicit ref below. const removed = await roomManager.removeUserFromRoom(wf(workflowId), socketId) if (!removed) { - // The socket is still connected and hasn't moved/re-joined, yet the removal - // wasn't confirmed — a transport error the manager swallowed into false, or a - // lost race. Defer and retry next sweep rather than leave a revoked - // collaborator's stale presence entry behind. - throw new Error('room-state removal not confirmed') + // `false` conflates two outcomes: the entry was already gone (a no-op), or a + // transport error the manager swallowed. Only retry when the socket is still mapped + // to THIS room — then a false result is a genuine, deferrable failure. When a healthy + // getRoomForSocket above returned no workflow mapping (`currentWorkflowId === null`), + // the presence entry is already gone, so the cleanup is complete: dropping it avoids + // re-enqueuing a still-connected socket forever. (A real Redis outage throws at + // getRoomForSocket and is deferred by the outer catch, never reaching here.) + if (currentWorkflowId === workflowId) { + throw new Error('room-state removal not confirmed') + } + pendingCleanups.delete(key) + return } await roomManager.broadcastPresenceUpdate(wf(workflowId)) diff --git a/apps/realtime/src/handlers/tables.test.ts b/apps/realtime/src/handlers/tables.test.ts index 1b09298b1c9..cf4413128ec 100644 --- a/apps/realtime/src/handlers/tables.test.ts +++ b/apps/realtime/src/handlers/tables.test.ts @@ -284,6 +284,53 @@ describe('setupTablesHandlers', () => { ) }) + it('aborts a join superseded during the post-authorize leave/sweep window', async () => { + const { socket, handlers } = createSocket() + // Hold A hung on its leave-prior lookup (which runs AFTER the post-authorize recheck), then + // fire a newer join B. When A resumes, the final generation guard before the membership + // commit must abort it — the post-authorize awaits are no longer an unguarded window. + let aReachedLookup: () => void = () => {} + const aAtLookup = new Promise((resolve) => { + aReachedLookup = resolve + }) + let releaseLookup: (value: unknown) => void = () => {} + const pendingLookup = new Promise((resolve) => { + releaseLookup = resolve + }) + let lookupCalls = 0 + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn(() => { + lookupCalls += 1 + if (lookupCalls === 1) { + aReachedLookup() + return pendingLookup + } + return Promise.resolve(null) + }), + }) + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const joinA = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-A' }) + await aAtLookup // A has passed its post-authorize recheck and is hung on the leave-prior lookup + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) // bumps generation, completes + releaseLookup(null) + await joinA + + expect(socket.join).toHaveBeenCalledWith('table:table-B') + expect(socket.join).not.toHaveBeenCalledWith('table:table-A') + expect(roomManager.addUserToRoom).not.toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-A' }, + expect.anything(), + expect.anything() + ) + }) + it('leaves the table room on leave', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager({ diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index efbe3f1dea1..83929c24773 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -131,6 +131,12 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // A newer JOIN started on this socket during authorize (or the socket dropped): // abort so a stale join can't leave the room the client has since moved to. + // Server-authenticated avatar for the presence roster. Resolved up-front so the guard + // below also covers this await (mirrors the file-doc join). + const avatarUrl = await resolveAvatarUrl(socket, userId) + + // Abort a JOIN superseded during authorize/avatar resolution — a newer JOIN (table + // switch), a LEAVE, or a disconnect. Registering below would strand the socket. if (joinGeneration !== joinAttempt || socket.disconnected) return // Leave a previously-joined table room if switching tables. @@ -162,6 +168,12 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR } } + // Final re-check immediately before the membership commit: a newer JOIN (table switch), a + // LEAVE, or a disconnect during the leave/sweep awaits above must abort here — otherwise + // this superseded join would join the room and register presence, stranding the socket in + // the wrong table. No await sits between this guard and addUserToRoom (the commit). + if (joinGeneration !== joinAttempt || socket.disconnected) return + socket.join(roomName(room)) const presence: UserPresence = { @@ -173,7 +185,7 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR joinedAt: Date.now(), lastActivity: Date.now(), role: authorized.workspacePermission ?? 'read', - avatarUrl: await resolveAvatarUrl(socket, userId), + avatarUrl, } await roomManager.addUserToRoom(room, socket.id, presence) @@ -196,6 +208,10 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR logger.info(`User ${userId} (${userName}) joined table room ${tableId}`) } catch (error) { logger.error('Error joining table room:', error) + // A superseded join (a newer join/leave bumped the generation) must NOT roll back — it + // would tear down room state a newer successful join to the same table now holds — nor + // signal an error for a table the client already left. + if (joinGeneration !== joinAttempt) return // Roll back any partial join so a failed attempt can't leave the socket in the // Socket.IO room or a stale presence entry behind, before signalling a retry. try { From dee847b2afe6a235751388a2cdc772df313616e2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:22:56 -0700 Subject: [PATCH 15/53] fix(realtime): guard table join leave-prior against superseding join MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 on #5991: a superseded join's leave-prior could still run — during its getRoomForSocket await a newer join commits to its room, so currentRoom is that newer room and the superseded join would leave/remove/broadcast it before the final guard aborts. Re-check the generation immediately after the lookup await, before the leave mutation. Extended the post-authorize-window test to assert the superseded join never tears down the newer join's room. --- apps/realtime/src/handlers/tables.test.ts | 17 ++++++++++++----- apps/realtime/src/handlers/tables.ts | 6 +++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/apps/realtime/src/handlers/tables.test.ts b/apps/realtime/src/handlers/tables.test.ts index cf4413128ec..3a8abcefff3 100644 --- a/apps/realtime/src/handlers/tables.test.ts +++ b/apps/realtime/src/handlers/tables.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { ROOM_TYPES, type RoomRef } from '@sim/realtime-protocol/rooms' import { TABLE_PRESENCE_EVENTS } from '@sim/realtime-protocol/table-presence' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { IRoomManager } from '@/rooms' @@ -293,13 +293,13 @@ describe('setupTablesHandlers', () => { const aAtLookup = new Promise((resolve) => { aReachedLookup = resolve }) - let releaseLookup: (value: unknown) => void = () => {} - const pendingLookup = new Promise((resolve) => { + let releaseLookup: (value: RoomRef | null) => void = () => {} + const pendingLookup = new Promise((resolve) => { releaseLookup = resolve }) let lookupCalls = 0 const roomManager = createRoomManager({ - getRoomForSocket: vi.fn(() => { + getRoomForSocket: vi.fn((): Promise => { lookupCalls += 1 if (lookupCalls === 1) { aReachedLookup() @@ -319,9 +319,16 @@ describe('setupTablesHandlers', () => { const joinA = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-A' }) await aAtLookup // A has passed its post-authorize recheck and is hung on the leave-prior lookup await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) // bumps generation, completes - releaseLookup(null) + // A resumes with the socket now registered on table-B (the newer join committed there). + releaseLookup({ type: ROOM_TYPES.TABLE, id: 'table-B' }) await joinA + // A must NOT tear down B's room via its leave-prior, nor register itself on table-A. + expect(socket.leave).not.toHaveBeenCalledWith('table:table-B') + expect(roomManager.removeUserFromRoom).not.toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-B' }, + 'socket-1' + ) expect(socket.join).toHaveBeenCalledWith('table:table-B') expect(socket.join).not.toHaveBeenCalledWith('table:table-A') expect(roomManager.addUserToRoom).not.toHaveBeenCalledWith( diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index 83929c24773..3926d15b5c7 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -139,8 +139,12 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // switch), a LEAVE, or a disconnect. Registering below would strand the socket. if (joinGeneration !== joinAttempt || socket.disconnected) return - // Leave a previously-joined table room if switching tables. + // Leave a previously-joined table room if switching tables. Re-check the generation + // after the lookup await: if a newer join committed to a room during it, `currentRoom` + // is now that room, and leaving it here would tear down the join the client actually + // holds. A superseded join must abort before this mutation. const currentRoom = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) + if (joinGeneration !== joinAttempt || socket.disconnected) return if (currentRoom && currentRoom.id !== tableId) { socket.leave(roomName(currentRoom)) await roomManager.removeUserFromRoom(currentRoom, socket.id) From ccfe29474f27349a06b46697117a1902a0f03ed3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:31:51 -0700 Subject: [PATCH 16/53] fix(realtime): roll back a table join superseded during addUserToRoom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 3 on #5991: after the final generation guard, A could join + register presence while a newer join B commits to its room during addUserToRoom's await — B's leave-prior can't observe A's half-written entry, so A's late write wins and strands the socket. Re-check after addUserToRoom and roll back A's own Socket.IO join + presence (scoped to A's room, never touching B). + a regression test hanging addUserToRoom mid-commit. --- apps/realtime/src/handlers/tables.test.ts | 55 +++++++++++++++++++++++ apps/realtime/src/handlers/tables.ts | 11 +++++ 2 files changed, 66 insertions(+) diff --git a/apps/realtime/src/handlers/tables.test.ts b/apps/realtime/src/handlers/tables.test.ts index 3a8abcefff3..b987dde1d53 100644 --- a/apps/realtime/src/handlers/tables.test.ts +++ b/apps/realtime/src/handlers/tables.test.ts @@ -338,6 +338,61 @@ describe('setupTablesHandlers', () => { ) }) + it('rolls back a join superseded while addUserToRoom is in flight', async () => { + const { socket, handlers } = createSocket() + // A passes every guard and hangs committing to table-A; a newer join B commits to table-B + // during the hang. When A resumes, the post-commit re-check must roll back A's own + // registration (scoped to table-A) without touching B. + let aReachedAdd: () => void = () => {} + const aAtAdd = new Promise((resolve) => { + aReachedAdd = resolve + }) + let releaseAdd: () => void = () => {} + const pendingAdd = new Promise((resolve) => { + releaseAdd = resolve + }) + let addCalls = 0 + const roomManager = createRoomManager({ + addUserToRoom: vi.fn((): Promise => { + addCalls += 1 + if (addCalls === 1) { + aReachedAdd() + return pendingAdd + } + return Promise.resolve() + }), + }) + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const joinA = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-A' }) + await aAtAdd // A has passed every guard and is hung committing to table-A + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) // bumps generation, commits to B + releaseAdd() + await joinA + + // A rolled back its own registration (scoped to table-A) and never touched table-B. + expect(socket.leave).toHaveBeenCalledWith('table:table-A') + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-A' }, + 'socket-1' + ) + expect(roomManager.removeUserFromRoom).not.toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-B' }, + 'socket-1' + ) + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-B' }, + 'socket-1', + expect.anything() + ) + }) + it('leaves the table room on leave', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager({ diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index 3926d15b5c7..7e9b68da5b1 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -194,6 +194,17 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR await roomManager.addUserToRoom(room, socket.id, presence) + // A newer join (table switch) or a leave may have committed while addUserToRoom was in + // flight — that newer join's own leave-prior can't reliably observe this half-written + // entry, so undo our registration here rather than strand the socket on the wrong table. + // Scoped to THIS room (`room`), so `removeUserFromRoom` only clears our socket→room map + // when it still points here and never touches the newer join's room. + if (joinGeneration !== joinAttempt || socket.disconnected) { + socket.leave(roomName(room)) + await roomManager.removeUserFromRoom(room, socket.id) + return + } + // Filter the join ack to live members so a new joiner never briefly sees a // ghost from an entry the sweep hasn't reclaimed yet. const presenceUsers = await filterVisiblePresence( From 5cfa2996d9c899b41b722762d7bbfea5ade932b5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:42:01 -0700 Subject: [PATCH 17/53] refactor(realtime): DRY table-join supersession guards; fix stale comment Cleanliness pass after the review rounds (no behavior change): - Extract the four identical `joinGeneration !== joinAttempt || socket.disconnected` checks into a named `superseded()` helper (the catch keeps its intentionally narrower check). - Remove a stale guard comment that was left stranded above the avatar resolve. - Document the best-effort rollback catch. --- apps/realtime/src/handlers/tables.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index 7e9b68da5b1..bc7acc58fb0 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -76,6 +76,10 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR socket.on(TABLE_PRESENCE_EVENTS.JOIN, async ({ tableId, tabSessionId }: JoinTablePayload) => { const joinAttempt = (joinGeneration += 1) currentTableId = tableId + // True once this JOIN has been superseded — a newer JOIN (table switch) bumped + // joinGeneration, or the socket disconnected. Re-checked after each async step below so a + // stale join can't mutate room state. (The catch uses a narrower check — see there.) + const superseded = () => joinGeneration !== joinAttempt || socket.disconnected try { const userId = socket.userId const userName = socket.userName @@ -129,22 +133,20 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR }) if (!authorized) return - // A newer JOIN started on this socket during authorize (or the socket dropped): - // abort so a stale join can't leave the room the client has since moved to. // Server-authenticated avatar for the presence roster. Resolved up-front so the guard // below also covers this await (mirrors the file-doc join). const avatarUrl = await resolveAvatarUrl(socket, userId) // Abort a JOIN superseded during authorize/avatar resolution — a newer JOIN (table // switch), a LEAVE, or a disconnect. Registering below would strand the socket. - if (joinGeneration !== joinAttempt || socket.disconnected) return + if (superseded()) return // Leave a previously-joined table room if switching tables. Re-check the generation // after the lookup await: if a newer join committed to a room during it, `currentRoom` // is now that room, and leaving it here would tear down the join the client actually // holds. A superseded join must abort before this mutation. const currentRoom = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) - if (joinGeneration !== joinAttempt || socket.disconnected) return + if (superseded()) return if (currentRoom && currentRoom.id !== tableId) { socket.leave(roomName(currentRoom)) await roomManager.removeUserFromRoom(currentRoom, socket.id) @@ -176,7 +178,7 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // LEAVE, or a disconnect during the leave/sweep awaits above must abort here — otherwise // this superseded join would join the room and register presence, stranding the socket in // the wrong table. No await sits between this guard and addUserToRoom (the commit). - if (joinGeneration !== joinAttempt || socket.disconnected) return + if (superseded()) return socket.join(roomName(room)) @@ -199,7 +201,7 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // entry, so undo our registration here rather than strand the socket on the wrong table. // Scoped to THIS room (`room`), so `removeUserFromRoom` only clears our socket→room map // when it still points here and never touches the newer join's room. - if (joinGeneration !== joinAttempt || socket.disconnected) { + if (superseded()) { socket.leave(roomName(room)) await roomManager.removeUserFromRoom(room, socket.id) return @@ -233,7 +235,10 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR const room = tableRoom(tableId) socket.leave(roomName(room)) await roomManager.removeUserFromRoom(room, socket.id) - } catch {} + } catch { + // Best-effort rollback — the original join failure is the one surfaced below, so a + // secondary cleanup error must not mask it or throw out of the error handler. + } socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { tableId, error: 'Failed to join table', From f09aa1758acfa4699133e97233bbf66618a85b2b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:51:32 -0700 Subject: [PATCH 18/53] fix(realtime): file-doc rebind must not drop the current doc or leave a writable ghost Two Cursor findings on the file-doc client-id ownership rebind: - On a document switch, the prior room was left BEFORE the ownership check, so a CLIENT_ID_IN_USE rejection dropped the socket from the old doc without joining the new one (contradicting its own comment). Run the ownership check first, and leave the previous doc only once the rebind is guaranteed to succeed. - Reclaiming a client id removed the stale prior socket from owners + awareness only; its socketToRoomName + Socket.IO membership remained, and handleMessage's SYNC path gates on socketToRoomName (not owners), so it stayed able to write document frames until disconnect. Fully evict the reclaimed socket. + 2 tests. --- apps/realtime/src/handlers/file-doc.test.ts | 52 ++++++++++++++++++++- apps/realtime/src/handlers/file-doc.ts | 40 +++++++++------- 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 1fbecde9078..5719c3eac04 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -33,6 +33,8 @@ interface SentMessage { /** An `io` mock that records every server-originated emit with its target/except. */ function createIo() { const sent: SentMessage[] = [] + /** Records `io.in(socketId).socketsLeave(room)` — a socket forced out of a room from outside. */ + const left: { socketId: string; room: string }[] = [] const to = vi.fn((target: string) => ({ except: (exclude: string) => ({ emit: (event: string, payload: unknown) => @@ -40,7 +42,12 @@ function createIo() { }), emit: (event: string, payload: unknown) => sent.push({ target, event, payload }), })) - return { io: { to } as unknown as IRoomManager['io'], sent } + const inFn = vi.fn((socketId: string) => ({ + socketsLeave: (room: string) => { + left.push({ socketId, room }) + }, + })) + return { io: { to, in: inFn } as unknown as IRoomManager['io'], sent, left } } /** Every socket id a test created, so `afterEach` can drop their rooms without a @@ -583,6 +590,49 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') }) + it('fully evicts a reclaimed prior socket so it can no longer write to the doc', async () => { + const { io, sent, left } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + const b = setup('socket-b', io) // same default user-1 + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) // reclaims client id 7 + + // The stale prior socket is forced out of the Socket.IO room... + expect(left).toContainEqual({ socketId: 'socket-a', room: ROOM_NAME }) + + // ...and its room mapping is cleared, so a later document (SYNC) frame from it is dropped + // (handleMessage's SYNC path gates on socketToRoomName): nothing is applied or relayed. + sent.length = 0 + const doc = new Y.Doc() + doc.getText('t').insert(0, 'x') + const updateFrame = frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(doc)) + ) + a.handlers[FILE_DOC_EVENTS.MESSAGE](updateFrame) + expect(sent.some((m) => m.event === FILE_DOC_EVENTS.MESSAGE)).toBe(false) + }) + + it('does not drop the current document when a switch is rejected for a foreign client id', async () => { + const { io } = createIo() + const a = setup('socket-a', io) // user-1 + const other = setup('socket-c', io, { userId: 'user-b' }) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 10 }) // a owns 10 in file-1 + await other.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 99 }) // user-b owns 99 in file-2 + a.socket.leave.mockClear() + a.socket.join.mockClear() + + // a tries to switch to file-2 but requests client id 99, owned by a DIFFERENT user → reject. + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 99 }) + + expect(a.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'CLIENT_ID_IN_USE' }) + ) + // The rejected switch must leave file-1 intact — a is not torn out of its current document. + expect(a.socket.leave).not.toHaveBeenCalledWith('workspace-file-doc:file-1') + expect(a.socket.join).not.toHaveBeenCalledWith('workspace-file-doc:file-2') + }) + it('re-elects a new seeder when the elected one misses the seed deadline', async () => { const { io, sent } = createIo() const a = setup('socket-a', io) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 1ca39540ceb..e77879d1ab3 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -457,33 +457,39 @@ export function setupWorkspaceFileDocHandlers( // here would leak a dead socket's room or bind the socket to the wrong document. if (socket.disconnected || joinGeneration.get(socket.id) !== generation) return - // Switched documents on the same socket — leave the previous one first (a - // socket edits at most one document). A duplicate join of the SAME room - // falls through and simply re-runs the sync handshake, idempotently. - const currentName = socketToRoomName.get(socket.id) - if (currentName && currentName !== name) { - socket.leave(currentName) - cleanupFileDocForSocket(socket.id, io) - } - const entry = getOrCreateRoom(io, room) - // A client id must be owned by at most one user, or a peer could bind an - // active collaborator's id and pass the per-frame ownership check to - // spoof/clear its caret. Distinguish a reconnect from a spoof by the owning - // user: the same user reclaiming its own client id (a dropped socket - // reconnecting reuses the Yjs client id, and its prior socket may not be - // cleaned up yet) takes over the stale binding; a DIFFERENT user is - // rejected. This runs BEFORE any state mutation below, so a rejected rebind - // leaves the socket's existing binding and caret untouched. + // A client id must be owned by at most one user, or a peer could bind an active + // collaborator's id and pass the per-frame ownership check to spoof/clear its caret. + // Distinguish a reconnect from a spoof by the owning user: the same user reclaiming its + // own client id (a dropped socket reconnecting reuses the Yjs client id, and its prior + // socket may not be cleaned up yet) takes over the stale binding; a DIFFERENT user is + // rejected. This runs BEFORE any teardown of the socket's current binding below, so a + // rejected rebind — even during a document switch — leaves the socket's existing document + // and caret untouched. for (const [otherSid, owner] of entry.owners) { if (owner.clientId !== clientId || otherSid === socket.id) continue if (owner.userId !== userId) { emitJoinError(socket, fileId, 'Client id already in use', 'CLIENT_ID_IN_USE', false) return } + // Fully evict the stale prior socket of the same user — owner + caret AND its room + // mapping + Socket.IO membership — so it can no longer send document (sync) frames: + // handleMessage's SYNC path gates on socketToRoomName, not owners. Done inline rather + // than via cleanupFileDocForSocket, which could destroyRoomIfIdle the room we're joining. entry.owners.delete(otherSid) awarenessProtocol.removeAwarenessStates(entry.awareness, [owner.clientId], null) + socketToRoomName.delete(otherSid) + io.in(otherSid).socketsLeave(name) + } + + // Only now that the rebind is guaranteed to succeed, leave a previously-joined document if + // switching (a socket edits at most one). A duplicate join of the SAME room falls through + // and simply re-runs the sync handshake, idempotently. + const currentName = socketToRoomName.get(socket.id) + if (currentName && currentName !== name) { + socket.leave(currentName) + cleanupFileDocForSocket(socket.id, io) } // Accepted: a same socket rebinding to a NEW client id clears its old caret From 2347dfc1577989895e0a30bfb4b80156f095fdc7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 18:20:07 -0700 Subject: [PATCH 19/53] refactor(realtime): serialize table join/leave to fix map-corruption at the root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 4 on #5991 surfaced a race the generation guards structurally cannot fix: two concurrent joins for one socket race on the single-valued socket→room map — a stalled addUserToRoom for table A lands late, clobbers a newer join's map entry to A, and the rollback then wipes it, stranding the socket (map empty while it holds table B). Guards protect JS suspension points; they can't stop an in-flight Redis write from landing late. Fix per architecture review: serialize this socket's JOIN + LEAVE on a per-socket promise chain so their multi-step async Redis commits can never interleave — restoring the atomic-commit property the synchronous sibling handlers get for free. This DELETES the leave-prior guard and the post-commit rollback (the code that caused the bug); four generation guards collapse to two identical superseded() checks (skip a superseded queued op + one pre-commit check). Reworked the interleaving-specific tests into a fast-switch-skips-superseded test; the leave- cancels-join tests are unchanged. Local to the tables handler — no shared-infra change. --- apps/realtime/src/handlers/tables.test.ts | 133 ++-------------------- apps/realtime/src/handlers/tables.ts | 116 +++++++++---------- 2 files changed, 71 insertions(+), 178 deletions(-) diff --git a/apps/realtime/src/handlers/tables.test.ts b/apps/realtime/src/handlers/tables.test.ts index b987dde1d53..8cfef916a02 100644 --- a/apps/realtime/src/handlers/tables.test.ts +++ b/apps/realtime/src/handlers/tables.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { ROOM_TYPES, type RoomRef } from '@sim/realtime-protocol/rooms' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { TABLE_PRESENCE_EVENTS } from '@sim/realtime-protocol/table-presence' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { IRoomManager } from '@/rooms' @@ -187,15 +187,10 @@ describe('setupTablesHandlers', () => { expect(toEmit).not.toHaveBeenCalled() }) - it('aborts a stale join whose authorize resolves after a newer join', async () => { + it('skips a superseded queued join on a fast table switch', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager() - // First join's authorize hangs until released; the second resolves immediately. - let releaseA: (value: unknown) => void = () => {} - const pendingA = new Promise((resolve) => { - releaseA = resolve - }) - mockAuthorizeRoom.mockReturnValueOnce(pendingA).mockResolvedValue({ + mockAuthorizeRoom.mockResolvedValue({ allowed: true, status: 200, workspaceId: 'ws-1', @@ -203,14 +198,19 @@ describe('setupTablesHandlers', () => { }) setupTablesHandlers(socket as unknown as SetupArg, roomManager) - const joinA = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-A' }) + // Two joins enqueued back-to-back (A then B). B bumps the generation synchronously, so A's + // queued run no-ops at its start check — only B commits. Because JOINs are serialized on one + // op chain, A's and B's Redis writes can never interleave (no map-clobber, no stranding). + handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-A' }) await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) - releaseA({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) - await joinA - // The newer join (B) wins; the stale A join aborts before touching room state. expect(socket.join).toHaveBeenCalledWith('table:table-B') expect(socket.join).not.toHaveBeenCalledWith('table:table-A') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-B' }, + 'socket-1', + expect.anything() + ) expect(roomManager.addUserToRoom).not.toHaveBeenCalledWith( { type: ROOM_TYPES.TABLE, id: 'table-A' }, expect.anything(), @@ -284,115 +284,6 @@ describe('setupTablesHandlers', () => { ) }) - it('aborts a join superseded during the post-authorize leave/sweep window', async () => { - const { socket, handlers } = createSocket() - // Hold A hung on its leave-prior lookup (which runs AFTER the post-authorize recheck), then - // fire a newer join B. When A resumes, the final generation guard before the membership - // commit must abort it — the post-authorize awaits are no longer an unguarded window. - let aReachedLookup: () => void = () => {} - const aAtLookup = new Promise((resolve) => { - aReachedLookup = resolve - }) - let releaseLookup: (value: RoomRef | null) => void = () => {} - const pendingLookup = new Promise((resolve) => { - releaseLookup = resolve - }) - let lookupCalls = 0 - const roomManager = createRoomManager({ - getRoomForSocket: vi.fn((): Promise => { - lookupCalls += 1 - if (lookupCalls === 1) { - aReachedLookup() - return pendingLookup - } - return Promise.resolve(null) - }), - }) - mockAuthorizeRoom.mockResolvedValue({ - allowed: true, - status: 200, - workspaceId: 'ws-1', - workspacePermission: 'admin', - }) - setupTablesHandlers(socket as unknown as SetupArg, roomManager) - - const joinA = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-A' }) - await aAtLookup // A has passed its post-authorize recheck and is hung on the leave-prior lookup - await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) // bumps generation, completes - // A resumes with the socket now registered on table-B (the newer join committed there). - releaseLookup({ type: ROOM_TYPES.TABLE, id: 'table-B' }) - await joinA - - // A must NOT tear down B's room via its leave-prior, nor register itself on table-A. - expect(socket.leave).not.toHaveBeenCalledWith('table:table-B') - expect(roomManager.removeUserFromRoom).not.toHaveBeenCalledWith( - { type: ROOM_TYPES.TABLE, id: 'table-B' }, - 'socket-1' - ) - expect(socket.join).toHaveBeenCalledWith('table:table-B') - expect(socket.join).not.toHaveBeenCalledWith('table:table-A') - expect(roomManager.addUserToRoom).not.toHaveBeenCalledWith( - { type: ROOM_TYPES.TABLE, id: 'table-A' }, - expect.anything(), - expect.anything() - ) - }) - - it('rolls back a join superseded while addUserToRoom is in flight', async () => { - const { socket, handlers } = createSocket() - // A passes every guard and hangs committing to table-A; a newer join B commits to table-B - // during the hang. When A resumes, the post-commit re-check must roll back A's own - // registration (scoped to table-A) without touching B. - let aReachedAdd: () => void = () => {} - const aAtAdd = new Promise((resolve) => { - aReachedAdd = resolve - }) - let releaseAdd: () => void = () => {} - const pendingAdd = new Promise((resolve) => { - releaseAdd = resolve - }) - let addCalls = 0 - const roomManager = createRoomManager({ - addUserToRoom: vi.fn((): Promise => { - addCalls += 1 - if (addCalls === 1) { - aReachedAdd() - return pendingAdd - } - return Promise.resolve() - }), - }) - mockAuthorizeRoom.mockResolvedValue({ - allowed: true, - status: 200, - workspaceId: 'ws-1', - workspacePermission: 'admin', - }) - setupTablesHandlers(socket as unknown as SetupArg, roomManager) - - const joinA = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-A' }) - await aAtAdd // A has passed every guard and is hung committing to table-A - await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) // bumps generation, commits to B - releaseAdd() - await joinA - - // A rolled back its own registration (scoped to table-A) and never touched table-B. - expect(socket.leave).toHaveBeenCalledWith('table:table-A') - expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith( - { type: ROOM_TYPES.TABLE, id: 'table-A' }, - 'socket-1' - ) - expect(roomManager.removeUserFromRoom).not.toHaveBeenCalledWith( - { type: ROOM_TYPES.TABLE, id: 'table-B' }, - 'socket-1' - ) - expect(roomManager.addUserToRoom).toHaveBeenCalledWith( - { type: ROOM_TYPES.TABLE, id: 'table-B' }, - 'socket-1', - expect.anything() - ) - }) - it('leaves the table room on leave', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager({ diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index bc7acc58fb0..bd5192883d5 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -62,24 +62,39 @@ function normalizeCellSelection(cell: unknown): TableCellSelection | undefined { * only because a workflow room's name equals its id). */ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { - // Monotonic per-socket join counter: each JOIN captures its number and, after the async - // authorize, aborts if a newer JOIN has started — a fast table switch A→B can otherwise - // let A's late handler leave B and strand the socket in room A while the client views B. + // Monotonic per-socket generation: each JOIN/LEAVE bumps it synchronously on arrival, and a + // queued or in-flight op that finds a newer generation aborts — a fast table switch A→B thus + // cancels A the instant B arrives. let joinGeneration = 0 - // The table the socket currently intends to be in (set when a join starts). A leave - // targeting it — or an unscoped leave — advances joinGeneration to cancel an in-flight - // join, so a join still awaiting authorization can't complete after the client left and - // strand the socket in the room (present in presence + still receiving broadcasts). A - // leave for a DIFFERENT table must NOT cancel it (a table switch), mirroring workspace-files. + // The table the socket currently intends to be in (set when a join is enqueued). A leave + // targeting it — or an unscoped leave — bumps the generation to cancel that join; a leave for a + // DIFFERENT table must NOT (a table switch), mirroring workspace-files. let currentTableId: string | null = null + // Serialize this socket's room mutations (JOIN + LEAVE) so their multi-step async Redis commits + // can never interleave: two concurrent joins would otherwise race on the single-valued + // socket→room map (a late addUserToRoom clobbering a newer join's entry). This restores the + // atomic-commit property the synchronous sibling handlers (file-doc, workspace-files) get for + // free. CELL_SELECTION is NOT chained — it only touches presence activity, never the map. + let opChain: Promise = Promise.resolve() - socket.on(TABLE_PRESENCE_EVENTS.JOIN, async ({ tableId, tabSessionId }: JoinTablePayload) => { + socket.on(TABLE_PRESENCE_EVENTS.JOIN, ({ tableId, tabSessionId }: JoinTablePayload) => { const joinAttempt = (joinGeneration += 1) currentTableId = tableId - // True once this JOIN has been superseded — a newer JOIN (table switch) bumped - // joinGeneration, or the socket disconnected. Re-checked after each async step below so a - // stale join can't mutate room state. (The catch uses a narrower check — see there.) + opChain = opChain + .then(() => runJoin(tableId, tabSessionId, joinAttempt)) + .catch((error) => logger.error('Error joining table room:', error)) + // Returned so callers awaiting this op (e.g. tests) can await its completion; Socket.IO + // ignores a handler's return value. + return opChain + }) + + async function runJoin(tableId: string, tabSessionId: string | undefined, joinAttempt: number) { + // True once this JOIN has been superseded — a newer JOIN/LEAVE bumped joinGeneration, or the + // socket disconnected. Because ops are serialized, no other op mutates room state while this + // one runs, so only two checks are needed: skip a superseded queued op (here), and one final + // check right before the membership commit. const superseded = () => joinGeneration !== joinAttempt || socket.disconnected + if (superseded()) return try { const userId = socket.userId const userName = socket.userName @@ -133,20 +148,13 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR }) if (!authorized) return - // Server-authenticated avatar for the presence roster. Resolved up-front so the guard - // below also covers this await (mirrors the file-doc join). + // Server-authenticated avatar for the presence roster. const avatarUrl = await resolveAvatarUrl(socket, userId) - // Abort a JOIN superseded during authorize/avatar resolution — a newer JOIN (table - // switch), a LEAVE, or a disconnect. Registering below would strand the socket. - if (superseded()) return - - // Leave a previously-joined table room if switching tables. Re-check the generation - // after the lookup await: if a newer join committed to a room during it, `currentRoom` - // is now that room, and leaving it here would tear down the join the client actually - // holds. A superseded join must abort before this mutation. + // Leave a previously-joined table room if switching tables. No generation guard is needed + // around this: serialization guarantees no concurrent op committed to a different room + // during the lookup, so `currentRoom` is the socket's genuine prior room, safe to leave. const currentRoom = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) - if (superseded()) return if (currentRoom && currentRoom.id !== tableId) { socket.leave(roomName(currentRoom)) await roomManager.removeUserFromRoom(currentRoom, socket.id) @@ -174,10 +182,8 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR } } - // Final re-check immediately before the membership commit: a newer JOIN (table switch), a - // LEAVE, or a disconnect during the leave/sweep awaits above must abort here — otherwise - // this superseded join would join the room and register presence, stranding the socket in - // the wrong table. No await sits between this guard and addUserToRoom (the commit). + // Final re-check before the membership commit: a LEAVE or a newer JOIN enqueued during the + // awaits above bumped the generation, or the socket disconnected. Abort before registering. if (superseded()) return socket.join(roomName(room)) @@ -194,19 +200,11 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR avatarUrl, } + // If the socket disconnects during this commit (disconnect cleanup runs off the op chain), + // this write can land after it, leaving a stale presence entry. Benign and self-correcting: + // filterVisiblePresence hides it and sweepStalePresence reclaims it (same as the siblings). await roomManager.addUserToRoom(room, socket.id, presence) - // A newer join (table switch) or a leave may have committed while addUserToRoom was in - // flight — that newer join's own leave-prior can't reliably observe this half-written - // entry, so undo our registration here rather than strand the socket on the wrong table. - // Scoped to THIS room (`room`), so `removeUserFromRoom` only clears our socket→room map - // when it still points here and never touches the newer join's room. - if (superseded()) { - socket.leave(roomName(room)) - await roomManager.removeUserFromRoom(room, socket.id) - return - } - // Filter the join ack to live members so a new joiner never briefly sees a // ghost from an entry the sweep hasn't reclaimed yet. const presenceUsers = await filterVisiblePresence( @@ -225,9 +223,10 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR logger.info(`User ${userId} (${userName}) joined table room ${tableId}`) } catch (error) { logger.error('Error joining table room:', error) - // A superseded join (a newer join/leave bumped the generation) must NOT roll back — it - // would tear down room state a newer successful join to the same table now holds — nor - // signal an error for a table the client already left. + // If a newer JOIN/LEAVE superseded this one while it ran, skip the rollback + error: the + // next serialized op cleans up any partial registration via its leave-prior, and the client + // already moved off this table so the error is moot. A disconnect (not a supersession) still + // falls through and rolls back — hence the generation-only check, not the full `superseded`. if (joinGeneration !== joinAttempt) return // Roll back any partial join so a failed attempt can't leave the socket in the // Socket.IO room or a stale presence entry behind, before signalling a retry. @@ -246,27 +245,30 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR retryable: true, }) } + } + + socket.on(TABLE_PRESENCE_EVENTS.LEAVE, (payload?: { tableId?: string }) => { + // Cancel an in-flight/queued join whose table the client is now leaving (or an unscoped + // leave). Scope to the current table intent so a stale/deferred leave for a DIFFERENT table + // can't cancel the join the client has since switched to. Bumped synchronously here — before + // the teardown is enqueued — so it cancels a running join at its next generation check. + if (!payload?.tableId || payload.tableId === currentTableId) { + joinGeneration += 1 + currentTableId = null + } + opChain = opChain + .then(() => runLeave(payload)) + .catch((error) => logger.error('Error leaving table room:', error)) + return opChain }) - socket.on(TABLE_PRESENCE_EVENTS.LEAVE, async (payload?: { tableId?: string }) => { + async function runLeave(payload?: { tableId?: string }) { try { - // Cancel an in-flight join whose table the client is now leaving (or an unscoped - // leave): a join still awaiting authorization would otherwise complete after the - // client left — joining the room, registering presence, and broadcasting a ghost - // until disconnect. Guard on the current table intent so a stale/deferred leave for - // a DIFFERENT table can't abort the join the client has since switched to. Runs - // before the teardown below because the racing join has registered nothing yet - // (getRoomForSocket returns null), so only this generation bump can stop it. - if (!payload?.tableId || payload.tableId === currentTableId) { - joinGeneration += 1 - currentTableId = null - } if (!roomManager.isReady()) return const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) if (!room) return - // Scope the leave to a specific table when the client provides one: a deferred - // leave from a prior view must not evict the socket from a room it has since - // switched into (table A→B leaves A's leave targeting B). + // Scope the leave to a specific table when the client provides one: a deferred leave from a + // prior view must not evict the socket from a room it has since switched into. if (payload?.tableId && payload.tableId !== room.id) return socket.leave(roomName(room)) await roomManager.removeUserFromRoom(room, socket.id) @@ -274,7 +276,7 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR } catch (error) { logger.error('Error leaving table room:', error) } - }) + } socket.on(TABLE_PRESENCE_EVENTS.CELL_SELECTION, async ({ cell }: { cell: unknown }) => { try { From 06cf68f0872afbce9547336d467eeda873cf7157 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 18:29:19 -0700 Subject: [PATCH 20/53] fix(realtime): always roll back a failed table join; re-elect file-doc seeder on reclaim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings: - Table join: the catch skipped rollback when superseded, but a socket.join that landed before addUserToRoom threw leaves the socket in the Socket.IO room with no matching socket->room map entry — unreclaimable by any later op (cleanup keys off the map). Under serialization the skip is unnecessary (the newer op hasn't committed), so always roll back. Simpler + fixes the strand. - File-doc reclaim: fully evicting the prior socket didn't release the seeder role if it held it, so electSeederIfNeeded (which no-ops while seederSocketId is set) never re-elected and an unseeded doc stayed empty until the deadline. Clear the role on eviction so the join's election picks a new seeder. + 2 regression tests. --- apps/realtime/src/handlers/file-doc.test.ts | 16 ++++++++++++++++ apps/realtime/src/handlers/file-doc.ts | 4 ++++ apps/realtime/src/handlers/tables.test.ts | 20 ++++++++++++++++++++ apps/realtime/src/handlers/tables.ts | 12 +++++------- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 5719c3eac04..f78df6d8a81 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -612,6 +612,22 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(sent.some((m) => m.event === FILE_DOC_EVENTS.MESSAGE)).toBe(false) }) + it('re-elects a seeder when the reclaimed socket held the seeder role', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + // a is the sole owner of an unseeded doc → elected seeder. + expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-a') + sent.length = 0 + + const b = setup('socket-b', io) // same user-1 reconnecting, reusing client id 7 + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + + // The reclaim evicts a (the seeder) and releases the role, so the join's election picks b — + // the doc gets seeded instead of waiting out the deadline. + expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') + }) + it('does not drop the current document when a switch is rejected for a foreign client id', async () => { const { io } = createIo() const a = setup('socket-a', io) // user-1 diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index e77879d1ab3..506f001ee53 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -481,6 +481,10 @@ export function setupWorkspaceFileDocHandlers( awarenessProtocol.removeAwarenessStates(entry.awareness, [owner.clientId], null) socketToRoomName.delete(otherSid) io.in(otherSid).socketsLeave(name) + // If the evicted socket held the seeder role, release it so the election at the end of + // this join re-elects (electSeederIfNeeded no-ops while seederSocketId is set) — otherwise + // an unseeded document would stay empty until the seed deadline expires. + if (entry.seederSocketId === otherSid) entry.seederSocketId = null } // Only now that the rebind is guaranteed to succeed, leave a previously-joined document if diff --git a/apps/realtime/src/handlers/tables.test.ts b/apps/realtime/src/handlers/tables.test.ts index 8cfef916a02..73b0ea62678 100644 --- a/apps/realtime/src/handlers/tables.test.ts +++ b/apps/realtime/src/handlers/tables.test.ts @@ -297,4 +297,24 @@ describe('setupTablesHandlers', () => { expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1') expect(roomManager.broadcastPresenceUpdate).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1') }) + + it('rolls back the Socket.IO membership when a join fails mid-commit', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + // socket.join lands first, then the presence write throws — the socket is now in the + // Socket.IO room with no matching socket→room map entry, unreclaimable by any later op. + addUserToRoom: vi.fn().mockRejectedValue(new Error('redis down')), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + + // The catch must always roll back the partial membership, not skip it. + expect(socket.leave).toHaveBeenCalledWith('table:table-1') + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1') + expect(socket.emit).toHaveBeenCalledWith( + TABLE_PRESENCE_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'JOIN_FAILED' }) + ) + }) }) diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index bd5192883d5..a29f50ac519 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -223,13 +223,11 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR logger.info(`User ${userId} (${userName}) joined table room ${tableId}`) } catch (error) { logger.error('Error joining table room:', error) - // If a newer JOIN/LEAVE superseded this one while it ran, skip the rollback + error: the - // next serialized op cleans up any partial registration via its leave-prior, and the client - // already moved off this table so the error is moot. A disconnect (not a supersession) still - // falls through and rolls back — hence the generation-only check, not the full `superseded`. - if (joinGeneration !== joinAttempt) return - // Roll back any partial join so a failed attempt can't leave the socket in the - // Socket.IO room or a stale presence entry behind, before signalling a retry. + // Always roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` + // that landed without a matching `addUserToRoom` (a throw in between) would otherwise leave + // the socket stranded in the Socket.IO room, unreclaimable by any later op. Safe to run even + // when superseded — serialization means the newer op hasn't committed yet, so this touches + // only this join's own (this-table) state, never the newer op's room. try { const room = tableRoom(tableId) socket.leave(roomName(room)) From 6ab59b96d8a9b8297acad2a050457d546b53d073 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 19:30:37 -0700 Subject: [PATCH 21/53] fix(realtime): close revoke-race ghost presence in workflow join MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An access-revalidation revoke landing between socket.join and addUserToRoom socketsLeaves the socket while its presence mapping does not yet exist, so cleanupEvictedSocket finds nothing to remove and the join then writes presence for a socket already out of the room — a ghost collaborator until the stale sweep. Hoist resolveAvatarUrl (the only await in that gap) above the re-auth check so the whole re-auth -> socket.join -> addUserToRoom section is await-free, matching the invariant the handler already relies on for the pre-join re-auth. + ordering regression test. --- apps/realtime/src/handlers/workflow.test.ts | 51 ++++++++++++++++++--- apps/realtime/src/handlers/workflow.ts | 15 ++++-- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/apps/realtime/src/handlers/workflow.test.ts b/apps/realtime/src/handlers/workflow.test.ts index a0ee506f2ae..c51beab0b5b 100644 --- a/apps/realtime/src/handlers/workflow.test.ts +++ b/apps/realtime/src/handlers/workflow.test.ts @@ -4,12 +4,21 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { IRoomManager } from '@/rooms' -const { mockGetWorkflowState, mockVerifyWorkflowAccess, mockResolveCurrentWorkflowRole } = - vi.hoisted(() => ({ - mockGetWorkflowState: vi.fn(), - mockVerifyWorkflowAccess: vi.fn(), - mockResolveCurrentWorkflowRole: vi.fn(), - })) +const { + mockGetWorkflowState, + mockVerifyWorkflowAccess, + mockResolveCurrentWorkflowRole, + mockResolveAvatarUrl, +} = vi.hoisted(() => ({ + mockGetWorkflowState: vi.fn(), + mockVerifyWorkflowAccess: vi.fn(), + mockResolveCurrentWorkflowRole: vi.fn(), + mockResolveAvatarUrl: vi.fn(), +})) + +vi.mock('@/handlers/avatar', () => ({ + resolveAvatarUrl: mockResolveAvatarUrl, +})) vi.mock('@sim/db', () => ({ db: { select: vi.fn() }, @@ -90,6 +99,36 @@ describe('setupWorkflowHandlers', () => { mockGetWorkflowState.mockResolvedValue({ id: 'workflow-1', state: {} }) mockVerifyWorkflowAccess.mockResolvedValue({ hasAccess: true, role: 'admin' }) mockResolveCurrentWorkflowRole.mockResolvedValue('admin') + mockResolveAvatarUrl.mockResolvedValue('avatar.png') + }) + + it('resolves the avatar before joining so no await sits between socket.join and addUserToRoom', async () => { + const order: string[] = [] + mockResolveAvatarUrl.mockImplementation(async () => { + order.push('avatar') + return 'avatar.png' + }) + const { socket, handlers } = createSocket({ + join: vi.fn(() => { + order.push('join') + }), + }) + const roomManager = createRoomManager({ + addUserToRoom: vi.fn(async () => { + order.push('add') + }), + }) + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + + // The avatar await must complete before socket.join; reintroducing it between + // join and addUserToRoom reopens the revoke-race ghost-presence window. + expect(order).toEqual(['avatar', 'join', 'add']) }) it('includes workflowId when authentication is missing', async () => { diff --git a/apps/realtime/src/handlers/workflow.ts b/apps/realtime/src/handlers/workflow.ts index 05b9e30fabf..2c42b540441 100644 --- a/apps/realtime/src/handlers/workflow.ts +++ b/apps/realtime/src/handlers/workflow.ts @@ -131,14 +131,23 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: } } + // Resolve the avatar before the critical section below. It is the only + // await that used to sit between socket.join and addUserToRoom, and a sweep + // eviction in that gap would socketsLeave the socket while its presence + // mapping did not yet exist — cleanupEvictedSocket would find nothing to + // remove, then this join would write presence for a socket already out of + // the room (a ghost collaborator until the stale sweep). Hoisting it keeps + // the whole re-auth -> socket.join -> addUserToRoom section await-free. + const avatarUrl = await resolveAvatarUrl(socket, userId) + // Re-authorize immediately before joining: the access-revalidation sweep // may have evicted this socket while the awaits above were in flight, and // its eviction is recorded in the shared role cache before it runs — so a // revoked user resolves to null here. The resolver is single-flighted per // (user, workflow), so this read cannot race the sweep's and overwrite a // recorded revocation with a stale role; and no awaits sit between this - // check and socket.join, so a sweep eviction cannot interleave after it - // and be reversed by this join. + // check and addUserToRoom (avatar resolution is hoisted above), so a sweep + // eviction cannot interleave inside the join and be reversed by it. const currentRole = await resolveCurrentWorkflowRole(userId, workflowId, userRole) if (currentRole === null) { logger.warn( @@ -157,8 +166,6 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: // Join the new room socket.join(workflowId) - const avatarUrl = await resolveAvatarUrl(socket, userId) - // Create presence entry const userPresence: UserPresence = { userId, From 5c3b03cff53cc19be46d53569d6b9914589703e2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 19:55:35 -0700 Subject: [PATCH 22/53] refactor(realtime): serialize workflow join/leave; drop dead room-authz limb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comprehensive independent audit follow-ups: - workflow.ts join/leave now use the same opChain + joinGeneration serialization as the sibling handlers (tables, file-doc, workspace-files). It was the only async presence path left unserialized, so a rapid workflow switch A->B (or a leave racing an in-flight join) could strand presence in room A — a ghost collaborator still receiving A's operation broadcasts until disconnect. The join now aborts a superseded op at start and again right before the membership commit, and the catch always rolls back a partial join. - leave-workflow drops the '&& session' gate: an idle user whose 1h session key expired (while the 24h room mapping is still live) can now leave cleanly instead of being stranded until disconnect. The room ref alone suffices. - authorizeRoom: remove the dead ROOM_TYPES.WORKFLOW resolver + its getActiveWorkflowContext import. Workflow authorizes through its own path and never flows through authorizeRoom; the map now honestly covers only the workspace-scoped types (files, file-doc, table). - Remove unused isSameRoom (zero callers) and a needless useMemo in PresenceAvatars (plain derivation, single copy). - Tests: 4 workflow serialization/leave regressions. All gates green: tsc (sim/realtime/packages) 0, 204 realtime + 11 protocol tests, biome, api-validation, boundaries, prune 14/25. --- apps/realtime/src/handlers/workflow.test.ts | 98 ++++++++++++++++++- apps/realtime/src/handlers/workflow.ts | 82 ++++++++++++---- .../components/presence/presence-avatars.tsx | 14 +-- packages/platform-authz/src/rooms.ts | 34 +++---- packages/realtime-protocol/src/rooms.test.ts | 10 -- packages/realtime-protocol/src/rooms.ts | 5 - 6 files changed, 181 insertions(+), 62 deletions(-) diff --git a/apps/realtime/src/handlers/workflow.test.ts b/apps/realtime/src/handlers/workflow.test.ts index c51beab0b5b..5c46c27d4dc 100644 --- a/apps/realtime/src/handlers/workflow.test.ts +++ b/apps/realtime/src/handlers/workflow.test.ts @@ -42,13 +42,14 @@ interface JoinWorkflowPayload { } function createSocket(overrides?: Partial>) { - const handlers: Record Promise | void> = {} + // leave-workflow takes no payload; join-workflow takes one — so the stored handler's arg is optional. + const handlers: Record Promise | void> = {} const socket = { id: 'socket-1', userId: 'user-1', userName: 'Test User', userImage: 'avatar.png', - on: vi.fn((event: string, handler: (payload: JoinWorkflowPayload) => Promise | void) => { + on: vi.fn((event: string, handler: (payload?: JoinWorkflowPayload) => Promise | void) => { handlers[event] = handler }), emit: vi.fn(), @@ -280,4 +281,97 @@ describe('setupWorkflowHandlers', () => { retryable: true, }) }) + + it('cancels a superseded queued join on a fast workflow switch', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + // Enqueue A without awaiting, then B: B bumps the generation synchronously, so A is superseded + // before its queued op runs and must never commit. + handlers['join-workflow']({ workflowId: 'workflow-a', tabSessionId: 'tab-1' }) + await handlers['join-workflow']({ workflowId: 'workflow-b', tabSessionId: 'tab-1' }) + + expect(socket.join).toHaveBeenCalledWith('workflow-b') + expect(socket.join).not.toHaveBeenCalledWith('workflow-a') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-b' }, + 'socket-1', + expect.anything() + ) + expect(roomManager.addUserToRoom).not.toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-a' }, + 'socket-1', + expect.anything() + ) + }) + + it('cancels an in-flight join when a leave is enqueued before it commits', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + await handlers['leave-workflow']() + + expect(socket.join).not.toHaveBeenCalled() + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + }) + + it('rolls back the workflow membership when addUserToRoom fails mid-commit', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + addUserToRoom: vi.fn().mockRejectedValue(new Error('redis down')), + }) + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + + expect(socket.leave).toHaveBeenCalledWith('workflow-1') + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-1' }, + 'socket-1' + ) + expect(socket.emit).toHaveBeenCalledWith( + 'join-workflow-error', + expect.objectContaining({ code: 'JOIN_WORKFLOW_FAILED' }) + ) + }) + + it('leaves the workflow room even when the session key has expired', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue({ type: 'workflow', id: 'workflow-1' }), + getUserSession: vi.fn().mockResolvedValue(null), + }) + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['leave-workflow']() + + expect(socket.leave).toHaveBeenCalledWith('workflow-1') + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-1' }, + 'socket-1' + ) + expect(roomManager.broadcastPresenceUpdate).toHaveBeenCalledWith({ + type: 'workflow', + id: 'workflow-1', + }) + }) }) diff --git a/apps/realtime/src/handlers/workflow.ts b/apps/realtime/src/handlers/workflow.ts index 2c42b540441..afc99d37c03 100644 --- a/apps/realtime/src/handlers/workflow.ts +++ b/apps/realtime/src/handlers/workflow.ts @@ -9,7 +9,38 @@ import { type IRoomManager, type UserPresence, workflowRoom as wf } from '@/room const logger = createLogger('WorkflowHandlers') export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { - socket.on('join-workflow', async ({ workflowId, tabSessionId }) => { + // Monotonic per-socket generation: each JOIN/LEAVE bumps it synchronously on arrival, and a + // queued or in-flight op that finds a newer generation aborts — a fast workflow switch A→B thus + // cancels A the instant B arrives. + let joinGeneration = 0 + // Serialize this socket's room mutations (JOIN + LEAVE) so their multi-step async Redis commits + // can never interleave: two concurrent joins would otherwise race on the single-valued + // socket→room map (a late addUserToRoom clobbering a newer join's entry, leaving the socket a + // ghost in the old room and receiving its operation broadcasts). This matches the sibling + // handlers (tables, file-doc, workspace-files). + let opChain: Promise = Promise.resolve() + + socket.on('join-workflow', ({ workflowId, tabSessionId }) => { + const joinAttempt = (joinGeneration += 1) + opChain = opChain + .then(() => runJoin(workflowId, tabSessionId, joinAttempt)) + .catch((error) => logger.error('Error joining workflow:', error)) + // Returned so callers awaiting this op (e.g. tests) can await its completion; Socket.IO + // ignores a handler's return value. + return opChain + }) + + async function runJoin( + workflowId: string, + tabSessionId: string | undefined, + joinAttempt: number + ) { + // True once this JOIN has been superseded — a newer JOIN/LEAVE bumped joinGeneration, or the + // socket disconnected. Because ops are serialized, no other op mutates room state while this + // one runs, so only two checks are needed: skip a superseded queued op (here), and one final + // check right before the membership commit. + const superseded = () => joinGeneration !== joinAttempt || socket.disconnected + if (superseded()) return try { const userId = socket.userId const userName = socket.userName @@ -163,6 +194,12 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: } userRole = currentRole + // Final re-check before the membership commit: a LEAVE or a newer JOIN enqueued during the + // awaits above bumped the generation, or the socket disconnected. Abort before registering. + // (This guards against a superseding op; the avatar hoist above guards against the off-chain + // access-revalidation sweep, which does not bump the generation.) + if (superseded()) return + // Join the new room socket.join(workflowId) @@ -207,7 +244,11 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: ) } catch (error) { logger.error('Error joining workflow:', error) - // Undo socket.join and room manager entry if any operation failed + // Always roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` + // that landed without a matching `addUserToRoom` (a throw in between) would otherwise strand + // the socket in the Socket.IO room, unreclaimable by any later op. Safe even when superseded — + // serialization means the newer op hasn't committed yet, so this touches only this join's own + // room state, never the newer op's. socket.leave(workflowId) await roomManager.removeUserFromRoom(wf(workflowId), socket.id) const isReady = roomManager.isReady() @@ -218,26 +259,33 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: retryable: true, }) } + } + + socket.on('leave-workflow', () => { + // A leave always cancels any in-flight/queued join for this socket (the client emits it with no + // payload — there is no partial-switch case as there is for tables). Bumped synchronously here, + // before the teardown is enqueued, so it cancels a running join at its next generation check. + joinGeneration += 1 + opChain = opChain + .then(() => runLeave()) + .catch((error) => logger.error('Error leaving workflow:', error)) + return opChain }) - socket.on('leave-workflow', async () => { + async function runLeave() { try { - if (!roomManager.isReady()) { - return - } - + if (!roomManager.isReady()) return const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) - const session = await roomManager.getUserSession(socket.id) - - if (room && session) { - socket.leave(room.id) - await roomManager.removeUserFromRoom(room, socket.id) - await roomManager.broadcastPresenceUpdate(room) - - logger.info(`User ${session.userId} (${session.userName}) left workflow ${room.id}`) - } + // The room ref alone is sufficient to leave; no session lookup is gated in front of it, so an + // idle user whose 1h session key expired (while the 24h room mapping is still live) can still + // leave cleanly instead of being stranded as a ghost until disconnect. + if (!room) return + socket.leave(room.id) + await roomManager.removeUserFromRoom(room, socket.id) + await roomManager.broadcastPresenceUpdate(room) + logger.info(`User ${socket.userId} (${socket.userName}) left workflow ${room.id}`) } catch (error) { logger.error('Error leaving workflow:', error) } - }) + } } diff --git a/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx index da5bcc68d68..a76ec009448 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx @@ -1,6 +1,5 @@ 'use client' -import { useMemo } from 'react' import { Avatar, AvatarFallback, AvatarImage, cn, Tooltip } from '@sim/emcn' import { getUserColor } from '@/lib/workspaces/colors' @@ -80,15 +79,10 @@ export function PresenceAvatars({ maxVisible = DEFAULT_MAX_VISIBLE, className, }: PresenceAvatarsProps) { - const { visibleUsers, overflowCount } = useMemo(() => { - if (users.length === 0) { - return { visibleUsers: [] as PresenceAvatarUser[], overflowCount: 0 } - } - const visible = users.slice(0, maxVisible) - const overflow = Math.max(0, users.length - maxVisible) - // Reverse so the rightmost avatar stays stable as new ones reveal on the left. - return { visibleUsers: [...visible].reverse(), overflowCount: overflow } - }, [users, maxVisible]) + // Reverse so the rightmost avatar stays stable as new ones reveal on the left. + // slice() already returns a fresh array, so the in-place reverse is safe. + const visibleUsers = users.slice(0, maxVisible).reverse() + const overflowCount = Math.max(0, users.length - maxVisible) if (visibleUsers.length === 0) { return null diff --git a/packages/platform-authz/src/rooms.ts b/packages/platform-authz/src/rooms.ts index 61b920d9142..19750096833 100644 --- a/packages/platform-authz/src/rooms.ts +++ b/packages/platform-authz/src/rooms.ts @@ -1,7 +1,6 @@ import { db, userTableDefinitions, workspace, workspaceFiles } from '@sim/db' import { ROOM_TYPES, type RoomRef, type RoomType } from '@sim/realtime-protocol/rooms' import { and, eq, isNull } from 'drizzle-orm' -import { getActiveWorkflowContext } from './workflow' import { type PermissionType, permissionSatisfies, @@ -73,20 +72,17 @@ async function resolveTableWorkspace(tableId: string): Promise = { - [ROOM_TYPES.WORKFLOW]: async (workflowId) => { - const context = await getActiveWorkflowContext(workflowId) - if (!context?.workspaceId) return null - return { - workspaceId: context.workspaceId, - workspaceOrganizationId: context.workspaceOrganizationId, - } - }, +const ROOM_WORKSPACE_RESOLVERS: Partial> = { // A workspace-files room is addressed directly by its workspace id. [ROOM_TYPES.WORKSPACE_FILES]: resolveWorkspaceRoomWorkspace, // A file-doc room is addressed by file id; resolve it to its workspace. @@ -104,10 +100,12 @@ export interface RoomAuthorizationResult { } /** - * Authorizes a user against a realtime room. Mirrors - * `authorizeWorkflowByWorkspacePermission` (the exemplary workflow authorizer) - * but generalized over room type: resolve the room's workspace, then gate on the - * user's effective workspace permission under the read < write < admin ordering. + * Authorizes a user against a workspace-scoped realtime room (workspace-files, + * file-doc, table). Mirrors `authorizeWorkflowByWorkspacePermission` (the + * exemplary workflow authorizer) but generalized over room type: resolve the + * room's workspace, then gate on the user's effective workspace permission under + * the read < write < admin ordering. Workflow rooms use their own authorizer and + * do not pass through here (see {@link ROOM_WORKSPACE_RESOLVERS}). * * Returns a denial (never throws) for unknown room type (400), missing/archived * resource (404), and insufficient permission (403), so realtime handlers and diff --git a/packages/realtime-protocol/src/rooms.test.ts b/packages/realtime-protocol/src/rooms.test.ts index cb37c3fca45..cbf9b3cd5b2 100644 --- a/packages/realtime-protocol/src/rooms.test.ts +++ b/packages/realtime-protocol/src/rooms.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from 'vitest' import { ALL_ROOM_TYPES, isRoomType, - isSameRoom, parseRoomName, ROOM_TYPES, type RoomRef, @@ -84,15 +83,6 @@ describe('isRoomType', () => { }) }) -describe('isSameRoom', () => { - it('compares by type and id', () => { - const a: RoomRef = { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-1' } - expect(isSameRoom(a, { ...a })).toBe(true) - expect(isSameRoom(a, { type: ROOM_TYPES.WORKFLOW, id: 'ws-1' })).toBe(false) - expect(isSameRoom(a, { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-2' })).toBe(false) - }) -}) - describe('ALL_ROOM_TYPES', () => { it('contains every declared room type', () => { expect([...ALL_ROOM_TYPES].sort()).toEqual([...Object.values(ROOM_TYPES)].sort()) diff --git a/packages/realtime-protocol/src/rooms.ts b/packages/realtime-protocol/src/rooms.ts index de6a97dd82d..f901f198f8d 100644 --- a/packages/realtime-protocol/src/rooms.ts +++ b/packages/realtime-protocol/src/rooms.ts @@ -96,11 +96,6 @@ export function parseRoomName(name: string): RoomRef | null { return { type: ROOM_TYPES.WORKFLOW, id: name } } -/** Whether two room refs address the same room. */ -export function isSameRoom(a: RoomRef, b: RoomRef): boolean { - return a.type === b.type && a.id === b.id -} - /** * The `presence-update` broadcast event name for a room type. `WORKFLOW` keeps * the historical bare `presence-update` name (client backward-compat); every From 2bfb58e6cc3b0674bd765867dfc008255c95d99a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 20:11:20 -0700 Subject: [PATCH 23/53] fix(realtime): align session TTL, harden committed joins from post-success rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-module comprehensive audit follow-ups: - redis-manager: SESSION_TTL now tracks SOCKET_ROOMS_TTL (was 1h vs 24h). The room set outlived the session, and since getRoomForSocket reads the room set while the workflow handlers gate edits/presence on `room && session`, an active-but-idle collaborator got wedged into 'session expired' after 1h — sticky until reload (activity only EXPIREs the already-gone session; only addUserToRoom re-HSETs it). Both keys refresh together, so they now expire together (restores the pre-refactor consistency, where both shared one TTL). - workflow.ts + tables.ts: a 'committed' flag stops the join catch from rolling back a genuinely-joined user when a trailing ack/broadcast/metric step fails on a Redis blip (a pure getUniqueUserCount log-metric failure could otherwise kick a live collaborator after success was already acked). - workflow.ts: leave-prior now guards `currentRoom.id !== workflowId` (a same-workflow re-join no longer leave→re-adds and flickers peers' presence), and the join ack is liveness-filtered via filterVisiblePresence — both for parity with the tables handler. - platform-authz: honest docstring + 400 message for the workspace-scoped-only authorizeRoom map (workflow authorizes via its own path). - caret-presence: corrected an over-stated batching comment. - +1 workflow regression test (post-success failure keeps the user joined). Gates: tsc (sim/realtime/packages) 0, 205 realtime + 11 protocol tests, biome, api-validation, boundaries, prune. --- apps/realtime/src/handlers/tables.ts | 17 +++++++--- apps/realtime/src/handlers/workflow.test.ts | 24 ++++++++++++++ apps/realtime/src/handlers/workflow.ts | 31 ++++++++++++++----- apps/realtime/src/rooms/redis-manager.ts | 11 +++++-- .../collaboration/caret-presence.ts | 8 +++-- packages/platform-authz/src/rooms.ts | 8 +++-- 6 files changed, 78 insertions(+), 21 deletions(-) diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index a29f50ac519..e1126d04484 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -95,6 +95,9 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // check right before the membership commit. const superseded = () => joinGeneration !== joinAttempt || socket.disconnected if (superseded()) return + // Flips true once addUserToRoom lands: past that point the user is genuinely joined, so a + // failure in the trailing ack/broadcast steps must NOT roll them back (see the catch). + let committed = false try { const userId = socket.userId const userName = socket.userName @@ -204,6 +207,7 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // this write can land after it, leaving a stale presence entry. Benign and self-correcting: // filterVisiblePresence hides it and sweepStalePresence reclaims it (same as the siblings). await roomManager.addUserToRoom(room, socket.id, presence) + committed = true // Filter the join ack to live members so a new joiner never briefly sees a // ghost from an entry the sweep hasn't reclaimed yet. @@ -223,11 +227,14 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR logger.info(`User ${userId} (${userName}) joined table room ${tableId}`) } catch (error) { logger.error('Error joining table room:', error) - // Always roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` - // that landed without a matching `addUserToRoom` (a throw in between) would otherwise leave - // the socket stranded in the Socket.IO room, unreclaimable by any later op. Safe to run even - // when superseded — serialization means the newer op hasn't committed yet, so this touches - // only this join's own (this-table) state, never the newer op's room. + // Past the membership commit the user is genuinely joined; a failure in the trailing + // ack/broadcast steps must not tear them out of the room. Leave them joined. + if (committed) return + // Roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` that + // landed without a matching `addUserToRoom` (a throw in between) would otherwise leave the + // socket stranded in the Socket.IO room, unreclaimable by any later op. Safe to run even when + // superseded — serialization means the newer op hasn't committed yet, so this touches only + // this join's own (this-table) state, never the newer op's room. try { const room = tableRoom(tableId) socket.leave(roomName(room)) diff --git a/apps/realtime/src/handlers/workflow.test.ts b/apps/realtime/src/handlers/workflow.test.ts index 5c46c27d4dc..35543f6f931 100644 --- a/apps/realtime/src/handlers/workflow.test.ts +++ b/apps/realtime/src/handlers/workflow.test.ts @@ -350,6 +350,30 @@ describe('setupWorkflowHandlers', () => { ) }) + it('does not roll back a committed join when a post-success step fails', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + // Trailing broadcast (post-addUserToRoom, post-success-ack) fails on a Redis blip. + broadcastPresenceUpdate: vi.fn().mockRejectedValue(new Error('redis blip')), + }) + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + + // The user is genuinely joined and was acked; the trailing failure must NOT tear them out. + expect(socket.emit).toHaveBeenCalledWith( + 'join-workflow-success', + expect.objectContaining({ workflowId: 'workflow-1' }) + ) + expect(socket.leave).not.toHaveBeenCalled() + expect(roomManager.removeUserFromRoom).not.toHaveBeenCalled() + expect(socket.emit).not.toHaveBeenCalledWith('join-workflow-error', expect.anything()) + }) + it('leaves the workflow room even when the session key has expired', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager({ diff --git a/apps/realtime/src/handlers/workflow.ts b/apps/realtime/src/handlers/workflow.ts index afc99d37c03..31d5094fd3a 100644 --- a/apps/realtime/src/handlers/workflow.ts +++ b/apps/realtime/src/handlers/workflow.ts @@ -5,6 +5,7 @@ import { resolveAvatarUrl } from '@/handlers/avatar' import type { AuthenticatedSocket } from '@/middleware/auth' import { resolveCurrentWorkflowRole, verifyWorkflowAccess } from '@/middleware/permissions' import { type IRoomManager, type UserPresence, workflowRoom as wf } from '@/rooms' +import { filterVisiblePresence } from '@/rooms/presence-visibility' const logger = createLogger('WorkflowHandlers') @@ -41,6 +42,9 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: // check right before the membership commit. const superseded = () => joinGeneration !== joinAttempt || socket.disconnected if (superseded()) return + // Flips true once addUserToRoom lands: past that point the user is genuinely joined, so a + // failure in the trailing ack/broadcast/metric steps must NOT roll them back (see the catch). + let committed = false try { const userId = socket.userId const userName = socket.userName @@ -95,9 +99,10 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: return } - // Leave current workflow room if in one + // Leave a previously-joined workflow room if switching workflows. Guard on a DIFFERENT id so a + // re-join of the SAME workflow doesn't leave→re-add and flicker presence for peers. const currentRoom = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) - if (currentRoom) { + if (currentRoom && currentRoom.id !== workflowId) { socket.leave(currentRoom.id) await roomManager.removeUserFromRoom(currentRoom, socket.id) await roomManager.broadcastPresenceUpdate(currentRoom) @@ -216,11 +221,17 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: avatarUrl, } - // Add user to room + // Add user to room — the membership commit. await roomManager.addUserToRoom(wf(workflowId), socket.id, userPresence) + committed = true - // Get current presence list for the join acknowledgment - const presenceUsers = await roomManager.getRoomUsers(wf(workflowId)) + // Get current presence list for the join acknowledgment, filtered to live members so a new + // joiner never sees a ghost from an entry the stale sweep hasn't reclaimed yet. + const presenceUsers = await filterVisiblePresence( + roomManager.io, + wf(workflowId), + await roomManager.getRoomUsers(wf(workflowId)) + ) // Get workflow state const workflowState = await getWorkflowState(workflowId) @@ -244,9 +255,13 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: ) } catch (error) { logger.error('Error joining workflow:', error) - // Always roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` - // that landed without a matching `addUserToRoom` (a throw in between) would otherwise strand - // the socket in the Socket.IO room, unreclaimable by any later op. Safe even when superseded — + // Past the membership commit the user is genuinely joined; a failure in the trailing + // ack/broadcast/metric steps must not tear them out of the room (a benign Redis blip on a + // pure log-metric call would otherwise kick a live collaborator). Leave them joined. + if (committed) return + // Roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` that + // landed without a matching `addUserToRoom` (a throw in between) would otherwise strand the + // socket in the Socket.IO room, unreclaimable by any later op. Safe even when superseded — // serialization means the newer op hasn't committed yet, so this touches only this join's own // room state, never the newer op's. socket.leave(workflowId) diff --git a/apps/realtime/src/rooms/redis-manager.ts b/apps/realtime/src/rooms/redis-manager.ts index cffb9a6d661..919ea8bf336 100644 --- a/apps/realtime/src/rooms/redis-manager.ts +++ b/apps/realtime/src/rooms/redis-manager.ts @@ -31,8 +31,15 @@ const KEYS = { /** TTL for the socket's room-set. Long enough that an idle-but-connected socket is not evicted. */ const SOCKET_ROOMS_TTL = 24 * 60 * 60 -/** TTL for the shared session key; refreshed on every activity update. */ -const SESSION_TTL = 60 * 60 +/** + * The shared session key MUST share the room-set TTL. `getRoomForSocket` reads the room-set and the + * workflow handlers gate edits/presence on `room && session`, so a session that expired while the + * room-set is still alive would wedge an active-but-idle collaborator into "session expired" until a + * full reload — the activity update only `EXPIRE`s the session, which cannot resurrect an + * already-gone key (only `addUserToRoom` re-`HSET`s it). Both are refreshed together on every + * activity, so they expire together. + */ +const SESSION_TTL = SOCKET_ROOMS_TTL /** * Atomic single-room removal. Removes a socket from one room's presence, drops diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts index eff0ce70534..8484739446a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts @@ -107,9 +107,11 @@ export function createCaretActivityExtension(awareness: Awareness): Extension { new Plugin({ key: new PluginKey('collaborationCaretActivity'), view: (editorView) => { - // Coalesce bursts of awareness changes into a single rAF: at most one forced - // layout read + one flush per frame, however many peers moved. Accumulate the - // changed client ids, then re-activate each matching (reused) caret node. + // Coalesce bursts of awareness changes into a single rAF per frame, however many + // peers moved: accumulate the changed client ids, then re-activate each matching + // (reused) caret node once. The shared editorRight is read once up front; each moving + // caret then does one getBoundingClientRect for its edge-flip measure (bounded by the + // small number of concurrently-moving peers, not the awareness event rate). let raf = 0 const pending = new Set() const flush = () => { diff --git a/packages/platform-authz/src/rooms.ts b/packages/platform-authz/src/rooms.ts index 19750096833..3688271d93a 100644 --- a/packages/platform-authz/src/rooms.ts +++ b/packages/platform-authz/src/rooms.ts @@ -21,8 +21,8 @@ export interface RoomWorkspace { /** * Resolves a room's owning workspace from its {@link RoomRef.id}. Returns `null` * when the underlying resource is missing/archived (→ a 404 authorization - * result). One resolver per {@link RoomType}; this is the single place a new - * room type declares its resource→workspace lookup. + * result). One resolver per workspace-scoped {@link RoomType}; this is the single + * place a new such room type declares its resource→workspace lookup. */ export type RoomWorkspaceResolver = (roomId: string) => Promise @@ -120,10 +120,12 @@ export async function authorizeRoom(params: { const resolver = ROOM_WORKSPACE_RESOLVERS[room.type] if (!resolver) { + // Either an unknown type or one deliberately outside this authorizer (workflow authorizes via + // its own path); both are not-authorizable-here → 400. return { allowed: false, status: 400, - message: `Unknown room type: ${room.type}`, + message: `Room type not authorizable here: ${room.type}`, workspaceId: null, workspacePermission: null, } From 42d1d322af6991dfd1259043d15198700f78b763 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 20:28:38 -0700 Subject: [PATCH 24/53] fix(realtime): narrow join commit-guard to post-success; skip empty presence broadcast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Cursor findings on the prior audit-fix commit: - The 'committed' guard in join-workflow/join-table was too broad: a failure BETWEEN the membership commit and the success ack (e.g. getWorkflowState) hit 'if (committed) return' and emitted neither success nor error, hanging the client while it sat in the room. Replaced with the narrower shape: only the purely-decorative post-success steps (peer broadcast + log-metric) are wrapped best-effort; anything before the success ack still rolls back and surfaces a retryable error, so the client retries instead of hanging — while the original goal (a benign broadcast/metric blip never kicking a live, acked user) holds. - broadcastPresenceUpdate read the roster via getRoomUsers, which swallows a Redis transport error to []. On a disconnect broadcast that emitted an empty roster and cleared every remaining collaborator's presence until the next healthy update. Split out a throwing readRoomUsers; broadcastPresenceUpdate now skips the broadcast on a read failure (getRoomUsers keeps its swallow contract). - Tests: pre-success failure rolls back + retryable error (no hang); post-success failure keeps the user joined. Gates: realtime tsc 0, 206 realtime tests, biome, boundaries, prune. --- apps/realtime/src/handlers/tables.ts | 23 ++++++----- apps/realtime/src/handlers/workflow.test.ts | 25 +++++++++++ apps/realtime/src/handlers/workflow.ts | 32 +++++++------- apps/realtime/src/rooms/redis-manager.ts | 46 +++++++++++++++------ 4 files changed, 87 insertions(+), 39 deletions(-) diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index e1126d04484..86c0f0da8a8 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -95,9 +95,6 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // check right before the membership commit. const superseded = () => joinGeneration !== joinAttempt || socket.disconnected if (superseded()) return - // Flips true once addUserToRoom lands: past that point the user is genuinely joined, so a - // failure in the trailing ack/broadcast steps must NOT roll them back (see the catch). - let committed = false try { const userId = socket.userId const userName = socket.userName @@ -207,7 +204,6 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // this write can land after it, leaving a stale presence entry. Benign and self-correcting: // filterVisiblePresence hides it and sweepStalePresence reclaims it (same as the siblings). await roomManager.addUserToRoom(room, socket.id, presence) - committed = true // Filter the join ack to live members so a new joiner never briefly sees a // ghost from an entry the sweep hasn't reclaimed yet. @@ -222,19 +218,24 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR presenceUsers, }) - await roomManager.broadcastPresenceUpdate(room) + // Post-success, purely decorative: notify peers. The user is already joined and acked, so a + // Redis blip here must not surface as a join failure — swallow it (the next healthy broadcast + // reconciles peers). Kept OUT of the rollback catch below, which is only for pre-success failures. + try { + await roomManager.broadcastPresenceUpdate(room) + } catch (error) { + logger.warn(`Post-join presence broadcast failed for table room ${tableId}`, error) + } logger.info(`User ${userId} (${userName}) joined table room ${tableId}`) } catch (error) { logger.error('Error joining table room:', error) - // Past the membership commit the user is genuinely joined; a failure in the trailing - // ack/broadcast steps must not tear them out of the room. Leave them joined. - if (committed) return // Roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` that // landed without a matching `addUserToRoom` (a throw in between) would otherwise leave the - // socket stranded in the Socket.IO room, unreclaimable by any later op. Safe to run even when - // superseded — serialization means the newer op hasn't committed yet, so this touches only - // this join's own (this-table) state, never the newer op's room. + // socket stranded in the Socket.IO room, unreclaimable by any later op. A failure between the + // commit and the success ack rolls back too and surfaces a retryable error, so the client + // retries rather than hanging. Safe to run even when superseded — serialization means the + // newer op hasn't committed yet, so this touches only this join's own (this-table) state. try { const room = tableRoom(tableId) socket.leave(roomName(room)) diff --git a/apps/realtime/src/handlers/workflow.test.ts b/apps/realtime/src/handlers/workflow.test.ts index 35543f6f931..b6c4ab26190 100644 --- a/apps/realtime/src/handlers/workflow.test.ts +++ b/apps/realtime/src/handlers/workflow.test.ts @@ -374,6 +374,31 @@ describe('setupWorkflowHandlers', () => { expect(socket.emit).not.toHaveBeenCalledWith('join-workflow-error', expect.anything()) }) + it('rolls back and surfaces a retryable error when a pre-success step fails after commit', async () => { + // getWorkflowState runs after addUserToRoom but before the success ack — its failure must roll + // back and emit a retryable error so the client retries, never hanging committed-but-unacked. + mockGetWorkflowState.mockRejectedValue(new Error('db blip')) + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + + expect(socket.emit).not.toHaveBeenCalledWith('join-workflow-success', expect.anything()) + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-1' }, + 'socket-1' + ) + expect(socket.emit).toHaveBeenCalledWith( + 'join-workflow-error', + expect.objectContaining({ code: 'JOIN_WORKFLOW_FAILED', retryable: true }) + ) + }) + it('leaves the workflow room even when the session key has expired', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager({ diff --git a/apps/realtime/src/handlers/workflow.ts b/apps/realtime/src/handlers/workflow.ts index 31d5094fd3a..21fde415c12 100644 --- a/apps/realtime/src/handlers/workflow.ts +++ b/apps/realtime/src/handlers/workflow.ts @@ -42,9 +42,6 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: // check right before the membership commit. const superseded = () => joinGeneration !== joinAttempt || socket.disconnected if (superseded()) return - // Flips true once addUserToRoom lands: past that point the user is genuinely joined, so a - // failure in the trailing ack/broadcast/metric steps must NOT roll them back (see the catch). - let committed = false try { const userId = socket.userId const userName = socket.userName @@ -223,7 +220,6 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: // Add user to room — the membership commit. await roomManager.addUserToRoom(wf(workflowId), socket.id, userPresence) - committed = true // Get current presence list for the join acknowledgment, filtered to live members so a new // joiner never sees a ghost from an entry the stale sweep hasn't reclaimed yet. @@ -246,22 +242,26 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: // Send workflow state socket.emit('workflow-state', workflowState) - // Broadcast presence update to all users in the room - await roomManager.broadcastPresenceUpdate(wf(workflowId)) - - const uniqueUserCount = await roomManager.getUniqueUserCount(wf(workflowId)) - logger.info( - `User ${userId} (${userName}) joined workflow ${workflowId}. Room now has ${uniqueUserCount} unique users.` - ) + // Post-success, purely decorative: notify peers and log the count. The user is already joined + // and acked, so a Redis blip here must not surface as a join failure — swallow it (the next + // healthy presence broadcast reconciles peers). It must stay OUT of the rollback catch below, + // which is only for pre-success failures. + try { + await roomManager.broadcastPresenceUpdate(wf(workflowId)) + const uniqueUserCount = await roomManager.getUniqueUserCount(wf(workflowId)) + logger.info( + `User ${userId} (${userName}) joined workflow ${workflowId}. Room now has ${uniqueUserCount} unique users.` + ) + } catch (error) { + logger.warn(`Post-join presence broadcast failed for workflow ${workflowId}`, error) + } } catch (error) { logger.error('Error joining workflow:', error) - // Past the membership commit the user is genuinely joined; a failure in the trailing - // ack/broadcast/metric steps must not tear them out of the room (a benign Redis blip on a - // pure log-metric call would otherwise kick a live collaborator). Leave them joined. - if (committed) return // Roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` that // landed without a matching `addUserToRoom` (a throw in between) would otherwise strand the - // socket in the Socket.IO room, unreclaimable by any later op. Safe even when superseded — + // socket in the Socket.IO room, unreclaimable by any later op. A failure between the commit + // and the success ack rolls back too and surfaces a retryable error — so the client retries + // rather than hanging (never left committed-but-unacked). Safe even when superseded — // serialization means the newer op hasn't committed yet, so this touches only this join's own // room state, never the newer op's. socket.leave(workflowId) diff --git a/apps/realtime/src/rooms/redis-manager.ts b/apps/realtime/src/rooms/redis-manager.ts index 919ea8bf336..139c8e3a121 100644 --- a/apps/realtime/src/rooms/redis-manager.ts +++ b/apps/realtime/src/rooms/redis-manager.ts @@ -298,19 +298,28 @@ export class RedisRoomManager implements IRoomManager { } } + /** + * Reads and parses the room roster. Throws on a transport error (so a caller can + * distinguish "genuinely empty" from "read failed"); a single corrupted entry is + * skipped, not fatal. + */ + private async readRoomUsers(room: RoomRef): Promise { + const users = await this.redis.hGetAll(KEYS.roomUsers(room)) + return Object.entries(users) + .map(([socketId, json]) => { + try { + return JSON.parse(json) as UserPresence + } catch { + logger.warn(`Corrupted user data for socket ${socketId}, skipping`) + return null + } + }) + .filter((u): u is UserPresence => u !== null) + } + async getRoomUsers(room: RoomRef): Promise { try { - const users = await this.redis.hGetAll(KEYS.roomUsers(room)) - return Object.entries(users) - .map(([socketId, json]) => { - try { - return JSON.parse(json) as UserPresence - } catch { - logger.warn(`Corrupted user data for socket ${socketId}, skipping`) - return null - } - }) - .filter((u): u is UserPresence => u !== null) + return await this.readRoomUsers(room) } catch (error) { logger.error(`Failed to get room users for ${room.type}:${room.id}:`, error) return [] @@ -374,7 +383,20 @@ export class RedisRoomManager implements IRoomManager { } async broadcastPresenceUpdate(room: RoomRef, excludeSocketId?: string): Promise { - const users = await this.getRoomUsers(room) + let users: UserPresence[] + try { + // Read via the throwing variant, NOT getRoomUsers: a transport error there returns `[]`, which + // would broadcast an empty roster and clear every remaining collaborator's presence until the + // next healthy update. Skip instead — the next successful join/activity broadcast (or the + // stale sweep) reconciles peers. + users = await this.readRoomUsers(room) + } catch (error) { + logger.error( + `Skipping presence broadcast for ${room.type}:${room.id} (roster read failed):`, + error + ) + return + } const visible = await filterVisiblePresence(this._io, room, users, excludeSocketId) // io.to() with the Redis adapter broadcasts to all pods. this._io.to(roomName(room)).emit(presenceEventName(room.type), visible) From 8ae486a64f9a1139216cca9396fa48b73f3a7bac Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 23:15:48 -0700 Subject: [PATCH 25/53] fix(realtime): harden seeder recovery, join-generation, and misc robustness Final line-by-line audit follow-ups (all LOW/MED, no P0/P1): - file-doc: a sole client whose seed FETCH fails was added to triedSeeders, re-election found nobody, and the document stayed permanently empty until reload. Re-offer seeding a bounded number of rounds (MAX_SEED_ROUNDS) before giving up. Also bound clientId to a non-negative integer (it is an ownership key). - tables + workspace-files: validate the room id BEFORE advancing joinGeneration, so a malformed/rejected join can't cancel a legitimate in-flight join. - workflow + tables + workspace-files: suppress the client-facing join error when the op was already superseded (a retryable error naming the abandoned room could make a client re-join and cancel its newer join). The rollback still runs. - redis-manager: set isConnected=true only after scriptLoad succeeds (and reset it on failure) so isReady() can't report ready while the Lua SHAs are null. - connection: apply the presence-bearing filter to the manager-removed set too (symmetry with the fallback path). - http.ts: validate workflowId on the four workflow endpoints (matching the files one). - client: clear a pending join-retry timer before rescheduling (reconnect churn no longer orphans a stray extra join); clear caret fade timers on plugin destroy; seed-effect cleanup reports NOT-ready (safe direction). - Tests: bounded seeder recovery, cell-selection strip-junk, TABLE round-trip, presenceEventName. Gates: tsc (sim/realtime/packages) 0, 208 realtime + 12 protocol tests, biome, api-validation, boundaries, prune. --- apps/realtime/src/handlers/connection.ts | 8 ++- apps/realtime/src/handlers/file-doc.test.ts | 16 +++++ apps/realtime/src/handlers/file-doc.ts | 37 +++++++++- apps/realtime/src/handlers/tables.test.ts | 31 ++++++++ apps/realtime/src/handlers/tables.ts | 26 ++++--- apps/realtime/src/handlers/workflow.ts | 4 ++ apps/realtime/src/handlers/workspace-files.ts | 70 ++++++++++--------- apps/realtime/src/rooms/redis-manager.ts | 7 +- apps/realtime/src/routes/http.ts | 8 +++ .../collaboration/caret-presence.ts | 11 +++ .../rich-markdown-editor.tsx | 5 +- .../files/hooks/use-workspace-files-room.ts | 3 + .../tables/[tableId]/hooks/use-table-room.ts | 3 + packages/realtime-protocol/src/rooms.test.ts | 15 ++++ 14 files changed, 197 insertions(+), 47 deletions(-) diff --git a/apps/realtime/src/handlers/connection.ts b/apps/realtime/src/handlers/connection.ts index bd6d143aa36..33d90b5bfb0 100644 --- a/apps/realtime/src/handlers/connection.ts +++ b/apps/realtime/src/handlers/connection.ts @@ -54,7 +54,13 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager // rooms empty). Attempt removal for any room the manager didn't already // remove — best-effort, since a transient Redis error can't be recovered here. const wasInRooms = new Map() - for (const room of removedRooms) wasInRooms.set(roomName(room), room) + // Only presence-bearing rooms get a corrective broadcast. Manager-removed rooms are + // presence-bearing by construction today (only workflow/table write the socket→room hash), + // but filter symmetrically with the fallback path below so a future room type that ever + // tracks presence here can't emit a bogus presence-update no client listens to. + for (const room of removedRooms) { + if (PRESENCE_BEARING_TYPES.has(room.type)) wasInRooms.set(roomName(room), room) + } for (const name of liveRoomNames) { // `wasInRooms.has(name)` already excludes every room the manager removed (same // room-name key via the roomName/parseRoomName bijection). Skip room types with no diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index f78df6d8a81..1ab6df0fb0a 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -665,6 +665,22 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') }) + it('re-offers seeding to a sole client that missed the deadline, then gives up after a bound', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + expect(sent.filter((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST).length).toBe(1) + + // The sole client keeps missing the deadline. Without recovery it would be permanently excluded + // (empty document forever); instead it is re-offered a bounded number of rounds. + for (let i = 0; i < 6; i++) vi.advanceTimersByTime(10_000) + + // Initial offer + MAX_SEED_ROUNDS (3) re-offers = 4 total, all to socket-a; then it stops. + const requests = sent.filter((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) + expect(requests.length).toBe(4) + expect(requests.every((m) => m.target === 'socket-a')).toBe(true) + }) + it('cancels the seed deadline once the document is seeded', async () => { const { io, sent } = createIo() const a = setup('socket-a', io) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 506f001ee53..2acee6b7a19 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -50,6 +50,16 @@ const logger = createLogger('FileDocHandlers') */ const SEED_DEADLINE_MS = 10_000 +/** + * Max times the room re-offers seeding to its owners after every one of them has + * missed a deadline. Without this, a sole client whose seed *fetch fails* (not just + * slow) is added to `triedSeeders`, re-election finds no un-tried owner, and the + * document is left permanently empty until that client reloads. Each exhausted round + * clears `triedSeeders` and re-offers, giving a transient failure a bounded number of + * retries (~MAX × deadline) before the room genuinely gives up. + */ +const MAX_SEED_ROUNDS = 3 + /** A socket's presence ownership within a room. */ interface FileDocOwner { /** @@ -81,6 +91,10 @@ interface FileDocRoom { /** Sockets that were elected but failed to seed within the deadline; skipped * on re-election so a single stuck/withholding client can't block the room. */ triedSeeders: Set + /** Count of full re-offer rounds spent after every owner missed a deadline; + * bounds the retry loop (see {@link MAX_SEED_ROUNDS}) so a permanently-broken + * seed doesn't reset forever. */ + seedRounds: number } /** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */ @@ -222,6 +236,19 @@ function electSeederIfNeeded(io: Server, room: FileDocRoom) { room.triedSeeders.add(elected) room.seederSocketId = null electSeederIfNeeded(io, room) + // If that left no seeder (every current owner has now been tried) while the doc is still + // unseeded and owners remain, re-offer the whole room a bounded number of times — otherwise a + // sole client whose seed fetch failed leaves the document permanently empty. + if ( + room.seederSocketId === null && + !isDocSeeded(room.doc) && + room.owners.size > 0 && + room.seedRounds < MAX_SEED_ROUNDS + ) { + room.seedRounds += 1 + room.triedSeeders.clear() + electSeederIfNeeded(io, room) + } }, SEED_DEADLINE_MS) } @@ -248,6 +275,7 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { seederSocketId: null, seedTimer: null, triedSeeders: new Set(), + seedRounds: 0, } fileDocRooms.set(name, room) @@ -418,7 +446,14 @@ export function setupWorkspaceFileDocHandlers( emitJoinError(socket, fileId, 'Realtime unavailable', 'ROOM_MANAGER_UNAVAILABLE', true) return } - if (typeof fileId !== 'string' || fileId.length === 0 || typeof clientId !== 'number') { + if ( + typeof fileId !== 'string' || + fileId.length === 0 || + // A Yjs clientID is a uint32; reject NaN/Infinity/negative/non-integer so a malformed id + // can't become a bogus ownership key. + !Number.isInteger(clientId) || + clientId < 0 + ) { emitJoinError(socket, fileId, 'Invalid join payload', 'INVALID_PAYLOAD', false) return } diff --git a/apps/realtime/src/handlers/tables.test.ts b/apps/realtime/src/handlers/tables.test.ts index 73b0ea62678..518c5ee376a 100644 --- a/apps/realtime/src/handlers/tables.test.ts +++ b/apps/realtime/src/handlers/tables.test.ts @@ -187,6 +187,37 @@ describe('setupTablesHandlers', () => { expect(toEmit).not.toHaveBeenCalled() }) + it('strips unknown/oversized fields from an otherwise-valid selection before storing or relaying', async () => { + const { socket, handlers, toEmit } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue(TABLE_ROOM), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.CELL_SELECTION]({ + cell: { + anchor: { rowId: 'row-1', columnId: 'col-a', junk: 'x'.repeat(10_000) }, + focus: { rowId: 'row-1', columnId: 'col-a' }, + editing: true, + bloat: 'x'.repeat(100_000), + }, + }) + + // Only the whitelisted fields survive — a hostile peer can't amplify an oversized object. + const expected = { + anchor: { rowId: 'row-1', columnId: 'col-a' }, + focus: { rowId: 'row-1', columnId: 'col-a' }, + editing: true, + } + expect(roomManager.updateUserActivity).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1', { + cell: expected, + }) + expect(toEmit).toHaveBeenCalledWith(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { + socketId: 'socket-1', + cell: expected, + }) + }) + it('skips a superseded queued join on a fast table switch', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager() diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts index 86c0f0da8a8..2474063c9a0 100644 --- a/apps/realtime/src/handlers/tables.ts +++ b/apps/realtime/src/handlers/tables.ts @@ -78,6 +78,17 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR let opChain: Promise = Promise.resolve() socket.on(TABLE_PRESENCE_EVENTS.JOIN, ({ tableId, tabSessionId }: JoinTablePayload) => { + // Validate the id BEFORE claiming a generation, so a malformed join can't advance + // joinGeneration and cancel a legitimate in-flight join for another table. + if (typeof tableId !== 'string' || tableId.length === 0) { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId: typeof tableId === 'string' ? tableId : '', + error: 'Invalid table id', + code: 'INVALID_PAYLOAD', + retryable: false, + }) + return + } const joinAttempt = (joinGeneration += 1) currentTableId = tableId opChain = opChain @@ -119,17 +130,6 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR return } - // Validate the client-supplied id before it reaches the DB query. - if (typeof tableId !== 'string' || tableId.length === 0) { - socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { - tableId: typeof tableId === 'string' ? tableId : '', - error: 'Invalid table id', - code: 'INVALID_PAYLOAD', - retryable: false, - }) - return - } - const room = tableRoom(tableId) const authorized = await resolveRoomJoinAuth({ @@ -244,6 +244,10 @@ export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IR // Best-effort rollback — the original join failure is the one surfaced below, so a // secondary cleanup error must not mask it or throw out of the error handler. } + // Suppress the client-facing error when this join was already superseded: the client has moved + // to a newer table, and a retryable error naming the abandoned one could make it re-join and + // supersede the newer join. The rollback above still runs. + if (superseded()) return socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { tableId, error: 'Failed to join table', diff --git a/apps/realtime/src/handlers/workflow.ts b/apps/realtime/src/handlers/workflow.ts index 21fde415c12..91598a762fc 100644 --- a/apps/realtime/src/handlers/workflow.ts +++ b/apps/realtime/src/handlers/workflow.ts @@ -266,6 +266,10 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: // room state, never the newer op's. socket.leave(workflowId) await roomManager.removeUserFromRoom(wf(workflowId), socket.id) + // Suppress the client-facing error when this join was already superseded: the client has moved + // to a newer workflow, and a retryable error naming the abandoned one could make it re-join and + // supersede the newer join (an A/B flicker). The rollback above still runs. + if (superseded()) return const isReady = roomManager.isReady() socket.emit('join-workflow-error', { workflowId, diff --git a/apps/realtime/src/handlers/workspace-files.ts b/apps/realtime/src/handlers/workspace-files.ts index a4b0604dc9f..1d5fa6ac5ce 100644 --- a/apps/realtime/src/handlers/workspace-files.ts +++ b/apps/realtime/src/handlers/workspace-files.ts @@ -44,41 +44,43 @@ export function setupWorkspaceFilesHandlers( let currentWorkspace: string | null = null socket.on('join-workspace-files', async ({ workspaceId }: JoinPayload) => { - const joinAttempt = (joinGeneration += 1) - currentWorkspace = workspaceId - try { - if (!socket.userId || !socket.userName) { - socket.emit('join-workspace-files-error', { - workspaceId, - error: 'Authentication required', - code: 'AUTHENTICATION_REQUIRED', - retryable: false, - }) - return - } + // Validate synchronously BEFORE claiming a generation, so a rejected/malformed join can't + // advance joinGeneration and cancel a legitimate in-flight join for another workspace. + if (!socket.userId || !socket.userName) { + socket.emit('join-workspace-files-error', { + workspaceId, + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + return + } - if (!roomManager.isReady()) { - socket.emit('join-workspace-files-error', { - workspaceId, - error: 'Realtime unavailable', - code: 'ROOM_MANAGER_UNAVAILABLE', - retryable: true, - }) - return - } + if (!roomManager.isReady()) { + socket.emit('join-workspace-files-error', { + workspaceId, + error: 'Realtime unavailable', + code: 'ROOM_MANAGER_UNAVAILABLE', + retryable: true, + }) + return + } - // Validate the client-supplied id before it reaches the DB query (join payloads are - // otherwise raw client input). - if (typeof workspaceId !== 'string' || workspaceId.length === 0) { - socket.emit('join-workspace-files-error', { - workspaceId: typeof workspaceId === 'string' ? workspaceId : '', - error: 'Invalid workspace id', - code: 'INVALID_PAYLOAD', - retryable: false, - }) - return - } + // Validate the client-supplied id before it reaches the DB query (join payloads are + // otherwise raw client input) and before advancing the generation. + if (typeof workspaceId !== 'string' || workspaceId.length === 0) { + socket.emit('join-workspace-files-error', { + workspaceId: typeof workspaceId === 'string' ? workspaceId : '', + error: 'Invalid workspace id', + code: 'INVALID_PAYLOAD', + retryable: false, + }) + return + } + const joinAttempt = (joinGeneration += 1) + currentWorkspace = workspaceId + try { const room = filesRoom(workspaceId) const authorized = await resolveRoomJoinAuth({ @@ -115,6 +117,10 @@ export function setupWorkspaceFilesHandlers( try { socket.leave(roomName(filesRoom(workspaceId))) } catch {} + // Suppress the client-facing error when this join was already superseded: the client has + // switched to a newer workspace, and a retryable error naming the abandoned one could make it + // re-join and cancel the newer join. The leave above still runs. + if (joinGeneration !== joinAttempt || socket.disconnected) return socket.emit('join-workspace-files-error', { workspaceId, error: 'Failed to join workspace files', diff --git a/apps/realtime/src/rooms/redis-manager.ts b/apps/realtime/src/rooms/redis-manager.ts index 139c8e3a121..21df72ef3b6 100644 --- a/apps/realtime/src/rooms/redis-manager.ts +++ b/apps/realtime/src/rooms/redis-manager.ts @@ -168,13 +168,18 @@ export class RedisRoomManager implements IRoomManager { try { await this.redis.connect() - this.isConnected = true this.removeRoomScriptSha = await this.redis.scriptLoad(REMOVE_ROOM_SCRIPT) this.updateActivityScriptSha = await this.redis.scriptLoad(UPDATE_ACTIVITY_SCRIPT) + // Mark ready only after the scripts load — isReady() gates removeUserFromRoom/updateUserActivity, + // which silently no-op without a script SHA. Setting the flag before scriptLoad would make + // isReady() lie if scriptLoad threw. + this.isConnected = true + logger.info('RedisRoomManager connected to Redis and scripts loaded') } catch (error) { + this.isConnected = false logger.error('Failed to connect to Redis:', error) throw error } diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index d8db9473cb6..0b6264b8f8b 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -42,6 +42,10 @@ function readRequestBody(req: IncomingMessage): Promise { }) } +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 +} + function sendSuccess(res: ServerResponse): void { res.writeHead(200, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ success: true })) @@ -102,6 +106,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId } = JSON.parse(body) + if (!isNonEmptyString(workflowId)) return sendError(res, 'Invalid workflowId', 400) await workflowRoomService.handleWorkflowDeletion(workflowId) sendSuccess(res) } catch (error) { @@ -116,6 +121,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId } = JSON.parse(body) + if (!isNonEmptyString(workflowId)) return sendError(res, 'Invalid workflowId', 400) await workflowRoomService.handleWorkflowUpdate(workflowId) sendSuccess(res) } catch (error) { @@ -130,6 +136,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId } = JSON.parse(body) + if (!isNonEmptyString(workflowId)) return sendError(res, 'Invalid workflowId', 400) await workflowRoomService.handleWorkflowDeployed(workflowId) sendSuccess(res) } catch (error) { @@ -144,6 +151,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId, timestamp } = JSON.parse(body) + if (!isNonEmptyString(workflowId)) return sendError(res, 'Invalid workflowId', 400) await workflowRoomService.handleWorkflowRevert(workflowId, timestamp) sendSuccess(res) } catch (error) { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts index 8484739446a..fdbfadce592 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts @@ -135,6 +135,17 @@ export function createCaretActivityExtension(awareness: Awareness): Extension { destroy: () => { awareness.off('change', onChange) if (raf) cancelAnimationFrame(raf) + // Clear any pending fade timers for this editor's carets so they don't fire on + // detached nodes after unmount (harmless no-op, but a genuine leaked timer). + for (const caret of editorView.dom.querySelectorAll( + '.collaboration-carets__caret' + )) { + const timer = caretFadeTimers.get(caret) + if (timer) { + clearTimeout(timer) + caretFadeTimers.delete(caret) + } + } }, } }, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index c7bd689c7d7..833da70e7cb 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -688,7 +688,10 @@ export function LoadedRichMarkdownEditor({ provider.off('synced', onProgress) provider.off('join-error', onJoinError) config.unobserve(report) - onCollabReadyChange(true) + // Report NOT ready on teardown — the safe direction. If this effect ever re-runs while mounted + // (a future dep change), briefly gating autosave off is harmless; reporting `true` here could + // ungate it while the doc is unready. + onCollabReadyChange(false) } }, [collaboration, editor, onCollabReadyChange, setCollabReady]) diff --git a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts index d8ab7efa144..2a87d993cc6 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts @@ -55,6 +55,9 @@ export function useWorkspaceFilesRoom(workspaceId: string): void { logger.warn('Failed to join workspace files room', { code: data.code, error: data.error }) if (data.retryable && retries < MAX_JOIN_RETRIES) { retries += 1 + // Clear any still-pending retry before scheduling a new one, so reconnect churn can't + // orphan a timer that fires an extra join(). + if (retryTimer) clearTimeout(retryTimer) retryTimer = setTimeout(join, JOIN_RETRY_BASE_MS * retries) } } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts index 72e7c7036ce..57083ff75e6 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts @@ -94,6 +94,9 @@ export function useTableRoom(tableId: string): UseTableRoomResult { logger.warn('Failed to join table room', { code: data.code, error: data.error }) if (data.retryable && retries < MAX_JOIN_RETRIES) { retries += 1 + // Clear any still-pending retry before scheduling a new one, so reconnect churn can't + // orphan a timer that fires an extra join(). + if (retryTimer) clearTimeout(retryTimer) retryTimer = setTimeout(join, JOIN_RETRY_BASE_MS * retries) } } diff --git a/packages/realtime-protocol/src/rooms.test.ts b/packages/realtime-protocol/src/rooms.test.ts index cbf9b3cd5b2..bb3651e9978 100644 --- a/packages/realtime-protocol/src/rooms.test.ts +++ b/packages/realtime-protocol/src/rooms.test.ts @@ -3,6 +3,7 @@ import { ALL_ROOM_TYPES, isRoomType, parseRoomName, + presenceEventName, ROOM_TYPES, type RoomRef, roomName, @@ -49,6 +50,7 @@ describe('parseRoomName', () => { { type: ROOM_TYPES.WORKFLOW, id: 'wf-123' }, { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-456' }, { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: 'file-789' }, + { type: ROOM_TYPES.TABLE, id: 'table-abc' }, ] for (const ref of refs) { expect(parseRoomName(roomName(ref))).toEqual(ref) @@ -88,3 +90,16 @@ describe('ALL_ROOM_TYPES', () => { expect([...ALL_ROOM_TYPES].sort()).toEqual([...Object.values(ROOM_TYPES)].sort()) }) }) + +describe('presenceEventName', () => { + it('keeps the bare historical name for workflow and namespaces the rest', () => { + // Workflow keeps `presence-update` for client back-compat; every other type is namespaced so a + // socket in more than one room can demux its presence streams. + expect(presenceEventName(ROOM_TYPES.WORKFLOW)).toBe('presence-update') + expect(presenceEventName(ROOM_TYPES.TABLE)).toBe('table:presence-update') + expect(presenceEventName(ROOM_TYPES.WORKSPACE_FILES)).toBe('workspace-files:presence-update') + // No two types share an event name. + const names = ALL_ROOM_TYPES.map(presenceEventName) + expect(new Set(names).size).toBe(names.length) + }) +}) From 19a5abda0f94e1e05d73f5f5dc82d3d637c08f72 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 23:22:19 -0700 Subject: [PATCH 26/53] fix(realtime): gate isReady() on loaded script SHAs, not just connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the prior isConnected change, which was incomplete: the redis client's 'ready' event flips isConnected=true on connect — before initialize() loads the Lua scripts — so a bare isConnected check reports ready while removeUserFromRoom / updateUserActivity would silently no-op on a null SHA. Gate isReady() on the SHAs too, so the POST endpoints return a retryable 503 during that startup window instead of proceeding against unloaded scripts. Standard readiness-probe discipline. --- apps/realtime/src/rooms/redis-manager.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/realtime/src/rooms/redis-manager.ts b/apps/realtime/src/rooms/redis-manager.ts index 21df72ef3b6..7282f4a6778 100644 --- a/apps/realtime/src/rooms/redis-manager.ts +++ b/apps/realtime/src/rooms/redis-manager.ts @@ -160,7 +160,14 @@ export class RedisRoomManager implements IRoomManager { } isReady(): boolean { - return this.isConnected + // Gate on the loaded script SHAs, not just the connection: the `ready` event flips + // `isConnected` true as soon as the socket connects — before `initialize()` loads the Lua + // scripts — so a bare `isConnected` check would report ready while `removeUserFromRoom` / + // `updateUserActivity` would silently no-op on a null SHA. Reporting not-ready here makes the + // POST endpoints return a retryable 503 during that startup window instead. + return ( + this.isConnected && this.removeRoomScriptSha !== null && this.updateActivityScriptSha !== null + ) } async initialize(): Promise { From 74216285f22a3657d83553eff846228592a9c92c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 23:43:39 -0700 Subject: [PATCH 27/53] fix(files): offline read-only fallback when the realtime doc never syncs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the realtime server is unreachable (offline, server down, socket never connects), the collaborative editor would sit blank and read-only forever — content only arrives via provider sync events that never fire. Add a bounded connect-deadline to the Yjs provider: if no first sync lands within CONNECT_DEADLINE_MS, latch fatal and emit a synthetic non-retryable join-error — the exact path a real fatal rejection already uses, which seeds the file's stored content read-only. Latching fatal also stops a late reconnect from syncing server state in and merge-duplicating the locally-seeded content (the documented Yjs 'non-empty doc ignores initial value' gotcha). Deliberately NOT adding durable Yjs snapshot persistence / server-side seeding: TipTap can't run the markdown->Yjs conversion server-side (Collaboration extension errors under jsdom), and a durable binary snapshot would create a dual source of truth with the markdown file (edited by copilot / PUT / download). The client-seeder + bounded re-election is the correct architecture for a markdown-is-truth model; this closes its one real user-facing gap without persistence, a migration, or dual-truth. Timer cleared on first sync, on a real fatal rejection, and on destroy. +2 tests. Gates: sim tsc 0, 496 editor tests, biome. Needs live offline->reconnect verification. --- .../collaboration/file-doc-provider.test.ts | 51 +++++++++++++++++++ .../collaboration/file-doc-provider.ts | 46 +++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index ee9ee61923a..4f2356df729 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -273,4 +273,55 @@ describe('FileDocProvider', () => { doc.getText('default').insert(0, 'y') expect(emittedMessages(emit)).toHaveLength(0) }) + + it('gives up with a non-retryable join-error when the first sync never arrives (offline)', () => { + vi.useFakeTimers() + try { + const { provider, emit, fire } = createProvider(false) // socket never connects + const onError = vi.fn() + provider.on('join-error', onError) + + vi.advanceTimersByTime(12_000) + + // Surfaces the same non-retryable rejection the fatal path uses, so the editor falls back to + // showing the file read-only. + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ code: 'CONNECT_TIMEOUT', retryable: false }) + ) + expect(provider.joinError).toEqual( + expect.objectContaining({ code: 'CONNECT_TIMEOUT', retryable: false }) + ) + // Latched fatal: a later connect must not re-join (which could sync server state in and + // duplicate the locally-seeded content). + emit.mockClear() + fire('connect') + expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + } finally { + vi.useRealTimers() + } + }) + + it('does not fire the offline fallback once the first sync completes', () => { + vi.useFakeTimers() + try { + const { provider, fire } = createProvider(true) + const onError = vi.fn() + provider.on('join-error', onError) + + // Complete the initial sync (server SyncStep2) before the deadline. + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + const remote = new Y.Doc() + remote.getText('default').insert(0, 'hi') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, remote) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + + vi.advanceTimersByTime(12_000) + expect(onError).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index f0d24f9c23a..a82e47a8ede 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -28,6 +28,16 @@ interface FileDocProviderEvents { 'join-error': (error: JoinFileDocError) => void } +/** + * How long to wait for a first document sync before giving up. If the realtime server is + * unreachable (offline, server down, socket never connects) the provider would otherwise never + * sync and the editor would sit blank and read-only forever. On this deadline the provider latches + * fatal and surfaces a non-retryable `join-error` — the exact path a fatal rejection uses — so the + * editor falls back to showing the file's stored content read-only. Generous enough to clear a slow + * connect; a socket that connects at all syncs well within it (SyncStep2 for a fresh doc is cheap). + */ +const CONNECT_DEADLINE_MS = 12_000 + /** * The client half of the collaborative file-document protocol: a Yjs provider * that carries document sync + awareness over the shared, already-authenticated @@ -60,6 +70,8 @@ export class FileDocProvider extends ObservableV2 { /** Set on a non-retryable join rejection (e.g. lost write access) so the * provider stops attempting to (re)join until the owner tears it down. */ private fatal = false + /** Deadline for the first sync; fires the offline fallback if the server is never reached. */ + private connectTimer: ReturnType | null = null constructor( private readonly socket: Socket, @@ -90,6 +102,37 @@ export class FileDocProvider extends ObservableV2 { awareness.on('update', this.handleAwarenessUpdate) if (socket.connected) this.join() + + // Arm the offline fallback: if no first sync arrives before the deadline, give up (below). + this.connectTimer = setTimeout(this.handleConnectDeadline, CONNECT_DEADLINE_MS) + } + + /** + * The first sync never arrived within {@link CONNECT_DEADLINE_MS} — the realtime server is + * unreachable. Latch fatal (so a late reconnect can't sync server state in and merge-duplicate the + * content the editor is about to seed locally) and surface a synthetic non-retryable join-error, so + * the owner falls back to the read-only, non-collaborative view exactly as it does for a real + * fatal rejection. No-op if we already synced, already failed fatally, or were torn down. + */ + private handleConnectDeadline = () => { + this.connectTimer = null + if (this.synced || this.fatal || this.disposed) return + const error: JoinFileDocError = { + fileId: this.fileId, + error: 'Realtime connection timed out', + code: 'CONNECT_TIMEOUT', + retryable: false, + } + this.fatal = true + this.joinError = error + this.emit('join-error', [error]) + } + + private clearConnectTimer() { + if (this.connectTimer !== null) { + clearTimeout(this.connectTimer) + this.connectTimer = null + } } /** Join the room, binding our client id so the server only accepts awareness we own. */ @@ -128,6 +171,7 @@ export class FileDocProvider extends ObservableV2 { if (data.retryable === false) { this.fatal = true this.joinError = data + this.clearConnectTimer() } this.emit('join-error', [data]) } @@ -221,6 +265,7 @@ export class FileDocProvider extends ObservableV2 { private setSynced(synced: boolean) { if (this.synced === synced) return this.synced = synced + if (synced) this.clearConnectTimer() this.emit('synced', [synced]) } @@ -235,6 +280,7 @@ export class FileDocProvider extends ObservableV2 { return } this.disposed = true + this.clearConnectTimer() awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'provider-destroy') From a58f9289b234313ddc1f7c8c050ed5abf7836550 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 23:49:59 -0700 Subject: [PATCH 28/53] fix(realtime): validate workflow join id before generation bump; scope file-doc join rollback to its target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Cursor findings: - join-workflow bumped joinGeneration before validating workflowId (unlike tables / workspace-files, which I'd already fixed). A malformed/empty join could advance the counter and cancel a legitimate in-flight workflow switch. Validate the id first. +test. - The file-doc join catch called cleanupFileDocForSocket unconditionally, which keys off socketToRoomName. During a document SWITCH that fails before rebinding (e.g. a throw in client-id reclaim), that binding still points at the socket's PRIOR, valid document — so the rollback tore down a document the socket was validly in. Only run that cleanup when the binding already points at THIS join's target; otherwise the socket never registered as an owner here and the only leftover is a freshly-created empty room, dropped by destroyRoomIfIdle. Gates: realtime tsc 0, 209 tests, biome, boundaries, prune. --- apps/realtime/src/handlers/file-doc.ts | 11 +++++--- apps/realtime/src/handlers/workflow.test.ts | 28 +++++++++++++++++++++ apps/realtime/src/handlers/workflow.ts | 11 ++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 2acee6b7a19..35e7eefca6e 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -573,10 +573,13 @@ export function setupWorkspaceFileDocHandlers( try { const name = roomName(fileDocRoom(fileId)) socket.leave(name) - cleanupFileDocForSocket(socket.id, io) - // If the failure happened after `getOrCreateRoom` but before the socket - // registered as an owner, `cleanupFileDocForSocket` (which keys off - // `socketToRoomName`) can't drop the freshly-created empty room — do it here. + // Roll back ONLY this join's target room. cleanupFileDocForSocket keys off socketToRoomName, + // which — if the join failed before rebinding to the target (e.g. a switch that threw during + // client-id reclaim) — still points at the socket's PRIOR, valid document. Running it then + // would tear down a document the socket is validly in. So only run it when the binding + // already points at the target; otherwise the socket never registered as an owner of this + // room and the only leftover is a freshly-created empty room, dropped below. + if (socketToRoomName.get(socket.id) === name) cleanupFileDocForSocket(socket.id, io) destroyRoomIfIdle(name) } catch {} emitJoinError(socket, fileId, 'Failed to join file document', 'JOIN_FAILED', true) diff --git a/apps/realtime/src/handlers/workflow.test.ts b/apps/realtime/src/handlers/workflow.test.ts index b6c4ab26190..552db8125c5 100644 --- a/apps/realtime/src/handlers/workflow.test.ts +++ b/apps/realtime/src/handlers/workflow.test.ts @@ -310,6 +310,34 @@ describe('setupWorkflowHandlers', () => { ) }) + it('does not let a malformed join cancel a valid in-flight join', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + const validJoin = handlers['join-workflow']({ workflowId: 'workflow-a', tabSessionId: 'tab-1' }) + // A malformed join arrives mid-flight — it must be rejected WITHOUT advancing the generation, + // so it can't supersede the valid join already in flight. + handlers['join-workflow']({ workflowId: '', tabSessionId: 'tab-1' }) + await validJoin + + expect(socket.emit).toHaveBeenCalledWith( + 'join-workflow-error', + expect.objectContaining({ code: 'INVALID_PAYLOAD' }) + ) + // The valid join still committed — not superseded by the malformed one. + expect(socket.join).toHaveBeenCalledWith('workflow-a') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-a' }, + 'socket-1', + expect.anything() + ) + }) + it('cancels an in-flight join when a leave is enqueued before it commits', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager() diff --git a/apps/realtime/src/handlers/workflow.ts b/apps/realtime/src/handlers/workflow.ts index 91598a762fc..523c3747779 100644 --- a/apps/realtime/src/handlers/workflow.ts +++ b/apps/realtime/src/handlers/workflow.ts @@ -22,6 +22,17 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: let opChain: Promise = Promise.resolve() socket.on('join-workflow', ({ workflowId, tabSessionId }) => { + // Validate the id BEFORE claiming a generation, so a malformed join can't advance joinGeneration + // and cancel a legitimate in-flight switch (matches tables/workspace-files). + if (typeof workflowId !== 'string' || workflowId.length === 0) { + socket.emit('join-workflow-error', { + workflowId: typeof workflowId === 'string' ? workflowId : '', + error: 'Invalid workflow id', + code: 'INVALID_PAYLOAD', + retryable: false, + }) + return + } const joinAttempt = (joinGeneration += 1) opChain = opChain .then(() => runJoin(workflowId, tabSessionId, joinAttempt)) From f9f43c353bc351144435cfc8ea08a22cc000c9b0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 09:54:23 -0700 Subject: [PATCH 29/53] fix(files): drop late sync frames once fatal; file-doc join error suppression + retry-budget reset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final safety-audit findings: - CRITICAL: FileDocProvider.handleMessage had no fatal guard. After the connect deadline latched fatal and the editor fell back to a read-only local seed, a late SyncStep2 (slow server / flaky network / deploy) was still applied — merging server state into the seeded doc (content duplication) and flipping synced=true, which un-gated autosave and would persist the duplicate to the real file. fatal guarded (re)join but not inbound sync. Now handleMessage returns early when fatal. +test (late SyncStep2 after the deadline is ignored, doc stays empty + gated). - file-doc join catch now suppresses the client-facing error when superseded (matches workflow/tables/workspace-files) so a retryable error for an abandoned file can't make a client re-join and cancel the newer one. - table/workspace-files room hooks reset the retry budget on (re)connect so a prior full exhaustion doesn't block retries after a reconnect. - presence-visibility: corrected a stale TTL comment. Gates: tsc (sim/realtime) 0, 209 realtime + collab/hooks suites, biome, boundaries, prune. --- apps/realtime/src/handlers/file-doc.ts | 14 +++++++++- .../realtime/src/rooms/presence-visibility.ts | 7 ++--- .../collaboration/file-doc-provider.test.ts | 27 +++++++++++++++++++ .../collaboration/file-doc-provider.ts | 7 +++++ .../files/hooks/use-workspace-files-room.ts | 11 ++++++-- .../tables/[tableId]/hooks/use-table-room.ts | 11 ++++++-- 6 files changed, 69 insertions(+), 8 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 35e7eefca6e..ec77e7ca879 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -434,6 +434,9 @@ export function setupWorkspaceFileDocHandlers( let currentFileId: string | null = null socket.on(FILE_DOC_EVENTS.JOIN, async ({ fileId, clientId }: JoinFileDocPayload) => { + // Hoisted so the catch can tell whether this join was superseded (a switch to another file) + // before surfacing a retryable error for the abandoned one. + let generation: number | undefined try { const userId = socket.userId const userName = socket.userName @@ -460,7 +463,7 @@ export function setupWorkspaceFileDocHandlers( // Claim this JOIN's generation before the async authorize below, and record the file the // socket now intends to edit so a leave for it can cancel this join if it's still in-flight. - const generation = (joinGeneration.get(socket.id) ?? 0) + 1 + generation = (joinGeneration.get(socket.id) ?? 0) + 1 joinGeneration.set(socket.id, generation) currentFileId = fileId @@ -582,6 +585,15 @@ export function setupWorkspaceFileDocHandlers( if (socketToRoomName.get(socket.id) === name) cleanupFileDocForSocket(socket.id, io) destroyRoomIfIdle(name) } catch {} + // Suppress the client-facing error when this join was already superseded (a switch to another + // file, or a disconnect): the rollback above still ran, but a retryable error naming the + // abandoned file could make a client re-join it and cancel the newer one (matches the sibling + // handlers). + if ( + socket.disconnected || + (generation !== undefined && joinGeneration.get(socket.id) !== generation) + ) + return emitJoinError(socket, fileId, 'Failed to join file document', 'JOIN_FAILED', true) } }) diff --git a/apps/realtime/src/rooms/presence-visibility.ts b/apps/realtime/src/rooms/presence-visibility.ts index 407278850b6..56c22a0e861 100644 --- a/apps/realtime/src/rooms/presence-visibility.ts +++ b/apps/realtime/src/rooms/presence-visibility.ts @@ -4,9 +4,10 @@ import type { IRoomManager, UserPresence } from '@/rooms/types' /** * How stale a not-live presence entry must be before a join-time sweep reclaims - * it. Kept above the 1h socket-key TTL so a normally-idle collaborator is never - * evicted; only genuinely-orphaned entries (e.g. a crashed pod that never fired - * `disconnecting`) are cleared. Matches the workflow join sweep. + * it. The `liveIds` gate (not this threshold) is what protects an active + * collaborator; the threshold only bounds how long a genuinely-orphaned entry + * (e.g. a crashed pod that never fired `disconnecting`) lingers. Matches the + * workflow join sweep. */ const STALE_PRESENCE_THRESHOLD_MS = 75 * 60 * 1000 diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index 4f2356df729..7b36bb2d52d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -324,4 +324,31 @@ describe('FileDocProvider', () => { vi.useRealTimers() } }) + + it('ignores a late SyncStep2 that arrives after the connect deadline (no merge, stays gated)', () => { + vi.useFakeTimers() + try { + const { provider, doc, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + + // Deadline lapses with no first sync → fatal fallback (editor falls back to a read-only seed). + vi.advanceTimersByTime(12_000) + expect(provider.joinError).toEqual(expect.objectContaining({ code: 'CONNECT_TIMEOUT' })) + + // A delayed SyncStep2 finally arrives. Applying it would merge server content into the + // already-seeded doc (duplication) and flip synced→true (un-gating autosave), so it MUST be + // dropped once fatal. + const remote = new Y.Doc() + remote.getText('default').insert(0, 'server content') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, remote) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + expect(provider.synced).toBe(false) + expect(doc.getText('default').toString()).toBe('') + } finally { + vi.useRealTimers() + } + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index a82e47a8ede..6b9c7c0dc25 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -183,6 +183,13 @@ export class FileDocProvider extends ObservableV2 { } private handleMessage = (data: unknown) => { + // Once we've given up (a non-retryable rejection, or the connect deadline lapsed and the editor + // fell back to a read-only local seed), ignore ALL inbound frames. A late SyncStep2 arriving + // after the deadline would otherwise merge the server's state into the already-seeded doc — + // duplicating content — and flip `synced` true, which un-gates autosave and would persist the + // duplicate back to the real file. `fatal` guarding (re)join alone is not enough; it must also + // stop applying sync here. + if (this.fatal) return const bytes = toFileDocBytes(data) if (!bytes) return diff --git a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts index 2a87d993cc6..49ff31dcabb 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts @@ -40,6 +40,13 @@ export function useWorkspaceFilesRoom(workspaceId: string): void { const join = () => socket.emit('join-workspace-files', { workspaceId }) + // A fresh (re)connect gets a fresh retry budget, so a prior full exhaustion doesn't leave the + // socket unable to retry a failed re-join until the next success. + const handleConnect = () => { + retries = 0 + join() + } + const handleJoinSuccess = (data: { workspaceId: string }) => { if (data.workspaceId !== workspaceId) return retries = 0 @@ -68,14 +75,14 @@ export function useWorkspaceFilesRoom(workspaceId: string): void { // Join now if the socket is already connected; `connect` covers (re)connects. if (socket.connected) join() - socket.on('connect', join) + socket.on('connect', handleConnect) socket.on('join-workspace-files-success', handleJoinSuccess) socket.on('join-workspace-files-error', handleJoinError) socket.on('workspace-files-changed', handleChanged) return () => { if (retryTimer) clearTimeout(retryTimer) - socket.off('connect', join) + socket.off('connect', handleConnect) socket.off('join-workspace-files-success', handleJoinSuccess) socket.off('join-workspace-files-error', handleJoinError) socket.off('workspace-files-changed', handleChanged) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts index 57083ff75e6..df950dab585 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts @@ -75,6 +75,13 @@ export function useTableRoom(tableId: string): UseTableRoomResult { socket.emit(TABLE_PRESENCE_EVENTS.JOIN, { tableId, tabSessionId: tabSessionIdRef.current }) } + // A fresh (re)connect gets a fresh retry budget, so a prior full exhaustion doesn't leave the + // socket unable to retry a failed re-join until the next success. + const handleConnect = () => { + retries = 0 + join() + } + const handleJoinSuccess = (data: JoinTableSuccess) => { if (data.tableId !== tableId) return retries = 0 @@ -128,7 +135,7 @@ export function useTableRoom(tableId: string): UseTableRoomResult { // Join now if the socket is already connected; `connect` covers (re)connects. if (socket.connected) join() - socket.on('connect', join) + socket.on('connect', handleConnect) socket.on(TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, handleJoinSuccess) socket.on(TABLE_PRESENCE_EVENTS.JOIN_ERROR, handleJoinError) socket.on(TABLE_PRESENCE_UPDATE_EVENT, handlePresence) @@ -136,7 +143,7 @@ export function useTableRoom(tableId: string): UseTableRoomResult { return () => { if (retryTimer) clearTimeout(retryTimer) - socket.off('connect', join) + socket.off('connect', handleConnect) socket.off(TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, handleJoinSuccess) socket.off(TABLE_PRESENCE_EVENTS.JOIN_ERROR, handleJoinError) socket.off(TABLE_PRESENCE_UPDATE_EVENT, handlePresence) From ba726631ded11c7ea73694cff9728de18aa86350 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 28 Jul 2026 14:29:37 -0700 Subject: [PATCH 30/53] feat(collab-doc): server-authoritative Yjs seeding (#6008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server-authoritative Yjs seeding for collaborative file documents: the realtime relay fetches a Yjs seed built from the file's markdown (via the shared TipTap engine) and applies it once per room, replacing the client-seeder election/handshake entirely. - DOM-free markdown<->Yjs conversion core (markdownToYDoc / yDocToMarkdown / applyMarkdownToYDoc) reusing the client markdown engine for parity by construction - Internal x-api-key seed endpoint + realtime fetch; single attempt bounded under the client readiness deadline, guard-release for join-driven retry, read-only fallback on persistent failure - Client readiness gate = synced && server seed flag; jsdom wired for the Next standalone build (serverExternalPackages + outputFileTracingIncludes) Foundation only — copilot-into-doc + markdown projection + durable persistence are Stage C. --- apps/realtime/src/handlers/file-doc-seed.ts | 45 +++ apps/realtime/src/handlers/file-doc.test.ts | 330 ++++++++++-------- apps/realtime/src/handlers/file-doc.ts | 147 +++----- .../api/internal/file-doc/seed/route.test.ts | 70 ++++ .../app/api/internal/file-doc/seed/route.ts | 39 +++ .../collaboration/file-doc-provider.test.ts | 72 ++-- .../collaboration/file-doc-provider.ts | 109 +++--- .../use-file-doc-collaboration.ts | 2 +- .../rich-markdown-editor/markdown-parse.ts | 13 +- .../rich-markdown-editor.tsx | 29 +- apps/sim/lib/api/contracts/file-doc.ts | 33 ++ apps/sim/lib/collab-doc/README.md | 53 +++ apps/sim/lib/collab-doc/converter.test.ts | 77 ++++ apps/sim/lib/collab-doc/converter.ts | 105 ++++++ apps/sim/lib/collab-doc/index.ts | 6 + apps/sim/lib/collab-doc/seed.test.ts | 74 ++++ apps/sim/lib/collab-doc/seed.ts | 53 +++ .../workspace/workspace-file-manager.ts | 11 +- apps/sim/next.config.ts | 7 + apps/sim/package.json | 3 +- bun.lock | 3 +- packages/realtime-protocol/src/file-doc.ts | 64 +--- scripts/check-api-validation-contracts.ts | 4 +- 23 files changed, 962 insertions(+), 387 deletions(-) create mode 100644 apps/realtime/src/handlers/file-doc-seed.ts create mode 100644 apps/sim/app/api/internal/file-doc/seed/route.test.ts create mode 100644 apps/sim/app/api/internal/file-doc/seed/route.ts create mode 100644 apps/sim/lib/api/contracts/file-doc.ts create mode 100644 apps/sim/lib/collab-doc/README.md create mode 100644 apps/sim/lib/collab-doc/converter.test.ts create mode 100644 apps/sim/lib/collab-doc/converter.ts create mode 100644 apps/sim/lib/collab-doc/index.ts create mode 100644 apps/sim/lib/collab-doc/seed.test.ts create mode 100644 apps/sim/lib/collab-doc/seed.ts diff --git a/apps/realtime/src/handlers/file-doc-seed.ts b/apps/realtime/src/handlers/file-doc-seed.ts new file mode 100644 index 00000000000..6217d0d2e36 --- /dev/null +++ b/apps/realtime/src/handlers/file-doc-seed.ts @@ -0,0 +1,45 @@ +import { env, getBaseUrl } from '@/env' + +/** + * Bound the seed fetch so a slow/unreachable app never wedges a room's first join. Kept comfortably + * BELOW the client's readiness deadline (`READINESS_DEADLINE_MS`, 12s, in `file-doc-provider.ts`), so + * a single attempt either lands or fails within the window the client is willing to wait — otherwise + * the client gives up first and a late success can't reach it. Generous for a real seed: collab is + * gated client-side to ≤256 KB documents (`isRoundTripSafe`), which convert markdown→Yjs in well + * under a second; the 5 MB server cap is only a backstop and is never reached for a collab file. + */ +const SEED_FETCH_TIMEOUT_MS = 8_000 + +/** + * Ask the app to build a server-authoritative seed (markdown → Yjs) for a file's collaborative + * document. Only the app can — it owns the editor extensions + blob access — so the relay delegates + * over the internal endpoint. + * + * Returns the Yjs update to apply, or `null` for a genuinely empty/missing file (an empty document + * is correct). THROWS on a transport failure (non-2xx / network / timeout) so the caller can tell a + * real empty from a failure it should be allowed to retry. + */ +export async function fetchFileDocSeed( + workspaceId: string, + fileId: string +): Promise { + const response = await fetch(`${getBaseUrl()}/api/internal/file-doc/seed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ workspaceId, fileId }), + signal: AbortSignal.timeout(SEED_FETCH_TIMEOUT_MS), + }) + if (!response.ok) { + throw new Error(`Seed fetch failed for file ${fileId}: ${response.status}`) + } + const body = (await response.json()) as { update?: unknown } + const update = body?.update + // A well-formed response is `{ update: base64-string | null }`. Anything else is a contract + // violation, not a "genuinely empty file" — throw so the caller retries rather than silently + // treating a malformed body as empty and stranding the room unseeded. + if (update === null) return null + if (typeof update !== 'string') { + throw new Error(`Seed fetch for file ${fileId} returned a malformed body`) + } + return new Uint8Array(Buffer.from(update, 'base64')) +} diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 1ab6df0fb0a..6d569f4a9e8 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -1,7 +1,12 @@ /** * @vitest-environment node */ -import { FILE_DOC_EVENTS, FILE_DOC_MESSAGE_TYPE } from '@sim/realtime-protocol/file-doc' +import { + FILE_DOC_EVENTS, + FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SEED, +} from '@sim/realtime-protocol/file-doc' +import * as decoding from 'lib0/decoding' import * as encoding from 'lib0/encoding' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import * as awarenessProtocol from 'y-protocols/awareness' @@ -9,14 +14,19 @@ import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' import type { IRoomManager } from '@/rooms' -const { mockAuthorizeRoom } = vi.hoisted(() => ({ +const { mockAuthorizeRoom, mockFetchFileDocSeed } = vi.hoisted(() => ({ mockAuthorizeRoom: vi.fn(), + mockFetchFileDocSeed: vi.fn(), })) vi.mock('@sim/platform-authz/rooms', () => ({ authorizeRoom: mockAuthorizeRoom, })) +vi.mock('@/handlers/file-doc-seed', () => ({ + fetchFileDocSeed: mockFetchFileDocSeed, +})) + import { cleanupFileDocForSocket, setupWorkspaceFileDocHandlers } from '@/handlers/file-doc' type Handler = (payload?: unknown) => Promise | void @@ -95,6 +105,32 @@ function setup(id: string, io: IRoomManager['io'], socketOverrides?: Record { + await Promise.resolve() + await Promise.resolve() +} + +/** + * An encoded Yjs update shaped like the server seed builder's output: some content in the shared + * `default` type plus the {@link FILE_DOC_SEED} flag, so applying it marks the doc seeded. + */ +function encodedSeedUpdate(content: string): Uint8Array { + const doc = new Y.Doc() + doc.getText(FILE_DOC_FIELD).insert(0, content) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + return Y.encodeStateAsUpdate(doc) +} + +/** Apply a server sync reply frame (`[SYNC tag][sync message]`) into a fresh client doc. */ +function applySyncReply(frameBytes: Uint8Array, doc: Y.Doc): void { + const decoder = decoding.createDecoder(frameBytes) + decoding.readVarUint(decoder) // skip the message-type tag + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), doc, null) +} + /** Frame a Yjs message with its type tag, exactly as the client provider would. */ function frame(type: number, write: (encoder: encoding.Encoder) => void): Uint8Array { const encoder = encoding.createEncoder() @@ -128,15 +164,15 @@ function joinSuccessFileId(socket: { emit: ReturnType }) { describe('setupWorkspaceFileDocHandlers', () => { beforeEach(() => { vi.clearAllMocks() - // The seed deadline uses setTimeout; fake it so tests can drive it and so a - // real timer can never fire into a later test. - vi.useFakeTimers() mockAuthorizeRoom.mockResolvedValue({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'write', }) + // Default: the server seed builder returns no content (empty file). Tests that + // exercise seeding override this per-case with an encoded Yjs update. + mockFetchFileDocSeed.mockResolvedValue(null) }) afterEach(() => { @@ -146,8 +182,6 @@ describe('setupWorkspaceFileDocHandlers', () => { // map is cleared and never bleeds a counter into the next test. for (const id of createdSocketIds) cleanupFileDocForSocket(id, io, true) createdSocketIds.clear() - vi.clearAllTimers() - vi.useRealTimers() }) it('rejects join when the socket is not authenticated', async () => { @@ -211,8 +245,9 @@ describe('setupWorkspaceFileDocHandlers', () => { ) }) - it('joins the room, sends sync step 1, and asks the first client to seed', async () => { - const { io, sent } = createIo() + it('joins the room, sends sync step 1, and seeds the document from the server', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) + const { io } = createIo() const { socket, handlers } = setup('socket-1', io) await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) @@ -226,26 +261,142 @@ describe('setupWorkspaceFileDocHandlers', () => { ) expect((syncMessage?.[1] as Uint8Array)[0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) - // The lone joiner is elected to seed the empty document. - const seed = sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) - expect(seed).toEqual({ - target: 'socket-1', - event: FILE_DOC_EVENTS.SEED_REQUEST, - payload: { fileId: 'file-1' }, - }) + // The server seeds authoritatively from the file's stored markdown, keyed by (workspaceId, fileId). + await flushMicrotasks() + expect(mockFetchFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1') + + // The seeded state is served to a client that syncs: request step 2 and decode it. + socket.emit.mockClear() + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + const reply = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# From server') }) - it('asks only one client to seed across concurrent joiners of the same file', async () => { - const { io, sent } = createIo() + it('seeds the document only once from the server across concurrent joiners of the same file', async () => { + // Keep the first seed fetch IN FLIGHT so the doc is still unseeded when the second socket joins: + // that forces the dedup onto `serverSeedStarted` (the in-flight guard) rather than `isDocSeeded`. + let resolveSeed: (v: Uint8Array | null) => void = () => {} + mockFetchFileDocSeed.mockReturnValueOnce(new Promise((resolve) => (resolveSeed = resolve))) + const { io } = createIo() const a = setup('socket-a', io) const b = setup('socket-b', io) await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + // Second join happened with the fetch still pending; only after this does the seed land. + expect(mockFetchFileDocSeed).toHaveBeenCalledTimes(1) + resolveSeed(encodedSeedUpdate('# From server')) + await flushMicrotasks() + expect(mockFetchFileDocSeed).toHaveBeenCalledTimes(1) + }) + + it('marks an empty/absent-file doc seeded so clients still reach readiness', async () => { + // A genuinely absent file yields a null seed (a read error would throw, not return null). The + // relay must still flip `initialContentLoaded` so the client's `synced && seeded` gate opens. + mockFetchFileDocSeed.mockResolvedValue(null) + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) - const seeds = sent.filter((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) - expect(seeds).toHaveLength(1) - expect(seeds[0].target).toBe('socket-a') + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + + socket.emit.mockClear() + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + const reply = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('') + }) + + it('makes one seed attempt and releases the guard on failure so a later join retries', async () => { + mockFetchFileDocSeed + .mockRejectedValueOnce(new Error('transport blip')) + .mockResolvedValueOnce(encodedSeedUpdate('# Recovered')) + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + + // First join: a single attempt that fails — no in-room retry loop. + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + expect(mockFetchFileDocSeed).toHaveBeenCalledTimes(1) + + // The guard was released, so a subsequent join re-attempts and this time the seed lands. + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + await flushMicrotasks() + expect(mockFetchFileDocSeed).toHaveBeenCalledTimes(2) + + socket.emit.mockClear() + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + const reply = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# Recovered') + }) + + it('does not seed a room that was dropped while the seed fetch was in flight', async () => { + let resolveSeed: (v: Uint8Array | null) => void = () => {} + mockFetchFileDocSeed.mockReturnValueOnce(new Promise((resolve) => (resolveSeed = resolve))) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + // The only owner leaves → the room (and its doc) is destroyed while the fetch is still pending. + cleanupFileDocForSocket('socket-1', io, true) + // Resolving now must not touch the destroyed doc or throw (liveness re-check after the await). + resolveSeed(encodedSeedUpdate('# Too late')) + await expect(flushMicrotasks()).resolves.toBeUndefined() + }) + + it('still seeds when content was synced into the doc before the seed returned', async () => { + // Defensive: the guard is `isDocSeeded`, NOT doc-emptiness. In practice a fresh client never + // writes ahead of the seed (@tiptap/y-tiptap suppresses the empty-paragraph placeholder and real + // edits are readiness-gated), but even if some update landed content in the doc before the seed + // fetch resolved, the seed must still apply and set the flag — or the client's + // `synced && initialContentLoaded` gate would never open. + let resolveSeed: (v: Uint8Array | null) => void = () => {} + mockFetchFileDocSeed.mockReturnValueOnce(new Promise((resolve) => (resolveSeed = resolve))) + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + // The client syncs a placeholder update — content in the doc, but no seed flag. + const placeholder = new Y.Doc() + placeholder.getText(FILE_DOC_FIELD).insert(0, 'x') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(placeholder)) + ) + ) + resolveSeed(encodedSeedUpdate('# Seeded')) + await flushMicrotasks() + + socket.emit.mockClear() + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + const reply = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toContain('# Seeded') }) it('relays a document update to the rest of the room, excluding the sender', async () => { @@ -391,33 +542,22 @@ describe('setupWorkspaceFileDocHandlers', () => { ).not.toThrow() }) - it('hands the seeder role to a remaining client when the elected one leaves before seeding', async () => { - const { io, sent } = createIo() - const a = setup('socket-a', io) - const b = setup('socket-b', io) - await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) - await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) - sent.length = 0 - - // The seeder leaves before it ever seeded; b remains. - cleanupFileDocForSocket('socket-a', io) - - const seed = sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) - expect(seed?.target).toBe('socket-b') - }) - - it('drops the document when the last editor leaves, re-seeding a fresh joiner', async () => { - const { io, sent } = createIo() + it('drops the document when the last editor leaves, re-seeding a fresh joiner from the server', async () => { + const { io } = createIo() const a = setup('socket-a', io) await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() cleanupFileDocForSocket('socket-a', io) - sent.length = 0 + // The room was dropped with its last owner: a fresh joiner starts a new document, so the server + // is asked to seed it again (a stale in-memory doc is never reused across an empty gap). + mockFetchFileDocSeed.mockClear() const b = setup('socket-b', io) await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + await flushMicrotasks() - const seed = sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) - expect(seed?.target).toBe('socket-b') + expect(b.socket.join).toHaveBeenCalledWith(ROOM_NAME) + expect(mockFetchFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1') }) it('aborts a join superseded by a newer join during authorization (no cross-binding)', async () => { @@ -451,10 +591,11 @@ describe('setupWorkspaceFileDocHandlers', () => { await pending expect(s.socket.join).not.toHaveBeenCalled() - // No room leaked: a fresh joiner starts a new document and is elected to seed. + // No room leaked: a fresh joiner starts a new document and joins cleanly. const b = setup('socket-b', io) await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) - expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') + expect(b.socket.join).toHaveBeenCalledWith(ROOM_NAME) + expect(joinSuccessFileId(b.socket)).toBe('file-1') }) it('does not abort an in-flight join when a leave for a different file arrives', async () => { @@ -552,27 +693,6 @@ describe('setupWorkspaceFileDocHandlers', () => { expect((reply?.[1] as Uint8Array)[0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) }) - it('does not re-elect a seeder once the document is marked seeded', async () => { - const { io, sent } = createIo() - const a = setup('socket-a', io) - await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) // a elected seeder - // a seeds: set the CRDT initialContentLoaded flag on the server doc. - const seeded = new Y.Doc() - seeded.getMap('config').set('initialContentLoaded', true) - a.handlers[FILE_DOC_EVENTS.MESSAGE]( - frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => - syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(seeded)) - ) - ) - const b = setup('socket-b', io) - await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) - sent.length = 0 - - // The seeder leaves a SEEDED doc → no re-election (no duplicate seed). - cleanupFileDocForSocket('socket-a', io) - expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)).toBeUndefined() - }) - it('leaves the previous document when a socket switches files', async () => { const { io, sent } = createIo() const s = setup('socket-a', io) @@ -582,12 +702,15 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(s.socket.leave).toHaveBeenCalledWith('workspace-file-doc:file-1') expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') - // file-1's room was dropped (socket-a was its only owner): a fresh joiner of - // file-1 starts a new document and is elected to seed. - sent.length = 0 + // file-1's room was dropped (socket-a was its only owner): a fresh joiner of file-1 starts a new + // document, so the server is asked to seed it again. + await flushMicrotasks() + mockFetchFileDocSeed.mockClear() const b = setup('socket-b', io) await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) - expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') + await flushMicrotasks() + expect(b.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-1') + expect(mockFetchFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1') }) it('fully evicts a reclaimed prior socket so it can no longer write to the doc', async () => { @@ -612,22 +735,6 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(sent.some((m) => m.event === FILE_DOC_EVENTS.MESSAGE)).toBe(false) }) - it('re-elects a seeder when the reclaimed socket held the seeder role', async () => { - const { io, sent } = createIo() - const a = setup('socket-a', io) - await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) - // a is the sole owner of an unseeded doc → elected seeder. - expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-a') - sent.length = 0 - - const b = setup('socket-b', io) // same user-1 reconnecting, reusing client id 7 - await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) - - // The reclaim evicts a (the seeder) and releases the role, so the join's election picks b — - // the doc gets seeded instead of waiting out the deadline. - expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') - }) - it('does not drop the current document when a switch is rejected for a foreign client id', async () => { const { io } = createIo() const a = setup('socket-a', io) // user-1 @@ -649,61 +756,6 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(a.socket.join).not.toHaveBeenCalledWith('workspace-file-doc:file-2') }) - it('re-elects a new seeder when the elected one misses the seed deadline', async () => { - const { io, sent } = createIo() - const a = setup('socket-a', io) - const b = setup('socket-b', io) - await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) - await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) - expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-a') - sent.length = 0 - - // socket-a never seeds; the deadline lapses. - vi.advanceTimersByTime(10_000) - - // The remaining un-tried client is asked to seed instead. - expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)?.target).toBe('socket-b') - }) - - it('re-offers seeding to a sole client that missed the deadline, then gives up after a bound', async () => { - const { io, sent } = createIo() - const a = setup('socket-a', io) - await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) - expect(sent.filter((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST).length).toBe(1) - - // The sole client keeps missing the deadline. Without recovery it would be permanently excluded - // (empty document forever); instead it is re-offered a bounded number of rounds. - for (let i = 0; i < 6; i++) vi.advanceTimersByTime(10_000) - - // Initial offer + MAX_SEED_ROUNDS (3) re-offers = 4 total, all to socket-a; then it stops. - const requests = sent.filter((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST) - expect(requests.length).toBe(4) - expect(requests.every((m) => m.target === 'socket-a')).toBe(true) - }) - - it('cancels the seed deadline once the document is seeded', async () => { - const { io, sent } = createIo() - const a = setup('socket-a', io) - const b = setup('socket-b', io) - await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) - await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) - - // socket-a seeds within the deadline. - const seeded = new Y.Doc() - seeded.getMap('config').set('initialContentLoaded', true) - a.handlers[FILE_DOC_EVENTS.MESSAGE]( - frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => - syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(seeded)) - ) - ) - sent.length = 0 - - vi.advanceTimersByTime(10_000) - - // No re-election: the successful seed cancelled the deadline. - expect(sent.find((m) => m.event === FILE_DOC_EVENTS.SEED_REQUEST)).toBeUndefined() - }) - it('broadcasts a server-authenticated presence roster on join, one entry per session', async () => { const { io, sent } = createIo() const a = setup('socket-a', io, { userId: 'user-a', userName: 'Ada', userImage: 'ada.png' }) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index ec77e7ca879..0dd8829c87f 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -36,30 +36,13 @@ import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' import { resolveAvatarUrl } from '@/handlers/avatar' +import { fetchFileDocSeed } from '@/handlers/file-doc-seed' import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' const logger = createLogger('FileDocHandlers') -/** - * How long an elected seeder has to import the initial content before the server - * re-elects another client. Seeding is a local, synchronous editor write, so this - * only trips when a seeder never completes it (a bug, or a client withholding the - * seed) — it keeps the document from staying empty for the rest of the room. - */ -const SEED_DEADLINE_MS = 10_000 - -/** - * Max times the room re-offers seeding to its owners after every one of them has - * missed a deadline. Without this, a sole client whose seed *fetch fails* (not just - * slow) is added to `triedSeeders`, re-election finds no un-tried owner, and the - * document is left permanently empty until that client reloads. Each exhausted round - * clears `triedSeeders` and re-offers, giving a transient failure a bounded number of - * retries (~MAX × deadline) before the room genuinely gives up. - */ -const MAX_SEED_ROUNDS = 3 - /** A socket's presence ownership within a room. */ interface FileDocOwner { /** @@ -78,23 +61,15 @@ interface FileDocOwner { } interface FileDocRoom { - /** The `workspace_files.id` this room edits (for seed-request payloads). */ + /** The `workspace_files.id` this room edits. */ fileId: string doc: Y.Doc awareness: awarenessProtocol.Awareness /** socketId → its presence ownership. */ owners: Map - /** The socket currently elected to seed initial content, or `null`. */ - seederSocketId: string | null - /** Deadline timer for the current seeder to complete the seed, or `null`. */ - seedTimer: ReturnType | null - /** Sockets that were elected but failed to seed within the deadline; skipped - * on re-election so a single stuck/withholding client can't block the room. */ - triedSeeders: Set - /** Count of full re-offer rounds spent after every owner missed a deadline; - * bounds the retry loop (see {@link MAX_SEED_ROUNDS}) so a permanently-broken - * seed doesn't reset forever. */ - seedRounds: number + /** True once the server-side seed fetch has started, so concurrent joins don't each fetch. + * Reset on a fetch FAILURE so a later join can retry (a genuinely empty file stays empty). */ + serverSeedStarted: boolean } /** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */ @@ -183,73 +158,51 @@ function awarenessUpdateClientIds(update: Uint8Array): number[] { return ids } -function clearSeedTimer(room: FileDocRoom) { - if (room.seedTimer !== null) { - clearTimeout(room.seedTimer) - room.seedTimer = null - } -} - /** - * Drop a room's document + awareness (and its seed timer) once it has no owners, - * so an idle file holds no memory. A later joiner re-creates and re-seeds it. + * Drop a room's document + awareness once it has no owners, so an idle file holds no memory. + * A later joiner re-creates and re-seeds it from the file's current markdown. */ function destroyRoomIfIdle(name: string) { const room = fileDocRooms.get(name) if (!room || room.owners.size > 0) return - clearSeedTimer(room) room.awareness.destroy() room.doc.destroy() fileDocRooms.delete(name) } /** - * Elect a client to seed an unseeded document and ask it to import the stored - * markdown, if one is needed and not already assigned. Called after a join (to - * elect the newcomer), after the elected seeder leaves (to hand the role to a - * remaining client), and after a seed deadline lapses (to pass over a client that - * never seeded). Skips clients that already failed to seed, and arms a deadline so - * a stuck or withholding seeder can't leave the document empty for the whole room. - * A no-op once the document is seeded, a seeder is already assigned, or no - * un-tried client remains. + * Seed a room's document server-side, once, on the first join: ask the app to build the seed (the + * file's current markdown → Yjs, through the exact editor engine) and apply it, which relays the + * content to every connected client via `doc.on('update')`. No client is elected to import content. + * + * `isDocSeeded` is the sufficient guard: content only ever reaches the doc alongside the seed flag + * (this seed, or a client's offline fallback), so an unseeded doc is genuinely empty and safe to seed. + * A genuinely empty/missing file returns `null` (a read error throws instead), so still set the flag — + * an empty doc must reach readiness, not wait forever. After the fetch, re-check the room is still + * live and unseeded (an owner may have left, or a client seeded it, while the fetch was in flight). + * + * Recovery on failure is deliberately simple — no in-room retry loop: a single attempt bounded by a + * timeout shorter than the client's readiness deadline, then release the guard. A transient failure + * is re-attempted by the next join/reconnect; a persistent one lets the connected client's readiness + * deadline lapse into its read-only fallback. (An in-room backoff retry can outlast that client + * deadline, so it would keep trying a doc the client has already given up on — worse, not better.) */ -function electSeederIfNeeded(io: Server, room: FileDocRoom) { - if (room.seederSocketId !== null || isDocSeeded(room.doc)) return - - let elected: string | null = null - for (const socketId of room.owners.keys()) { - if (!room.triedSeeders.has(socketId)) { - elected = socketId - break - } +async function ensureServerSeed( + name: string, + room: FileDocRoom, + workspaceId: string +): Promise { + if (room.serverSeedStarted || isDocSeeded(room.doc)) return + room.serverSeedStarted = true + try { + const update = await fetchFileDocSeed(workspaceId, room.fileId) + if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return + if (update) Y.applyUpdate(room.doc, update) + else room.doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + } catch (error) { + logger.warn(`Server seed failed for file ${room.fileId} (workspace ${workspaceId})`, error) + room.serverSeedStarted = false } - if (elected === null) return - - room.seederSocketId = elected - io.to(elected).emit(FILE_DOC_EVENTS.SEED_REQUEST, { fileId: room.fileId }) - - clearSeedTimer(room) - room.seedTimer = setTimeout(() => { - room.seedTimer = null - // Only act if this election is still the pending one and it never seeded. - if (room.seederSocketId !== elected || isDocSeeded(room.doc)) return - room.triedSeeders.add(elected) - room.seederSocketId = null - electSeederIfNeeded(io, room) - // If that left no seeder (every current owner has now been tried) while the doc is still - // unseeded and owners remain, re-offer the whole room a bounded number of times — otherwise a - // sole client whose seed fetch failed leaves the document permanently empty. - if ( - room.seederSocketId === null && - !isDocSeeded(room.doc) && - room.owners.size > 0 && - room.seedRounds < MAX_SEED_ROUNDS - ) { - room.seedRounds += 1 - room.triedSeeders.clear() - electSeederIfNeeded(io, room) - } - }, SEED_DEADLINE_MS) } /** @@ -272,16 +225,11 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { doc, awareness, owners: new Map(), - seederSocketId: null, - seedTimer: null, - triedSeeders: new Set(), - seedRounds: 0, + serverSeedStarted: false, } fileDocRooms.set(name, room) doc.on('update', (update: Uint8Array, origin: unknown) => { - // Once the document is seeded, the seed deadline is moot — cancel it. - if (room.seedTimer !== null && isDocSeeded(doc)) clearSeedTimer(room) const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) syncProtocol.writeUpdate(encoder, update) @@ -370,8 +318,8 @@ function handleMessage(socket: AuthenticatedSocket, data: unknown) { /** * Remove a socket from its file-doc room: clear its awareness state (so its caret - * disappears for everyone else), hand off the seeder role if it left before - * seeding, and drop the room's document when the last collaborator leaves. + * disappears for everyone else) and drop the room's document when the last + * collaborator leaves. * Exported for the disconnect handler; safe to call for a socket in no room. */ export function cleanupFileDocForSocket(socketId: string, io: Server, endOfLife = false): void { @@ -401,13 +349,6 @@ export function cleanupFileDocForSocket(socketId: string, io: Server, endOfLife broadcastFileDocPresence(io, name, room) } - // Hand off the seeder role: if the elected seeder left before it seeded, elect - // a remaining client so the document doesn't stay permanently empty. - if (room.seederSocketId === socketId) { - room.seederSocketId = null - electSeederIfNeeded(io, room) - } - destroyRoomIfIdle(name) } @@ -519,10 +460,6 @@ export function setupWorkspaceFileDocHandlers( awarenessProtocol.removeAwarenessStates(entry.awareness, [owner.clientId], null) socketToRoomName.delete(otherSid) io.in(otherSid).socketsLeave(name) - // If the evicted socket held the seeder role, release it so the election at the end of - // this join re-elects (electSeederIfNeeded no-ops while seederSocketId is set) — otherwise - // an unseeded document would stay empty until the seed deadline expires. - if (entry.seederSocketId === otherSid) entry.seederSocketId = null } // Only now that the rebind is guaranteed to succeed, leave a previously-joined document if @@ -568,7 +505,9 @@ export function setupWorkspaceFileDocHandlers( socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(awarenessEncoder)) } - electSeederIfNeeded(io, entry) + // Seed the document server-side (once). Fire-and-forget: the join completes immediately and + // the seed relays to this socket via `doc.on('update')` the moment it lands. + if (authorized.workspaceId) void ensureServerSeed(name, entry, authorized.workspaceId) logger.info(`User ${userId} joined file-doc room ${fileId}`) } catch (error) { diff --git a/apps/sim/app/api/internal/file-doc/seed/route.test.ts b/apps/sim/app/api/internal/file-doc/seed/route.test.ts new file mode 100644 index 00000000000..f64504f9ded --- /dev/null +++ b/apps/sim/app/api/internal/file-doc/seed/route.test.ts @@ -0,0 +1,70 @@ +/** + * @vitest-environment node + */ +import { createMockRequest } from '@sim/testing' +import { NextResponse } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCheckInternalApiKey, mockBuildFileDocSeed } = vi.hoisted(() => ({ + mockCheckInternalApiKey: vi.fn(), + mockBuildFileDocSeed: vi.fn(), +})) + +vi.mock('@/lib/copilot/request/http', () => ({ + checkInternalApiKey: mockCheckInternalApiKey, + createUnauthorizedResponse: () => NextResponse.json({ error: 'Unauthorized' }, { status: 401 }), +})) + +vi.mock('@/lib/collab-doc/seed', () => ({ + buildFileDocSeed: mockBuildFileDocSeed, +})) + +import { POST } from './route' + +function seedRequest(body: unknown) { + return createMockRequest('POST', body, { 'x-api-key': 'internal' }) +} + +describe('POST /api/internal/file-doc/seed', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckInternalApiKey.mockReturnValue({ success: true }) + }) + + // Regression guard for the auth-helper choice: the realtime relay authenticates with + // `x-api-key: INTERNAL_API_SECRET`, so this route MUST gate on `checkInternalApiKey`. Wiring the + // Bearer-JWT-only `checkInternalAuth` (which forbids `x-api-key`) 401s every real seed fetch. + it('401s when the internal api key is rejected, without building a seed', async () => { + mockCheckInternalApiKey.mockReturnValue({ success: false }) + const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' })) + expect(res.status).toBe(401) + expect(mockBuildFileDocSeed).not.toHaveBeenCalled() + }) + + it('returns the seed as base64 for an authorized request', async () => { + mockBuildFileDocSeed.mockResolvedValue({ update: new Uint8Array([1, 2, 3, 4]) }) + const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' })) + expect(res.status).toBe(200) + expect((await res.json()).update).toBe(Buffer.from([1, 2, 3, 4]).toString('base64')) + expect(mockBuildFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1') + }) + + it('returns update:null for a genuinely absent file', async () => { + mockBuildFileDocSeed.mockResolvedValue(null) + const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'missing' })) + expect(res.status).toBe(200) + expect((await res.json()).update).toBeNull() + }) + + it('400s on a body missing required fields (contract validation, after auth)', async () => { + const res = await POST(seedRequest({ workspaceId: 'ws-1' })) + expect(res.status).toBe(400) + expect(mockBuildFileDocSeed).not.toHaveBeenCalled() + }) + + it('500s when the seed build throws (a read error the relay should retry)', async () => { + mockBuildFileDocSeed.mockRejectedValue(new Error('db down')) + const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' })) + expect(res.status).toBe(500) + }) +}) diff --git a/apps/sim/app/api/internal/file-doc/seed/route.ts b/apps/sim/app/api/internal/file-doc/seed/route.ts new file mode 100644 index 00000000000..b6cbc83bfd0 --- /dev/null +++ b/apps/sim/app/api/internal/file-doc/seed/route.ts @@ -0,0 +1,39 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { buildFileDocSeedContract } from '@/lib/api/contracts/file-doc' +import { parseRequest } from '@/lib/api/server' +import { buildFileDocSeed } from '@/lib/collab-doc/seed' +import { checkInternalApiKey, createUnauthorizedResponse } from '@/lib/copilot/request/http' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + +const logger = createLogger('FileDocSeedAPI') + +/** + * POST /api/internal/file-doc/seed — build a server-authoritative collaborative-document seed + * (markdown → Yjs) for the realtime relay to apply on room creation. Internal only: gated on the + * shared `x-api-key: INTERNAL_API_SECRET` secret, matching the header the realtime relay sends + * (`apps/realtime/src/handlers/file-doc-seed.ts`) and the realtime server's own inbound validator. + */ +export const POST = withRouteHandler(async (request: NextRequest) => { + const auth = checkInternalApiKey(request) + if (!auth.success) return createUnauthorizedResponse() + + const parsed = await parseRequest(buildFileDocSeedContract, request, {}) + if (!parsed.success) return parsed.response + const { workspaceId, fileId } = parsed.data.body + + try { + const seed = await buildFileDocSeed(workspaceId, fileId) + return NextResponse.json({ + update: seed ? Buffer.from(seed.update).toString('base64') : null, + }) + } catch (error) { + logger.error('Failed to build file-doc seed', { workspaceId, fileId, error }) + return NextResponse.json( + { error: getErrorMessage(error, 'Failed to build seed') }, + { status: 500 } + ) + } +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index 7b36bb2d52d..29e31b53301 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -1,7 +1,11 @@ /** * @vitest-environment node */ -import { FILE_DOC_EVENTS, FILE_DOC_MESSAGE_TYPE } from '@sim/realtime-protocol/file-doc' +import { + FILE_DOC_EVENTS, + FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SEED, +} from '@sim/realtime-protocol/file-doc' import * as encoding from 'lib0/encoding' import type { Socket } from 'socket.io-client' import { describe, expect, it, vi } from 'vitest' @@ -83,30 +87,13 @@ describe('FileDocProvider', () => { expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) }) - it('emits seed-request and latches shouldSeed when the server elects it', () => { - const { provider, fire } = createProvider(true) - const seed = vi.fn() - provider.on('seed-request', seed) - expect(provider.shouldSeed).toBe(false) - - fire(FILE_DOC_EVENTS.SEED_REQUEST, { fileId: 'file-1' }) - - expect(seed).toHaveBeenCalledTimes(1) - // Latched so a consumer subscribing after the event can still detect election. - expect(provider.shouldSeed).toBe(true) - }) - - it('ignores acks and seed requests for a different file', () => { - const { provider, emit, fire } = createProvider(true) - const seed = vi.fn() - provider.on('seed-request', seed) + it('ignores a join ack for a different file', () => { + const { emit, fire } = createProvider(true) emit.mockClear() fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'other-file' }) - fire(FILE_DOC_EVENTS.SEED_REQUEST, { fileId: 'other-file' }) - expect(seed).not.toHaveBeenCalled() - expect(provider.shouldSeed).toBe(false) + // No sync/awareness exchange starts for a file this provider does not own. expect(emittedMessages(emit)).toHaveLength(0) }) @@ -286,10 +273,10 @@ describe('FileDocProvider', () => { // Surfaces the same non-retryable rejection the fatal path uses, so the editor falls back to // showing the file read-only. expect(onError).toHaveBeenCalledWith( - expect.objectContaining({ code: 'CONNECT_TIMEOUT', retryable: false }) + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) ) expect(provider.joinError).toEqual( - expect.objectContaining({ code: 'CONNECT_TIMEOUT', retryable: false }) + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) ) // Latched fatal: a later connect must not re-join (which could sync server state in and // duplicate the locally-seeded content). @@ -301,17 +288,18 @@ describe('FileDocProvider', () => { } }) - it('does not fire the offline fallback once the first sync completes', () => { + it('does not fire the fallback once the doc is synced AND seeded', () => { vi.useFakeTimers() try { const { provider, fire } = createProvider(true) const onError = vi.fn() provider.on('join-error', onError) - // Complete the initial sync (server SyncStep2) before the deadline. + // The initial sync brings BOTH content and the server seed flag before the deadline. fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) const remote = new Y.Doc() remote.getText('default').insert(0, 'hi') + remote.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) syncProtocol.writeSyncStep2(encoder, remote) @@ -325,7 +313,37 @@ describe('FileDocProvider', () => { } }) - it('ignores a late SyncStep2 that arrives after the connect deadline (no merge, stays gated)', () => { + it('fires the fallback when the doc synced but the server seed never landed', () => { + vi.useFakeTimers() + try { + const { provider, fire } = createProvider(true) + const onError = vi.fn() + provider.on('join-error', onError) + + // The socket syncs an empty doc, but the server-side seed never arrives (its build persistently + // failed) — `synced` is true yet `initialContentLoaded` is never set. + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + const remote = new Y.Doc() + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, remote) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + + vi.advanceTimersByTime(12_000) + + // The readiness deadline still fires → the editor falls back to the stored content read-only, + // and `synced` is dropped so the `synced && seeded` gate stays closed (read-only, not editable). + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + ) + expect(provider.synced).toBe(false) + } finally { + vi.useRealTimers() + } + }) + + it('ignores a late SyncStep2 that arrives after the readiness deadline (no merge, stays gated)', () => { vi.useFakeTimers() try { const { provider, doc, fire } = createProvider(true) @@ -333,7 +351,7 @@ describe('FileDocProvider', () => { // Deadline lapses with no first sync → fatal fallback (editor falls back to a read-only seed). vi.advanceTimersByTime(12_000) - expect(provider.joinError).toEqual(expect.objectContaining({ code: 'CONNECT_TIMEOUT' })) + expect(provider.joinError).toEqual(expect.objectContaining({ code: 'READINESS_TIMEOUT' })) // A delayed SyncStep2 finally arrives. Applying it would merge server content into the // already-seeded doc (duplication) and flip synced→true (un-gating autosave), so it MUST be diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index 6b9c7c0dc25..eda4308ed56 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -1,9 +1,9 @@ import { FILE_DOC_EVENTS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SEED, type JoinFileDocError, type JoinFileDocSuccess, - type SeedRequestPayload, toFileDocBytes, } from '@sim/realtime-protocol/file-doc' import * as decoding from 'lib0/decoding' @@ -17,26 +17,26 @@ import type * as Y from 'yjs' /** * Events emitted by {@link FileDocProvider}. * - `synced`: the first full document sync with the server completed. - * - `seed-request`: the server elected this client to seed the document's initial - * content from the file's stored markdown (the editor does the import, guarded - * by the CRDT `initialContentLoaded` flag). * - `join-error`: the server rejected the join (e.g. lost write access). */ interface FileDocProviderEvents { synced: (synced: boolean) => void - 'seed-request': () => void 'join-error': (error: JoinFileDocError) => void } /** - * How long to wait for a first document sync before giving up. If the realtime server is - * unreachable (offline, server down, socket never connects) the provider would otherwise never - * sync and the editor would sit blank and read-only forever. On this deadline the provider latches - * fatal and surfaces a non-retryable `join-error` — the exact path a fatal rejection uses — so the - * editor falls back to showing the file's stored content read-only. Generous enough to clear a slow - * connect; a socket that connects at all syncs well within it (SyncStep2 for a fresh doc is cheap). + * How long to wait to reach a USABLE editor — connected, synced, AND seeded (`initialContentLoaded` + * set by the server seed) — before giving up. It guards two failure modes with one timer: + * - the realtime server is unreachable, so the first sync never arrives; and + * - the socket syncs an empty doc but the server-side seed never lands (its build persistently fails + * / exhausts its retries), which `synced` alone would wrongly treat as "connected, all good". + * + * On the deadline the provider latches fatal and surfaces a non-retryable `join-error` — the exact + * path a fatal rejection uses — so the editor falls back to showing the file's stored content + * read-only instead of a permanently blank pane. Generous enough to clear a slow connect + seed + * round-trip; a healthy cold open reaches readiness well within it. */ -const CONNECT_DEADLINE_MS = 12_000 +const READINESS_DEADLINE_MS = 12_000 /** * The client half of the collaborative file-document protocol: a Yjs provider @@ -54,14 +54,8 @@ const CONNECT_DEADLINE_MS = 12_000 export class FileDocProvider extends ObservableV2 { synced = false /** - * Latched `true` when the server elects this client to seed the document. The - * `seed-request` event is transient and can fire before a consumer subscribes, - * so consumers read this flag on subscription rather than relying on the event. - */ - shouldSeed = false - /** - * The latched non-retryable join rejection, or `null`. Like {@link shouldSeed}, - * the `join-error` event is transient and can fire before a consumer subscribes, + * The latched non-retryable join rejection, or `null`. The `join-error` event is + * transient and can fire before a consumer subscribes, * so consumers read this on subscription to detect a fatal failure they missed. */ joinError: JoinFileDocError | null = null @@ -70,8 +64,8 @@ export class FileDocProvider extends ObservableV2 { /** Set on a non-retryable join rejection (e.g. lost write access) so the * provider stops attempting to (re)join until the owner tears it down. */ private fatal = false - /** Deadline for the first sync; fires the offline fallback if the server is never reached. */ - private connectTimer: ReturnType | null = null + /** Deadline for reaching readiness (synced + seeded); fires the fallback if it is never reached. */ + private readinessTimer: ReturnType | null = null constructor( private readonly socket: Socket, @@ -96,42 +90,58 @@ export class FileDocProvider extends ObservableV2 { socket.on(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) socket.on(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) socket.on(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) - socket.on(FILE_DOC_EVENTS.SEED_REQUEST, this.handleSeedRequest) socket.on('connect', this.handleConnect) doc.on('update', this.handleDocUpdate) awareness.on('update', this.handleAwarenessUpdate) + // Watch the seed flag so reaching "seeded" (server seed applied) can clear the readiness deadline. + doc.getMap(FILE_DOC_SEED.configMap).observe(this.handleConfigChange) if (socket.connected) this.join() - // Arm the offline fallback: if no first sync arrives before the deadline, give up (below). - this.connectTimer = setTimeout(this.handleConnectDeadline, CONNECT_DEADLINE_MS) + // Arm the fallback: if we don't reach readiness (synced + seeded) before the deadline, give up. + this.readinessTimer = setTimeout(this.handleReadinessDeadline, READINESS_DEADLINE_MS) + } + + /** Whether the server seed has recorded the initial content on the doc. */ + private isSeeded(): boolean { + return this.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag) === true + } + + /** Clear the readiness deadline once the editor is usable (synced AND seeded). */ + private handleConfigChange = () => { + if (this.synced && this.isSeeded()) this.clearReadinessTimer() } /** - * The first sync never arrived within {@link CONNECT_DEADLINE_MS} — the realtime server is - * unreachable. Latch fatal (so a late reconnect can't sync server state in and merge-duplicate the - * content the editor is about to seed locally) and surface a synthetic non-retryable join-error, so - * the owner falls back to the read-only, non-collaborative view exactly as it does for a real - * fatal rejection. No-op if we already synced, already failed fatally, or were torn down. + * Readiness was never reached within {@link READINESS_DEADLINE_MS} — either the realtime server is + * unreachable (never synced) or it synced but the server-side seed never landed (synced yet + * unseeded). Reset `synced` (so the editor gates read-only), latch fatal (so a late reconnect or + * seed can't sync server state in and merge-duplicate the content the editor is about to render + * locally), and surface a synthetic non-retryable join-error — the exact path a fatal rejection + * uses — so the owner falls back to the read-only view of the file's stored content instead of a + * blank pane. No-op if we already reached readiness, already failed fatally, or were torn down. */ - private handleConnectDeadline = () => { - this.connectTimer = null - if (this.synced || this.fatal || this.disposed) return + private handleReadinessDeadline = () => { + this.readinessTimer = null + if ((this.synced && this.isSeeded()) || this.fatal || this.disposed) return const error: JoinFileDocError = { fileId: this.fileId, - error: 'Realtime connection timed out', - code: 'CONNECT_TIMEOUT', + error: 'Realtime document was not ready in time', + code: 'READINESS_TIMEOUT', retryable: false, } this.fatal = true this.joinError = error + // Drop `synced` so the editor's `synced && seeded` gate stays closed → the fallback renders the + // stored content read-only rather than becoming editable on a doc the server never seeded. + this.setSynced(false) this.emit('join-error', [error]) } - private clearConnectTimer() { - if (this.connectTimer !== null) { - clearTimeout(this.connectTimer) - this.connectTimer = null + private clearReadinessTimer() { + if (this.readinessTimer !== null) { + clearTimeout(this.readinessTimer) + this.readinessTimer = null } } @@ -171,17 +181,11 @@ export class FileDocProvider extends ObservableV2 { if (data.retryable === false) { this.fatal = true this.joinError = data - this.clearConnectTimer() + this.clearReadinessTimer() } this.emit('join-error', [data]) } - private handleSeedRequest = (data: SeedRequestPayload) => { - if (data.fileId !== this.fileId) return - this.shouldSeed = true - this.emit('seed-request', []) - } - private handleMessage = (data: unknown) => { // Once we've given up (a non-retryable rejection, or the connect deadline lapsed and the editor // fell back to a read-only local seed), ignore ALL inbound frames. A late SyncStep2 arriving @@ -221,6 +225,11 @@ export class FileDocProvider extends ObservableV2 { } private handleDocUpdate = (update: Uint8Array, origin: unknown) => { + // Once fatal (a non-retryable rejection, or the readiness deadline lapsed), the editor may render + // the stored content into the doc locally as its read-only fallback. Never relay those local + // writes — the server never seeded this doc, so echoing them would push unseeded content to peers + // (and each fallen-back client would do so, union-duplicating). A fatal client is fully local. + if (this.fatal) return // Updates we applied from the server carry `this` as origin — don't echo them. if (origin === this) return const encoder = encoding.createEncoder() @@ -272,7 +281,9 @@ export class FileDocProvider extends ObservableV2 { private setSynced(synced: boolean) { if (this.synced === synced) return this.synced = synced - if (synced) this.clearConnectTimer() + // Readiness needs synced AND seeded; only clear the deadline when both hold (the seed may have + // arrived first, or may still be pending — `handleConfigChange` clears it if seeded arrives later). + if (synced && this.isSeeded()) this.clearReadinessTimer() this.emit('synced', [synced]) } @@ -287,7 +298,7 @@ export class FileDocProvider extends ObservableV2 { return } this.disposed = true - this.clearConnectTimer() + this.clearReadinessTimer() awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'provider-destroy') @@ -295,9 +306,9 @@ export class FileDocProvider extends ObservableV2 { this.socket.off(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) this.socket.off(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) this.socket.off(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) - this.socket.off(FILE_DOC_EVENTS.SEED_REQUEST, this.handleSeedRequest) this.socket.off('connect', this.handleConnect) this.doc.off('update', this.handleDocUpdate) + this.doc.getMap(FILE_DOC_SEED.configMap).unobserve(this.handleConfigChange) this.awareness.off('update', this.handleAwarenessUpdate) super.destroy() diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts index 475ea92fb2e..5afd9470a21 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts @@ -19,7 +19,7 @@ export interface FileDocCollaboration { /** * The realtime provider, or `null` until the socket is available. `doc` and * `awareness` exist before it connects, so the editor can bind immediately; the - * provider is consumed for seeding events (`synced` / `seed-request`). + * provider is consumed for its readiness signal (`synced`) and fatal `join-error`. */ provider: FileDocProvider | null /** diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts index c38fac63008..d8255eef7e6 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts @@ -193,8 +193,19 @@ function stripTrailingEmptyParagraphs(doc: JSONContent): JSONContent { * normalization the live editor applies, keeping the output identical to `editor.getMarkdown()`. */ export function serializeMarkdownBody(body: string): string { + return serializeDocToMarkdown(parseMarkdownToDoc(body)) +} + +/** + * Serialize a ProseMirror document (as TipTap {@link JSONContent}) to the editor's canonical + * markdown. Loaded via `setContent` so it passes through the same schema normalization the live + * editor applies — output identical to `editor.getMarkdown()`. The server-side collab-doc converter + * uses this to project a Yjs doc back to markdown through the exact client engine (parity by + * construction), so it must stay the single serialize path (do not inline `getMarkdown` elsewhere). + */ +export function serializeDocToMarkdown(doc: JSONContent): string { const editor = parserEditor() - editor.commands.setContent(parseMarkdownToDoc(body), { contentType: 'json' }) + editor.commands.setContent(doc, { contentType: 'json' }) return editor.getMarkdown() } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 833da70e7cb..5717ed80c6a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -284,7 +284,7 @@ export function LoadedRichMarkdownEditor({ /** * Initial editor content. When collaborating, the Y.Doc is the source of truth — - * start empty and let the seed handshake fill it (below); otherwise seed from the + * start empty and let the server-seeded Yjs sync fill it (below); otherwise seed from the * parsed markdown (chunked parse is linear vs the editor's ~O(n²) whole-body parse). */ const [initialContent] = useState(() => @@ -619,9 +619,9 @@ export function LoadedRichMarkdownEditor({ /** * The collaborative document lifecycle. In one effect because the three concerns * are one state machine keyed off the same provider events: - * - **seed** the doc from the loaded markdown when this client is elected and - * synced (content + `initialContentLoaded` flag in ONE Yjs transaction, so a - * re-election can never duplicate content — the relay's exactly-once contract); + * - **observe** readiness: the server seeds the doc authoritatively (content + + * `initialContentLoaded` flag in ONE Yjs update), so the client only watches for + * synced AND seeded — it never imports content itself on the happy path; * - **gate** the parent's autosave until the doc is synced AND seeded, so an * empty/still-syncing doc can never overwrite the real file's markdown mirror; * - **fall back** on a fatal join: seed the loaded content so it is SHOWN, but @@ -632,8 +632,9 @@ export function LoadedRichMarkdownEditor({ * * `ready` (synced+seeded) gates BOTH the editor's editability (a user must never * type into an empty/unsynced doc) and the parent's autosave. Non-collaborative - * documents are never gated. `provider.shouldSeed` / `provider.joinError` are - * latched, so events that fired before this subscription are not missed. + * documents are never gated. The server seeds the doc authoritatively (content + the + * seed flag arrive together), so the client only observes readiness. `provider.joinError` + * is latched, so a fatal rejection that fired before this subscription is not missed. */ useEffect(() => { const setReady = (ready: boolean) => { @@ -667,25 +668,23 @@ export function LoadedRichMarkdownEditor({ return } + // The server seeds the document (content + the `initialContentLoaded` flag), so readiness is + // simply "synced AND seeded" — no client-side seed import on the happy path. `seedFromLoaded` + // remains only for the offline fallback below (a non-retryable join error / readiness timeout), + // where it locally renders the file read-only since the server can't be reached. const report = () => setReady(provider.synced && config.get('initialContentLoaded') === true) - const onProgress = () => { - if (provider.shouldSeed && provider.synced) seedFromLoaded() - report() - } const onJoinError = (error: JoinFileDocError) => { if (error.retryable === false) seedFromLoaded() } - provider.on('seed-request', onProgress) - provider.on('synced', onProgress) + provider.on('synced', report) provider.on('join-error', onJoinError) config.observe(report) - onProgress() + report() if (provider.joinError) onJoinError(provider.joinError) return () => { - provider.off('seed-request', onProgress) - provider.off('synced', onProgress) + provider.off('synced', report) provider.off('join-error', onJoinError) config.unobserve(report) // Report NOT ready on teardown — the safe direction. If this effect ever re-runs while mounted diff --git a/apps/sim/lib/api/contracts/file-doc.ts b/apps/sim/lib/api/contracts/file-doc.ts new file mode 100644 index 00000000000..27337936f90 --- /dev/null +++ b/apps/sim/lib/api/contracts/file-doc.ts @@ -0,0 +1,33 @@ +import { z } from 'zod' +import { defineRouteContract } from '@/lib/api/contracts/types' + +export const buildFileDocSeedBodySchema = z.object({ + workspaceId: z.string().min(1, 'workspaceId is required'), + fileId: z.string().min(1, 'fileId is required'), +}) +export type BuildFileDocSeedBody = z.input + +export const buildFileDocSeedResponseSchema = z.object({ + /** + * The base64-encoded Yjs update that seeds the file's collaborative document, or `null` when the + * file is missing/deleted (the caller treats that as an empty document). + */ + update: z.string().nullable(), +}) +export type BuildFileDocSeedResponse = z.output + +/** + * Internal, `x-api-key`-gated: the realtime relay asks the app to build a server-authoritative seed + * for a file's collaborative document (markdown → Yjs, through the exact client engine). Only the app + * can do this — it owns the editor extensions + blob access — so realtime delegates instead of + * embedding a second markdown/ProseMirror stack. + */ +export const buildFileDocSeedContract = defineRouteContract({ + method: 'POST', + path: '/api/internal/file-doc/seed', + body: buildFileDocSeedBodySchema, + response: { + mode: 'json', + schema: buildFileDocSeedResponseSchema, + }, +}) diff --git a/apps/sim/lib/collab-doc/README.md b/apps/sim/lib/collab-doc/README.md new file mode 100644 index 00000000000..4613bb9c3b7 --- /dev/null +++ b/apps/sim/lib/collab-doc/README.md @@ -0,0 +1,53 @@ +# `@/lib/collab-doc` — server-side collaborative-document conversion + +Server-side conversion between a file's **markdown** (the durable source of truth) and its +collaborative **Yjs document**, so the server can own the doc: seed it, project it back to +markdown, and let the agent write into it while a user is typing. + +## Why this exists + +Collaborative file editing had two writers with no shared CRDT: copilot `edit_content` wrote +markdown straight to the file while the user typed into an ephemeral, client-seeded Yjs doc. They +couldn't reconcile — the agent's edit didn't stream into the editor, and last-writer clobbered. The +fix is a **server-authoritative Yjs doc** both sides write into, with markdown as a projection. + +## What Stage A (this module) provides + +| Function | Purpose | +|---|---| +| `markdownToYDoc(md)` | Cold-start seed: file markdown → a fresh `Y.Doc`. | +| `yDocToMarkdown(ydoc)` | Projection: `Y.Doc` → the file's canonical markdown. | +| `applyMarkdownToYDoc(ydoc, md)` | Agent write: merge new content into a live `Y.Doc` as a minimal CRDT diff (no clobber). | + +### Design decisions (why it's not hacky) + +- **Parity by construction.** The markdown↔ProseMirror step reuses the *exact* client engine + (`parseMarkdownToDoc` / `serializeDocToMarkdown`, `@tiptap/markdown` on the shared extension set) — + not a second markdown implementation — so the server can never diverge from what the editor + renders. The custom-fidelity constructs (tables, footnotes, raw HTML, `sim:` mentions) are covered + by the same code that covers them in the browser; the round-trip test asserts equivalence. +- **Same Yjs binding as the browser.** ProseMirror↔Yjs uses `@tiptap/y-tiptap` (what TipTap's + Collaboration extension uses), pinned to the same version and sharing the same `prosemirror-model` + / `yjs` instances (peer deps) — so the structure the server produces is byte-compatible with the + client, targeting the same `'default'` fragment. +- **Merge, not replace.** `applyMarkdownToYDoc` uses `updateYFragment` (the primitive `ySyncPlugin` + runs on every keystroke) to apply only the diff, so Yjs reconciles the agent's write with in-flight + remote edits. The test proves an agent write and a concurrent remote edit both survive. +- **Server-only, DOM via jsdom.** The markdown engine builds a (never-mounted) TipTap editor that + needs a DOM; on the server it's backed by a single lazily-created `jsdom` window. Lazy-required so + the client bundle never pulls jsdom in. + +## Server-authoritative seeding (shipped alongside this module) + +The realtime relay seeds each room's document from this module over an internal endpoint +(`buildFileDocSeed` → `POST /api/internal/file-doc/seed` → `ensureServerSeed`), which let the entire +client-seeder subsystem (election / deadlines / `triedSeeders` / `MAX_SEED_ROUNDS` / the +`SEED_REQUEST` handshake) be deleted. The client's connect-deadline offline fallback is deliberately +**kept** — it is unrelated to seeding. No feature flag: the cutover is all-at-once. + +## Remaining stages (future PRs) + +- **Durable persistence.** A DB column for the Yjs binary + debounced snapshotting, so a document + survives with no collaborators connected instead of being re-seeded from markdown on cold open. +- **Copilot into the doc + projection.** `edit_content` calls `applyMarkdownToYDoc` when a doc is + live; a debounced `yDocToMarkdown` projection keeps the file's markdown current. diff --git a/apps/sim/lib/collab-doc/converter.test.ts b/apps/sim/lib/collab-doc/converter.test.ts new file mode 100644 index 00000000000..7ce8899057f --- /dev/null +++ b/apps/sim/lib/collab-doc/converter.test.ts @@ -0,0 +1,77 @@ +/** + * @vitest-environment jsdom + */ +import { describe, expect, it } from 'vitest' +import * as Y from 'yjs' +import { serializeMarkdownBody } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' +import { applyMarkdownToYDoc, markdownToYDoc, yDocToMarkdown } from './converter' + +/** Representative markdown covering the custom-fidelity constructs (tables, code, lists, marks). */ +const SAMPLES = [ + '# Title\n\nA paragraph with **bold**, *italic*, and `code`.', + '- one\n- two\n - nested\n\n1. first\n2. second', + '> a quote\n\n```ts\nconst x = 1\n```', + '| a | b |\n| --- | --- |\n| 1 | 2 |\n| pipe \\| here | y |', + 'Text with a [link](https://example.com) and a break.\n\nSecond paragraph.', + // Custom-fidelity constructs — the exact cases a second markdown engine would diverge on, and + // that only survive because the server reuses the client's own @tiptap/markdown engine. + 'A footnote reference[^1].\n\n[^1]: the footnote body.', + 'Before.\n\n
untouched raw html
\n\nAfter.', + '- [ ] todo\n- [x] done', +] + +describe('collab-doc converter', () => { + it('round-trips markdown through the Yjs doc identically to the client engine', () => { + for (const md of SAMPLES) { + // yDocToMarkdown(markdownToYDoc(md)) must equal the client's own canonical serialization — + // both go through the exact same @tiptap/markdown engine, so the Yjs hop must be lossless. + expect(yDocToMarkdown(markdownToYDoc(md))).toBe(serializeMarkdownBody(md)) + } + }) + + it('projects an empty doc without throwing', () => { + expect(yDocToMarkdown(markdownToYDoc(''))).toBe(serializeMarkdownBody('')) + }) + + it('applies new content into an existing doc (agent write)', () => { + const ydoc = markdownToYDoc('# Hello\n\nWorld.') + applyMarkdownToYDoc(ydoc, '# Hello\n\nWorld and then some more.') + expect(yDocToMarkdown(ydoc)).toBe(serializeMarkdownBody('# Hello\n\nWorld and then some more.')) + }) + + it('clears an existing doc to empty without throwing (agent write of empty content)', () => { + const ydoc = markdownToYDoc('# Hello\n\nWorld.') + expect(() => applyMarkdownToYDoc(ydoc, '')).not.toThrow() + expect(yDocToMarkdown(ydoc)).toBe(serializeMarkdownBody('')) + }) + + it('merges an agent write with a concurrent remote edit (CRDT, no clobber)', () => { + // Two clients start from the same state. + const server = markdownToYDoc('# Doc\n\nAlpha paragraph.\n\nBeta paragraph.') + const remote = new Y.Doc() + Y.applyUpdate(remote, Y.encodeStateAsUpdate(server)) + + // The remote user edits the FIRST paragraph's text directly on the shared type, concurrently… + const remoteFrag = remote.getXmlFragment('default') + remote.transact(() => { + // second child (index 1) is the first paragraph; append " EDITED" to its text node. + const firstPara = remoteFrag.get(1) as Y.XmlElement + const textNode = firstPara.get(0) as Y.XmlText + textNode.insert(textNode.toString().length, ' EDITED') + }) + + // …while the agent rewrites the SECOND paragraph via the converter on the server doc. + applyMarkdownToYDoc( + server, + '# Doc\n\nAlpha paragraph.\n\nBeta paragraph, expanded by the agent.' + ) + + // Exchange updates both ways (as the relay would) and both edits must survive the merge. + Y.applyUpdate(server, Y.encodeStateAsUpdate(remote, Y.encodeStateVector(server))) + Y.applyUpdate(remote, Y.encodeStateAsUpdate(server, Y.encodeStateVector(remote))) + + const merged = yDocToMarkdown(server) + expect(merged).toContain('Alpha paragraph. EDITED') + expect(merged).toContain('expanded by the agent') + }) +}) diff --git a/apps/sim/lib/collab-doc/converter.ts b/apps/sim/lib/collab-doc/converter.ts new file mode 100644 index 00000000000..e18c1ca8ffc --- /dev/null +++ b/apps/sim/lib/collab-doc/converter.ts @@ -0,0 +1,105 @@ +import { getSchema } from '@tiptap/core' +import { Node as ProseMirrorNode, type Schema } from '@tiptap/pm/model' +import { + initProseMirrorDoc, + prosemirrorJSONToYDoc, + updateYFragment, + yDocToProsemirrorJSON, +} from '@tiptap/y-tiptap' +import type * as Y from 'yjs' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + parseMarkdownToDoc, + serializeDocToMarkdown, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' + +/** + * Server-side conversion between a file's markdown and its collaborative Yjs document. + * + * The markdown ↔ ProseMirror step reuses the EXACT client engine (`parseMarkdownToDoc` / + * `serializeDocToMarkdown`, both driven by `@tiptap/markdown` on the shared extension set), so the + * server can never diverge from what the editor renders — parity by construction, not by a second + * markdown implementation. The ProseMirror ↔ Yjs step uses `@tiptap/y-tiptap` (the same binding + * TipTap's Collaboration extension uses in the browser), so the Yjs structure the server produces is + * byte-compatible with the client's. + * + * The TipTap editor the markdown engine builds needs a DOM; on the server we back it with `jsdom` + * (see {@link ensureDomForTipTap}). This module is server-only by construction — it must never reach + * the client bundle (jsdom + the full editor would bloat it and break in the browser). It is kept + * out of that bundle by `require`-ing `jsdom` lazily (never a static top-level import) and by being + * imported only from server code (the seed builder + its internal route); there is no `import + * 'server-only'` marker because this repo does not use that package. + */ + +/** + * The Yjs `XmlFragment` name TipTap's Collaboration extension binds to. The client configures + * `Collaboration.configure({ document })` with no explicit `field`, so it uses TipTap's default, + * `'default'`. The server MUST target the same fragment or the client would sync an empty document. + */ +export const COLLAB_DOC_FIELD = 'default' + +let cachedSchema: Schema | null = null + +/** The shared ProseMirror schema, built headlessly from the exact client extension set. */ +function markdownSchema(): Schema { + if (!cachedSchema) cachedSchema = getSchema(createMarkdownContentExtensions()) + return cachedSchema +} + +let domReady = false + +/** + * Ensure a DOM exists for the TipTap editor the markdown engine constructs. In a `jsdom` test + * environment `document` already exists and this is a no-op; in a plain Node server it installs a + * single shared jsdom window's globals once. Kept minimal and idempotent — TipTap only needs + * `document`/`window`/`navigator` to build its (never-mounted) editor for parse/serialize. + */ +function ensureDomForTipTap(): void { + if (domReady || typeof document !== 'undefined') { + domReady = true + return + } + // Lazy require so the client bundle never pulls jsdom in. + const { JSDOM } = require('jsdom') as typeof import('jsdom') + const { window } = new JSDOM('') + const g = globalThis as unknown as Record + g.window ??= window + g.document ??= window.document + g.navigator ??= window.navigator + domReady = true +} + +/** Convert a file's markdown to a fresh collaborative {@link Y.Doc} (cold-start seed). */ +export function markdownToYDoc(markdown: string): Y.Doc { + ensureDomForTipTap() + const json = parseMarkdownToDoc(markdown) + return prosemirrorJSONToYDoc(markdownSchema(), json, COLLAB_DOC_FIELD) +} + +/** Project a collaborative {@link Y.Doc} back to the file's canonical markdown. */ +export function yDocToMarkdown(ydoc: Y.Doc): string { + ensureDomForTipTap() + const json = yDocToProsemirrorJSON(ydoc, COLLAB_DOC_FIELD) + return serializeDocToMarkdown(json) +} + +/** + * Apply new markdown content into an EXISTING collaborative {@link Y.Doc} as a minimal CRDT diff, + * merging with any concurrent user edits rather than replacing the document. This is how the agent + * writes into a live doc: `updateYFragment` computes exactly the changes between the fragment's + * current content and the target and applies them as Yjs operations — the same primitive TipTap's + * `ySyncPlugin` uses on every keystroke — so Yjs reconciles them with in-flight remote edits. + */ +export function applyMarkdownToYDoc(ydoc: Y.Doc, markdown: string): void { + ensureDomForTipTap() + const schema = markdownSchema() + const target = ProseMirrorNode.fromJSON(schema, parseMarkdownToDoc(markdown)) + const fragment = ydoc.getXmlFragment(COLLAB_DOC_FIELD) + // `updateYFragment` diffs against the fragment's CURRENT content, so it needs the fragment↔PM + // binding metadata (the element/mark mapping the live editor's ySyncPlugin normally maintains). + // `initProseMirrorDoc` reconstructs it from the fragment's present state. + const { meta } = initProseMirrorDoc(fragment, schema) + ydoc.transact(() => { + updateYFragment(ydoc, fragment, target, meta) + }) +} diff --git a/apps/sim/lib/collab-doc/index.ts b/apps/sim/lib/collab-doc/index.ts new file mode 100644 index 00000000000..ffee345264e --- /dev/null +++ b/apps/sim/lib/collab-doc/index.ts @@ -0,0 +1,6 @@ +export { + applyMarkdownToYDoc, + COLLAB_DOC_FIELD, + markdownToYDoc, + yDocToMarkdown, +} from './converter' diff --git a/apps/sim/lib/collab-doc/seed.test.ts b/apps/sim/lib/collab-doc/seed.test.ts new file mode 100644 index 00000000000..c3ba8bf9bcf --- /dev/null +++ b/apps/sim/lib/collab-doc/seed.test.ts @@ -0,0 +1,74 @@ +/** + * @vitest-environment jsdom + */ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' + +const { mockGetWorkspaceFile, mockFetchBuffer } = vi.hoisted(() => ({ + mockGetWorkspaceFile: vi.fn(), + mockFetchBuffer: vi.fn(), +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + getWorkspaceFile: mockGetWorkspaceFile, + fetchWorkspaceFileBuffer: mockFetchBuffer, +})) + +import { serializeMarkdownBody } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' +import { yDocToMarkdown } from './converter' +import { buildFileDocSeed } from './seed' + +describe('buildFileDocSeed', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetWorkspaceFile.mockResolvedValue({ + id: 'file-1', + name: 'note.md', + key: 'k', + context: 'workspace', + }) + }) + + it('builds a seed whose applied update reproduces the file body (through the client engine)', async () => { + mockFetchBuffer.mockResolvedValue(Buffer.from('# Title\n\nHello **world**.', 'utf-8')) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + expect(seed).not.toBeNull() + + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(yDocToMarkdown(doc)).toBe(serializeMarkdownBody('# Title\n\nHello **world**.')) + }) + + it('strips frontmatter — only the body seeds the collaborative doc', async () => { + mockFetchBuffer.mockResolvedValue(Buffer.from('---\ntitle: X\n---\n\n# Body\n\ntext.', 'utf-8')) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + const md = yDocToMarkdown(doc) + expect(md).not.toContain('title: X') + expect(md).toBe(serializeMarkdownBody('# Body\n\ntext.')) + }) + + it('marks the seeded doc as initial-content-loaded so the client needs no seeder handshake', async () => { + mockFetchBuffer.mockResolvedValue(Buffer.from('# Body', 'utf-8')) + const seed = await buildFileDocSeed('ws-1', 'file-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + }) + + it('returns null for a missing file', async () => { + mockGetWorkspaceFile.mockResolvedValue(null) + expect(await buildFileDocSeed('ws-1', 'missing')).toBeNull() + }) + + it('requests the file with throwOnError so a read failure is not mistaken for an empty file', async () => { + mockGetWorkspaceFile.mockRejectedValue(new Error('db down')) + // Propagates instead of returning null — the relay must retry, never seed blank over a real file. + await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toThrow('db down') + expect(mockGetWorkspaceFile).toHaveBeenCalledWith('ws-1', 'file-1', { throwOnError: true }) + }) +}) diff --git a/apps/sim/lib/collab-doc/seed.ts b/apps/sim/lib/collab-doc/seed.ts new file mode 100644 index 00000000000..38c37c690b1 --- /dev/null +++ b/apps/sim/lib/collab-doc/seed.ts @@ -0,0 +1,53 @@ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import * as Y from 'yjs' +import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contexts/workspace' +import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { markdownToYDoc } from './converter' + +/** + * The largest file we will build a collaborative seed for. Beyond this the editor uses its + * non-collaborative path anyway; converting a huge document server-side would be wasted work. + */ +const MAX_SEED_BYTES = 5 * 1024 * 1024 + +/** A collaborative document's initial state, encoded as a Yjs update the relay can apply. */ +export interface FileDocSeed { + /** `Y.encodeStateAsUpdate` of the seeded document — apply with `Y.applyUpdate`. */ + update: Uint8Array +} + +/** + * Build the server-side seed for a file's collaborative document: load the file's current markdown + * and convert it — through the exact client engine (see {@link markdownToYDoc}) — into a Yjs update. + * + * This is what makes seeding server-authoritative: the realtime relay applies this to a fresh room's + * document instead of electing a client to import the content, so the whole client-seeder subsystem + * (election / deadlines / retries) goes away. The frontmatter is stripped exactly as the client's + * seed did — it is file metadata, not part of the collaborative body. + * + * Returns `null` ONLY when the file is genuinely absent (deleted/never-existed). A transient read + * failure THROWS (`throwOnError`) rather than returning `null`, so the relay retries instead of + * mistaking a DB blip for an empty file and seeding blank content over the real document. + */ +export async function buildFileDocSeed( + workspaceId: string, + fileId: string +): Promise { + const record = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) + if (!record) return null + + const buffer = await fetchWorkspaceFileBuffer(record, { maxBytes: MAX_SEED_BYTES }) + const markdown = buffer.toString('utf-8') + const { body } = splitFrontmatter(markdown) + + const ydoc = markdownToYDoc(body) + try { + // Mark the document seeded IN the same doc, so the client's readiness gate + // (`synced && initialContentLoaded === true`) recognizes a server-seeded doc without any + // client-seeder handshake, and a stray re-election can never seed on top of it. + ydoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + return { update: Y.encodeStateAsUpdate(ydoc) } + } finally { + ydoc.destroy() + } +} diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 6e5985315e7..2d6640451fd 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -937,12 +937,18 @@ export async function resolveWorkspaceFileReference( } /** - * Get a specific workspace file + * Get a specific workspace file. + * + * By default a DB error is logged and swallowed to `null` — for most callers "couldn't load it" + * and "doesn't exist" are handled the same way. Pass `{ throwOnError: true }` when the caller must + * distinguish a genuinely-absent file (`null`) from a transient read failure (throws): the + * collaborative-doc seed builder relies on this so a DB blip never looks like an empty file and gets + * seeded as blank content over the real document. */ export async function getWorkspaceFile( workspaceId: string, fileId: string, - options?: { includeDeleted?: boolean } + options?: { includeDeleted?: boolean; throwOnError?: boolean } ): Promise { try { const { includeDeleted = false } = options ?? {} @@ -970,6 +976,7 @@ export async function getWorkspaceFile( return mapSingleWorkspaceFileRecord(files[0], workspaceId) } catch (error) { logger.error(`Failed to get workspace file ${fileId}:`, error) + if (options?.throwOnError) throw error return null } } diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index 1ee1b2389e9..de05dcbb585 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -132,9 +132,16 @@ const nextConfig: NextConfig = { '@daytonaio/sdk', '@earendil-works/pi-ai', '@earendil-works/pi-coding-agent', + // The collab-doc seed converter lazily `require`s jsdom for a headless TipTap editor. Keep it + // external so webpack doesn't try to bundle jsdom's dynamic internal requires. + 'jsdom', ], outputFileTracingIncludes: { '/api/tools/stagehand/*': ['./node_modules/ws/**/*'], + // The seed endpoint's lazy `require('jsdom')` is invisible to the standalone file tracer, so + // force jsdom (and its transitive deps, followed from its static requires) into the trace — + // otherwise a Docker/standalone build omits it and the endpoint 500s with MODULE_NOT_FOUND. + '/api/internal/file-doc/seed': ['./node_modules/jsdom/**/*'], '/*': [ './node_modules/sharp/**/*', './node_modules/@img/**/*', diff --git a/apps/sim/package.json b/apps/sim/package.json index c672ba01836..2f052f564b3 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -131,6 +131,7 @@ "@tiptap/react": "3.26.1", "@tiptap/starter-kit": "3.26.1", "@tiptap/suggestion": "3.26.1", + "@tiptap/y-tiptap": "3.0.7", "@trigger.dev/sdk": "4.4.3", "@typescript/typescript6": "^6.0.2", "ajv": "8.18.0", @@ -172,6 +173,7 @@ "jose": "6.0.11", "js-tiktoken": "1.0.21", "js-yaml": "4.3.0", + "jsdom": "^26.0.0", "jszip": "3.10.1", "lib0": "0.2.117", "lru-cache": "11.3.6", @@ -255,7 +257,6 @@ "@vitejs/plugin-react": "^4.3.4", "@vitest/coverage-v8": "^4.1.0", "autoprefixer": "10.4.21", - "jsdom": "^26.0.0", "postcss": "^8", "react-email": "4.3.2", "tailwindcss": "^3.4.1", diff --git a/bun.lock b/bun.lock index 4bf8d191d30..a713389eff2 100644 --- a/bun.lock +++ b/bun.lock @@ -209,6 +209,7 @@ "@tiptap/react": "3.26.1", "@tiptap/starter-kit": "3.26.1", "@tiptap/suggestion": "3.26.1", + "@tiptap/y-tiptap": "3.0.7", "@trigger.dev/sdk": "4.4.3", "@typescript/typescript6": "^6.0.2", "ajv": "8.18.0", @@ -250,6 +251,7 @@ "jose": "6.0.11", "js-tiktoken": "1.0.21", "js-yaml": "4.3.0", + "jsdom": "^26.0.0", "jszip": "3.10.1", "lib0": "0.2.117", "lru-cache": "11.3.6", @@ -333,7 +335,6 @@ "@vitejs/plugin-react": "^4.3.4", "@vitest/coverage-v8": "^4.1.0", "autoprefixer": "10.4.21", - "jsdom": "^26.0.0", "postcss": "^8", "react-email": "4.3.2", "tailwindcss": "^3.4.1", diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index 9d2fbe3a5db..5590d7988b6 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -21,13 +21,6 @@ export const FILE_DOC_EVENTS = { JOIN_SUCCESS: 'join-file-doc-success', /** Server → client: join rejected ({@link JoinFileDocError}). */ JOIN_ERROR: 'join-file-doc-error', - /** - * Server → client: this client is elected to seed the document's initial - * content from the file's stored markdown ({@link SeedRequestPayload}). Sent to - * exactly one client of an unseeded document — at join, or later to a remaining - * client if the previously-elected seeder disconnects before it seeds. - */ - SEED_REQUEST: 'file-doc-seed-request', /** Client → server: leave the session ({@link LeaveFileDocPayload}). */ LEAVE: 'leave-file-doc', /** Both directions: a framed Yjs message (binary), tagged by {@link FILE_DOC_MESSAGE_TYPE}. */ @@ -54,18 +47,26 @@ export const FILE_DOC_MESSAGE_TYPE = { export type FileDocMessageType = (typeof FILE_DOC_MESSAGE_TYPE)[keyof typeof FILE_DOC_MESSAGE_TYPE] /** - * Where the client records that it has seeded the document's initial content, - * stored inside the Yjs document as `doc.getMap(configMap).get(flag) === true`. - * Because it lives in the CRDT it merges across clients and the server can read - * it — the server uses it to decide whether to re-elect a seeder when an elected - * one disconnects before seeding. Client and server MUST use these exact keys. + * Where "the document's initial content has been seeded" is recorded, stored inside the Yjs + * document as `doc.getMap(configMap).get(flag) === true`. It lives in the CRDT so it merges across + * clients; the client reads it as half of its readiness gate (`synced && initialContentLoaded`). + * + * The server seeds authoritatively: it builds the document from the file's stored markdown and sets + * this flag in the SAME update, so a seeded document arrives content-and-flag together and the + * client needs no seed handshake. `configMap` is a reserved top-level Y.Map name; the editor must + * not use a top-level type of the same name (TipTap uses `getXmlFragment('default')`, no collision). * - * The seeding client MUST write the imported content AND set this flag in a - * SINGLE Yjs transaction (`doc.transact(...)`). Otherwise a seeder that dies - * between the two writes can leave content with no flag, and a re-elected client - * would seed again and duplicate it. `configMap` is a reserved top-level Y.Map - * name; the editor must not use a top-level type of the same name (TipTap uses - * `getXmlFragment('default')`, so there is no collision today). + * Consumer (editor hook) contract: + * 1. **Never overwrite content with an unseeded doc.** The markdown-mirror autosave MUST be gated on + * the document being both synced AND seeded — otherwise an empty/still-syncing doc could be saved + * over the real file (the one true data-loss path). + * 2. **One provider per socket.** Destroy the previous provider before creating the next (document + * switch), so a stale provider's binary-frame listener can't apply another document's updates. + * 3. **Treat a fatal (`retryable: false`) join error as terminal.** Latch it and fall back to a + * read-only view of the file's stored content — do not keep rejoining. The server auto-reclaims a + * same-user client-id collision silently (the reconnecting socket succeeds), so `CLIENT_ID_IN_USE` + * surfaces to a client only for the rare case of a *different* user colliding on the same random + * Yjs `clientID`; a page reload mints a new id and recovers. */ export const FILE_DOC_SEED = { configMap: 'config', @@ -88,33 +89,6 @@ export interface JoinFileDocSuccess { fileId: string } -/** - * Server → client seed election ({@link FILE_DOC_EVENTS.SEED_REQUEST}). The - * recipient imports the file's stored markdown into the (empty) document. The - * client still guards on the CRDT `initialContentLoaded` flag so a re-election - * that races an in-flight seed can never duplicate content. - * - * Consumer (editor hook) contract — the relay depends on these to be safe: - * 1. **Never overwrite content with an unseeded doc.** Autosave (the markdown - * mirror written through the content API) MUST be gated on the document being - * both synced AND seeded (`initialContentLoaded === true`). Otherwise an empty - * or still-syncing doc could be saved over the real file — the one true - * data-loss path, and the reason a withholding seeder is only a liveness - * nuisance rather than destructive. - * 2. **Seed atomically, or leave.** Write content + the flag in ONE - * `doc.transact(...)`; if seeding fails, emit {@link FILE_DOC_EVENTS.LEAVE} - * (or destroy the provider) so the server re-elects another client. - * 3. **One provider per socket.** Destroy the previous {@link FILE_DOC_EVENTS} - * provider before creating the next (document switch), so a stale provider's - * binary-frame listener can't apply another document's updates. - * 4. **Re-mint on CLIENT_ID_IN_USE.** On a `CLIENT_ID_IN_USE` join error, - * recreate the Yjs doc (fresh `clientID`) and rejoin rather than giving up — - * the id is transiently held by another socket of the same user. - */ -export interface SeedRequestPayload { - fileId: string -} - /** Server → client rejection of a {@link FILE_DOC_EVENTS.JOIN}. */ export interface JoinFileDocError { fileId: string diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 0751de73108..7fc239cffae 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 979, - zodRoutes: 979, + totalRoutes: 980, + zodRoutes: 980, nonZodRoutes: 0, } as const From 66eef09c870f0f8add8d77bd533075225018031b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 14:35:01 -0700 Subject: [PATCH 31/53] feat(collab-doc): Sim merge endpoint for copilot-into-doc (Stage C foundation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildFileDocMergeUpdate(docState, markdown) computes the minimal Yjs diff that turns a live document into target markdown, via applyMarkdownToYDoc (a real updateYFragment diff, not a replace) — so a copilot rewrite merges with concurrent user edits instead of clobbering them. Exposed over the internal x-api-key /api/internal/file-doc/merge endpoint the realtime relay will call: the relay owns the doc, the app owns the conversion engine, so the relay ships the current state and applies the returned diff. Tested incl. concurrent-edit no-clobber. --- .../app/api/internal/file-doc/merge/route.ts | 37 +++++++++++++ apps/sim/lib/api/contracts/file-doc.ts | 37 +++++++++++++ apps/sim/lib/collab-doc/index.ts | 2 + apps/sim/lib/collab-doc/merge.test.ts | 52 +++++++++++++++++++ apps/sim/lib/collab-doc/merge.ts | 26 ++++++++++ scripts/check-api-validation-contracts.ts | 4 +- 6 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 apps/sim/app/api/internal/file-doc/merge/route.ts create mode 100644 apps/sim/lib/collab-doc/merge.test.ts create mode 100644 apps/sim/lib/collab-doc/merge.ts diff --git a/apps/sim/app/api/internal/file-doc/merge/route.ts b/apps/sim/app/api/internal/file-doc/merge/route.ts new file mode 100644 index 00000000000..40f7e5204c0 --- /dev/null +++ b/apps/sim/app/api/internal/file-doc/merge/route.ts @@ -0,0 +1,37 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { mergeFileDocContract } from '@/lib/api/contracts/file-doc' +import { parseRequest } from '@/lib/api/server' +import { buildFileDocMergeUpdate } from '@/lib/collab-doc/merge' +import { checkInternalApiKey, createUnauthorizedResponse } from '@/lib/copilot/request/http' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + +const logger = createLogger('FileDocMergeAPI') + +/** + * POST /api/internal/file-doc/merge — merge new markdown into a live collaborative document as a + * minimal Yjs diff (Stage C — copilot writing into an open doc). The realtime relay ships the current + * doc state; the app returns the diff to apply + relay. Internal only: gated on the shared + * `x-api-key: INTERNAL_API_SECRET` secret, matching the seed endpoint and the realtime relay. + */ +export const POST = withRouteHandler(async (request: NextRequest) => { + const auth = checkInternalApiKey(request) + if (!auth.success) return createUnauthorizedResponse() + + const parsed = await parseRequest(mergeFileDocContract, request, {}) + if (!parsed.success) return parsed.response + const { fileId, docState, markdown } = parsed.data.body + + try { + const update = buildFileDocMergeUpdate(Buffer.from(docState, 'base64'), markdown) + return NextResponse.json({ update: Buffer.from(update).toString('base64') }) + } catch (error) { + logger.error('Failed to merge markdown into file-doc', { fileId, error }) + return NextResponse.json( + { error: getErrorMessage(error, 'Failed to merge document') }, + { status: 500 } + ) + } +}) diff --git a/apps/sim/lib/api/contracts/file-doc.ts b/apps/sim/lib/api/contracts/file-doc.ts index 27337936f90..24d03100f53 100644 --- a/apps/sim/lib/api/contracts/file-doc.ts +++ b/apps/sim/lib/api/contracts/file-doc.ts @@ -31,3 +31,40 @@ export const buildFileDocSeedContract = defineRouteContract({ schema: buildFileDocSeedResponseSchema, }, }) + +export const mergeFileDocBodySchema = z.object({ + /** For logging/observability only — the merge itself is stateless. */ + fileId: z.string().min(1, 'fileId is required'), + /** Base64-encoded `Y.encodeStateAsUpdate` of the live document as the relay currently holds it. */ + docState: z.string().min(1, 'docState is required'), + /** The full markdown body the caller (e.g. copilot) wants the document to become. */ + markdown: z.string(), +}) +export type MergeFileDocBody = z.input + +export const mergeFileDocResponseSchema = z.object({ + /** + * Base64-encoded Yjs update — the minimal CRDT diff that transforms the supplied document state + * into `markdown`. The relay applies it to the live doc, which merges it with any concurrent user + * edits and relays it to every connected editor. Empty (a no-op update) when nothing changed. + */ + update: z.string(), +}) +export type MergeFileDocResponse = z.output + +/** + * Internal, `x-api-key`-gated: the realtime relay asks the app to merge new markdown into a live + * collaborative document as a minimal CRDT diff (Stage C — copilot writing into an open doc). The app + * owns the conversion engine; the relay owns the doc — so the relay ships the current doc state here, + * gets back the diff, and applies + relays it. Stateless (no blob/DB access): the caller supplies both + * the current state and the target markdown. + */ +export const mergeFileDocContract = defineRouteContract({ + method: 'POST', + path: '/api/internal/file-doc/merge', + body: mergeFileDocBodySchema, + response: { + mode: 'json', + schema: mergeFileDocResponseSchema, + }, +}) diff --git a/apps/sim/lib/collab-doc/index.ts b/apps/sim/lib/collab-doc/index.ts index ffee345264e..1309372b0bf 100644 --- a/apps/sim/lib/collab-doc/index.ts +++ b/apps/sim/lib/collab-doc/index.ts @@ -4,3 +4,5 @@ export { markdownToYDoc, yDocToMarkdown, } from './converter' +export { buildFileDocMergeUpdate } from './merge' +export { buildFileDocSeed, type FileDocSeed } from './seed' diff --git a/apps/sim/lib/collab-doc/merge.test.ts b/apps/sim/lib/collab-doc/merge.test.ts new file mode 100644 index 00000000000..92820ee8a60 --- /dev/null +++ b/apps/sim/lib/collab-doc/merge.test.ts @@ -0,0 +1,52 @@ +/** + * @vitest-environment jsdom + */ +import { describe, expect, it } from 'vitest' +import * as Y from 'yjs' +import { serializeMarkdownBody } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' +import { markdownToYDoc, yDocToMarkdown } from './converter' +import { buildFileDocMergeUpdate } from './merge' + +describe('buildFileDocMergeUpdate', () => { + it('produces a diff that turns the live doc into the target markdown', () => { + const live = markdownToYDoc('# Title\n\nOriginal.') + const update = buildFileDocMergeUpdate(Y.encodeStateAsUpdate(live), '# Title\n\nRewritten.') + Y.applyUpdate(live, update) + expect(yDocToMarkdown(live)).toBe(serializeMarkdownBody('# Title\n\nRewritten.')) + }) + + it('returns an empty (no-op) diff when the markdown already matches', () => { + const live = markdownToYDoc('# Same\n\nBody.') + const before = Y.encodeStateVector(live) + const update = buildFileDocMergeUpdate(Y.encodeStateAsUpdate(live), yDocToMarkdown(live)) + Y.applyUpdate(live, update) + // Nothing new was introduced: the doc's state vector is unchanged by applying the diff. + expect(Y.encodeStateVector(live)).toEqual(before) + }) + + it('merges a copilot rewrite with a concurrent remote edit (no clobber)', () => { + // The relay captures the live doc's state and asks the app for the diff… + const live = markdownToYDoc('# Doc\n\nAlpha paragraph.\n\nBeta paragraph.') + const capturedState = Y.encodeStateAsUpdate(live) + + // …meanwhile a remote user edits the FIRST paragraph directly on the live doc, AFTER the capture. + const frag = live.getXmlFragment('default') + live.transact(() => { + const firstPara = frag.get(1) as Y.XmlElement + const textNode = firstPara.get(0) as Y.XmlText + textNode.insert(textNode.toString().length, ' EDITED') + }) + + // Copilot rewrites the SECOND paragraph; the diff is computed against the CAPTURED (older) state. + const update = buildFileDocMergeUpdate( + capturedState, + '# Doc\n\nAlpha paragraph.\n\nBeta paragraph, expanded by the agent.' + ) + // Applying the diff to the now-advanced live doc must preserve BOTH edits. + Y.applyUpdate(live, update) + + const merged = yDocToMarkdown(live) + expect(merged).toContain('Alpha paragraph. EDITED') + expect(merged).toContain('expanded by the agent') + }) +}) diff --git a/apps/sim/lib/collab-doc/merge.ts b/apps/sim/lib/collab-doc/merge.ts new file mode 100644 index 00000000000..0715fb80f15 --- /dev/null +++ b/apps/sim/lib/collab-doc/merge.ts @@ -0,0 +1,26 @@ +import * as Y from 'yjs' +import { applyMarkdownToYDoc } from './converter' + +/** + * Compute the minimal Yjs diff that turns `docState` — the live collaborative document as the realtime + * relay currently holds it — into `markdown`. This is the Stage C "copilot writes into the open doc" + * primitive: the relay owns the doc but not the conversion engine, so it ships the current state here + * and applies the returned diff, which Yjs merges with any concurrent user edits before relaying it to + * every connected editor. + * + * `applyMarkdownToYDoc` performs a real `updateYFragment` diff (not a replace), so unrelated + * paragraphs the user is editing are preserved. The returned update is relative to the document's + * state at call time (`Y.encodeStateAsUpdate(doc, before)`), so it is exactly the change to apply — and + * is empty (a no-op update) when `markdown` already matches the document. + */ +export function buildFileDocMergeUpdate(docState: Uint8Array, markdown: string): Uint8Array { + const doc = new Y.Doc() + try { + Y.applyUpdate(doc, docState) + const before = Y.encodeStateVector(doc) + applyMarkdownToYDoc(doc, markdown) + return Y.encodeStateAsUpdate(doc, before) + } finally { + doc.destroy() + } +} diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 7fc239cffae..1e073231776 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 980, - zodRoutes: 980, + totalRoutes: 981, + zodRoutes: 981, nonZodRoutes: 0, } as const From 17f17e8afaee6a8ee77d0b65a55e572136cc31c6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 14:42:54 -0700 Subject: [PATCH 32/53] =?UTF-8?q?feat(collab-doc):=20realtime=20apply-edit?= =?UTF-8?q?=20=E2=80=94=20merge=20copilot=20markdown=20into=20a=20live=20d?= =?UTF-8?q?oc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The relay can now stream a copilot edit into open editors: applyMarkdownToLiveFileDoc finds the seeded live room, ships its state to the app's /merge endpoint for a minimal CRDT diff, applies it (relaying to every editor, reconciled with concurrent user edits), and reports 'no-live-room' so the caller falls back to a direct file write when nothing is open. - Generalize the realtime->app request module (file-doc-seed.ts -> file-doc-app.ts) with a shared POST helper + fetchFileDocSeed/fetchFileDocMerge - POST /api/file-doc/apply-edit on the internal x-api-key HTTP surface, returning { applied } - Tests for the seeded-room merge relay and the no-live-room fallback --- apps/realtime/src/handlers/file-doc-app.ts | 77 +++++++++++++++++++++ apps/realtime/src/handlers/file-doc-seed.ts | 45 ------------ apps/realtime/src/handlers/file-doc.test.ts | 47 ++++++++++++- apps/realtime/src/handlers/file-doc.ts | 28 +++++++- apps/realtime/src/routes/http.ts | 21 ++++++ 5 files changed, 169 insertions(+), 49 deletions(-) create mode 100644 apps/realtime/src/handlers/file-doc-app.ts delete mode 100644 apps/realtime/src/handlers/file-doc-seed.ts diff --git a/apps/realtime/src/handlers/file-doc-app.ts b/apps/realtime/src/handlers/file-doc-app.ts new file mode 100644 index 00000000000..7ec74f6a61d --- /dev/null +++ b/apps/realtime/src/handlers/file-doc-app.ts @@ -0,0 +1,77 @@ +import { env, getBaseUrl } from '@/env' + +/** + * The relay's client for the app's internal file-doc endpoints. The app owns the markdown↔Yjs + * conversion engine (TipTap + jsdom) and blob/DB access; the relay owns the live document. So for any + * operation that needs conversion, the relay delegates here over the shared `x-api-key` channel. + */ + +/** + * Bound each request so a slow/unreachable app never wedges the relay. 8s is generous for the + * underlying markdown↔Yjs conversion — collab is gated client-side to ≤256 KB documents + * (`isRoundTripSafe`), which convert in well under a second. The seed additionally must stay under the + * client's readiness deadline (`READINESS_DEADLINE_MS`, 12s, in `file-doc-provider.ts`), which this + * satisfies. + */ +const APP_REQUEST_TIMEOUT_MS = 8_000 + +function postToApp(path: string, payload: unknown): Promise { + return fetch(`${getBaseUrl()}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(APP_REQUEST_TIMEOUT_MS), + }) +} + +/** + * Ask the app to build a server-authoritative seed (markdown → Yjs) for a file's collaborative + * document. Returns the Yjs update to apply, or `null` for a genuinely empty/missing file (an empty + * document is correct). THROWS on a transport failure (non-2xx / network / timeout / malformed body) + * so the caller can tell a real empty from a failure it should be allowed to retry. + */ +export async function fetchFileDocSeed( + workspaceId: string, + fileId: string +): Promise { + const response = await postToApp('/api/internal/file-doc/seed', { workspaceId, fileId }) + if (!response.ok) { + throw new Error(`Seed fetch failed for file ${fileId}: ${response.status}`) + } + const body = (await response.json()) as { update?: unknown } + const update = body?.update + // A well-formed response is `{ update: base64-string | null }`. Anything else is a contract + // violation, not a "genuinely empty file" — throw so the caller retries rather than silently + // treating a malformed body as empty and stranding the room unseeded. + if (update === null) return null + if (typeof update !== 'string') { + throw new Error(`Seed fetch for file ${fileId} returned a malformed body`) + } + return new Uint8Array(Buffer.from(update, 'base64')) +} + +/** + * Ask the app to merge new markdown into the live document as a minimal Yjs diff — Stage C, so a + * copilot edit streams into open editors instead of the file changing underneath them. The relay + * ships the document's current state and applies the returned diff (which Yjs reconciles with any + * concurrent user edits). THROWS on a transport failure or malformed body. + */ +export async function fetchFileDocMerge( + fileId: string, + docState: Uint8Array, + markdown: string +): Promise { + const response = await postToApp('/api/internal/file-doc/merge', { + fileId, + docState: Buffer.from(docState).toString('base64'), + markdown, + }) + if (!response.ok) { + throw new Error(`Merge fetch failed for file ${fileId}: ${response.status}`) + } + const body = (await response.json()) as { update?: unknown } + if (typeof body?.update !== 'string') { + throw new Error(`Merge fetch for file ${fileId} returned a malformed body`) + } + return new Uint8Array(Buffer.from(body.update, 'base64')) +} diff --git a/apps/realtime/src/handlers/file-doc-seed.ts b/apps/realtime/src/handlers/file-doc-seed.ts deleted file mode 100644 index 6217d0d2e36..00000000000 --- a/apps/realtime/src/handlers/file-doc-seed.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { env, getBaseUrl } from '@/env' - -/** - * Bound the seed fetch so a slow/unreachable app never wedges a room's first join. Kept comfortably - * BELOW the client's readiness deadline (`READINESS_DEADLINE_MS`, 12s, in `file-doc-provider.ts`), so - * a single attempt either lands or fails within the window the client is willing to wait — otherwise - * the client gives up first and a late success can't reach it. Generous for a real seed: collab is - * gated client-side to ≤256 KB documents (`isRoundTripSafe`), which convert markdown→Yjs in well - * under a second; the 5 MB server cap is only a backstop and is never reached for a collab file. - */ -const SEED_FETCH_TIMEOUT_MS = 8_000 - -/** - * Ask the app to build a server-authoritative seed (markdown → Yjs) for a file's collaborative - * document. Only the app can — it owns the editor extensions + blob access — so the relay delegates - * over the internal endpoint. - * - * Returns the Yjs update to apply, or `null` for a genuinely empty/missing file (an empty document - * is correct). THROWS on a transport failure (non-2xx / network / timeout) so the caller can tell a - * real empty from a failure it should be allowed to retry. - */ -export async function fetchFileDocSeed( - workspaceId: string, - fileId: string -): Promise { - const response = await fetch(`${getBaseUrl()}/api/internal/file-doc/seed`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, - body: JSON.stringify({ workspaceId, fileId }), - signal: AbortSignal.timeout(SEED_FETCH_TIMEOUT_MS), - }) - if (!response.ok) { - throw new Error(`Seed fetch failed for file ${fileId}: ${response.status}`) - } - const body = (await response.json()) as { update?: unknown } - const update = body?.update - // A well-formed response is `{ update: base64-string | null }`. Anything else is a contract - // violation, not a "genuinely empty file" — throw so the caller retries rather than silently - // treating a malformed body as empty and stranding the room unseeded. - if (update === null) return null - if (typeof update !== 'string') { - throw new Error(`Seed fetch for file ${fileId} returned a malformed body`) - } - return new Uint8Array(Buffer.from(update, 'base64')) -} diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 6d569f4a9e8..cc6bd85d527 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -14,20 +14,26 @@ import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' import type { IRoomManager } from '@/rooms' -const { mockAuthorizeRoom, mockFetchFileDocSeed } = vi.hoisted(() => ({ +const { mockAuthorizeRoom, mockFetchFileDocSeed, mockFetchFileDocMerge } = vi.hoisted(() => ({ mockAuthorizeRoom: vi.fn(), mockFetchFileDocSeed: vi.fn(), + mockFetchFileDocMerge: vi.fn(), })) vi.mock('@sim/platform-authz/rooms', () => ({ authorizeRoom: mockAuthorizeRoom, })) -vi.mock('@/handlers/file-doc-seed', () => ({ +vi.mock('@/handlers/file-doc-app', () => ({ fetchFileDocSeed: mockFetchFileDocSeed, + fetchFileDocMerge: mockFetchFileDocMerge, })) -import { cleanupFileDocForSocket, setupWorkspaceFileDocHandlers } from '@/handlers/file-doc' +import { + applyMarkdownToLiveFileDoc, + cleanupFileDocForSocket, + setupWorkspaceFileDocHandlers, +} from '@/handlers/file-doc' type Handler = (payload?: unknown) => Promise | void @@ -173,6 +179,8 @@ describe('setupWorkspaceFileDocHandlers', () => { // Default: the server seed builder returns no content (empty file). Tests that // exercise seeding override this per-case with an encoded Yjs update. mockFetchFileDocSeed.mockResolvedValue(null) + // Default: the merge builder returns a no-op update. Tests exercising copilot merges override it. + mockFetchFileDocMerge.mockResolvedValue(new Uint8Array()) }) afterEach(() => { @@ -399,6 +407,39 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toContain('# Seeded') }) + it('merges a copilot edit into a seeded live room and relays it to editors', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# Original')) + const { io, sent } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() // seed lands → room is seeded/live + + // The app returns a diff (here, an update introducing text) for the relay to apply. + const diff = new Y.Doc() + diff.getText(FILE_DOC_FIELD).insert(0, 'copilot content') + mockFetchFileDocMerge.mockResolvedValue(Y.encodeStateAsUpdate(diff)) + sent.length = 0 + + const result = await applyMarkdownToLiveFileDoc('file-1', '# Rewritten by copilot') + + expect(result).toBe('applied') + expect(mockFetchFileDocMerge).toHaveBeenCalledWith( + 'file-1', + expect.any(Uint8Array), + '# Rewritten by copilot' + ) + // Applying the diff fires doc.on('update') → the merge is broadcast to the whole room. + expect(sent.some((m) => m.event === FILE_DOC_EVENTS.MESSAGE && m.target === ROOM_NAME)).toBe( + true + ) + }) + + it('reports no-live-room (and does not call the app) when the file has no seeded room', async () => { + const result = await applyMarkdownToLiveFileDoc('file-1', '# anything') + expect(result).toBe('no-live-room') + expect(mockFetchFileDocMerge).not.toHaveBeenCalled() + }) + it('relays a document update to the rest of the room, excluding the sender', async () => { const { io, sent } = createIo() const a = setup('socket-a', io) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 0dd8829c87f..446cc1f9643 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -36,7 +36,7 @@ import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' import { resolveAvatarUrl } from '@/handlers/avatar' -import { fetchFileDocSeed } from '@/handlers/file-doc-seed' +import { fetchFileDocMerge, fetchFileDocSeed } from '@/handlers/file-doc-app' import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -205,6 +205,32 @@ async function ensureServerSeed( } } +/** + * Apply new markdown into a file's LIVE collaborative document (Stage C — copilot writing into an open + * doc). Ships the document's current state to the app to build a minimal Yjs diff, applies it — which + * fires `doc.on('update')` and relays the merge to every connected editor, reconciled with any + * concurrent user edits — and reports whether it landed. + * + * Returns `'no-live-room'` when the file has no seeded, occupied room: an unseeded/empty doc has no + * authoritative content to merge against, so the caller (copilot) writes the file directly instead and + * the seed picks up the new content on the next open. + */ +export async function applyMarkdownToLiveFileDoc( + fileId: string, + markdown: string +): Promise<'applied' | 'no-live-room'> { + const name = roomName(fileDocRoom(fileId)) + const room = fileDocRooms.get(name) + if (!room || room.owners.size === 0 || !isDocSeeded(room.doc)) return 'no-live-room' + + const update = await fetchFileDocMerge(fileId, Y.encodeStateAsUpdate(room.doc), markdown) + // The room may have been dropped while the diff was being built; never touch a destroyed doc. + if (fileDocRooms.get(name) !== room) return 'no-live-room' + // No transaction origin → `doc.on('update')` relays to the WHOLE room (every editor sees copilot). + Y.applyUpdate(room.doc, update) + return 'applied' +} + /** * Get (or lazily create) the authoritative document for a room, wiring the two * relay handlers exactly once: document updates and awareness changes are diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 0b6264b8f8b..dcde36a0609 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -2,6 +2,7 @@ import type { IncomingMessage, ServerResponse } from 'http' import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { safeCompare } from '@sim/security/compare' import { env } from '@/env' +import { applyMarkdownToLiveFileDoc } from '@/handlers/file-doc' import { type IRoomManager, WorkflowRoomService } from '@/rooms' interface Logger { @@ -183,6 +184,26 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } + // Merge a copilot edit into a file's LIVE collaborative document so it streams into open editors + // (Stage C). Returns `{ applied }`: when false, no seeded live room exists and the caller writes + // the file directly instead. Any live user edits are preserved — the app builds a minimal CRDT diff. + if (req.method === 'POST' && req.url === '/api/file-doc/apply-edit') { + try { + const body = await readRequestBody(req) + const { fileId, markdown } = JSON.parse(body) + if (!isNonEmptyString(fileId) || typeof markdown !== 'string') { + return sendError(res, 'Invalid fileId or markdown', 400) + } + const result = await applyMarkdownToLiveFileDoc(fileId, markdown) + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ applied: result === 'applied' })) + } catch (error) { + logger.error('Error applying copilot edit to live file-doc:', error) + sendError(res, 'Failed to apply edit to live document') + } + return + } + res.writeHead(404, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ error: 'Not found' })) } From e43e9a1674fd1b19e01eb2a8eef40f81a9d4e7eb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 14:48:02 -0700 Subject: [PATCH 33/53] feat(collab-doc): stream copilot edits into open editors (Stage C) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit edit_content now, after its durable file write, best-effort merges the same markdown into the file's live collaborative document (markdown files only). If a collaborator has it open, the edit streams into their editor as a CRDT merge — reconciled with their concurrent typing — instead of the file silently changing under them; the editor's existing autosave mirrors the merged doc back to the file. No-op when nothing is open. Never blocks or fails the edit. --- .../tools/server/files/edit-content.ts | 9 ++++ apps/sim/lib/realtime/notify.test.ts | 41 +++++++++++++++++++ apps/sim/lib/realtime/notify.ts | 32 +++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 apps/sim/lib/realtime/notify.test.ts diff --git a/apps/sim/lib/copilot/tools/server/files/edit-content.ts b/apps/sim/lib/copilot/tools/server/files/edit-content.ts index f854f670853..7eac749a63f 100644 --- a/apps/sim/lib/copilot/tools/server/files/edit-content.ts +++ b/apps/sim/lib/copilot/tools/server/files/edit-content.ts @@ -6,6 +6,7 @@ import { type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' import { isDocSandboxEnabled } from '@/lib/core/config/env-flags' +import { mergeEditIntoLiveFileDoc } from '@/lib/realtime/notify' import { updateWorkspaceFileContent } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { getE2BDocFormat } from './doc-compile' import { buildEmbeddedImageRefWarning } from './embedded-image-refs' @@ -241,6 +242,14 @@ export const editContentServerTool: BaseServerTool ({ getSocketServerUrl: () => 'http://realtime' })) +vi.mock('@/lib/core/config/env', () => ({ env: { INTERNAL_API_SECRET: 'secret' } })) + +import { mergeEditIntoLiveFileDoc } from './notify' + +describe('mergeEditIntoLiveFileDoc', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('POSTs the edit to the realtime apply-edit endpoint with the api key', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true }) + vi.stubGlobal('fetch', fetchMock) + + await mergeEditIntoLiveFileDoc('file-1', '# hello') + + expect(fetchMock).toHaveBeenCalledWith( + 'http://realtime/api/file-doc/apply-edit', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ 'x-api-key': 'secret' }), + body: JSON.stringify({ fileId: 'file-1', markdown: '# hello' }), + }) + ) + }) + + it('never throws when the realtime call fails (best-effort)', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('socket pod down'))) + await expect(mergeEditIntoLiveFileDoc('file-1', '# hello')).resolves.toBeUndefined() + }) + + it('never throws on a non-2xx response', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, status: 503 })) + await expect(mergeEditIntoLiveFileDoc('file-1', '# hello')).resolves.toBeUndefined() + }) +}) diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts index 71ef65c1ced..4fb8afcb2b1 100644 --- a/apps/sim/lib/realtime/notify.ts +++ b/apps/sim/lib/realtime/notify.ts @@ -8,6 +8,12 @@ const logger = createLogger('RealtimeNotify') /** Bound the wait on the realtime server so a slow/hung socket pod can't stall a file mutation. */ const NOTIFY_TIMEOUT_MS = 2000 +/** + * Bound the wait on the live-doc merge. Larger than {@link NOTIFY_TIMEOUT_MS} because apply-edit is a + * round trip (relay → app `/merge` → relay); still tight, since the merge is a sub-second conversion. + */ +const APPLY_EDIT_TIMEOUT_MS = 4000 + /** * Best-effort fan-out to the realtime server that a workspace's file tree changed, * so every browser currently viewing that workspace's files refetches. File @@ -41,3 +47,29 @@ export async function notifyWorkspaceFilesChanged(workspaceId: string): Promise< }) } } + +/** + * Best-effort: ask the realtime relay to merge a copilot edit into a file's LIVE collaborative + * document, so open editors see it stream in as a CRDT merge (Stage C) rather than the file changing + * underneath them. No-op when no editor is connected (the relay reports `applied: false`). The file + * itself is written durably by the caller regardless — this only drives the live view — so a dropped + * merge merely means editors see the change on their next reload. Never throws. + * + * Awaited (not fire-and-forget) so the fetch dispatches before the route handler returns; bounded to + * {@link APPLY_EDIT_TIMEOUT_MS}, so it adds latency only when the socket pod is unreachable. + */ +export async function mergeEditIntoLiveFileDoc(fileId: string, markdown: string): Promise { + try { + const response = await fetch(`${getSocketServerUrl()}/api/file-doc/apply-edit`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ fileId, markdown }), + signal: AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS), + }) + if (!response.ok) { + logger.warn('file-doc apply-edit failed', { fileId, status: response.status }) + } + } catch (error) { + logger.warn('file-doc apply-edit error', { fileId, error: getErrorMessage(error) }) + } +} From 470b9d0459cafd2f9ea79adb1814d7a9fc82754a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 15:01:36 -0700 Subject: [PATCH 34/53] fix(collab-doc): strip frontmatter on merge; gate live-merge to markdown - buildFileDocMergeUpdate now strips YAML frontmatter (splitFrontmatter().body) exactly as the seed does. Copilot passes full-file content, so without this the frontmatter merged into the doc as editor content and autosave wrote it back over the file (corruption). - Gate the live-doc merge on isMarkdownFileName (new server-safe helper) instead of the over-broad !isDoc, so code/text edits don't pay the realtime round-trip for a format the collaborative editor never renders. --- apps/sim/lib/collab-doc/merge.test.ts | 12 ++++++++++++ apps/sim/lib/collab-doc/merge.ts | 6 +++++- .../copilot/tools/server/files/edit-content.ts | 8 +++++--- apps/sim/lib/uploads/utils/file-utils.test.ts | 16 ++++++++++++++++ apps/sim/lib/uploads/utils/file-utils.ts | 10 ++++++++++ 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/collab-doc/merge.test.ts b/apps/sim/lib/collab-doc/merge.test.ts index 92820ee8a60..58602f56235 100644 --- a/apps/sim/lib/collab-doc/merge.test.ts +++ b/apps/sim/lib/collab-doc/merge.test.ts @@ -15,6 +15,18 @@ describe('buildFileDocMergeUpdate', () => { expect(yDocToMarkdown(live)).toBe(serializeMarkdownBody('# Title\n\nRewritten.')) }) + it('strips frontmatter — merges only the body, never the YAML', () => { + const live = markdownToYDoc('# Title\n\nBody.') + const update = buildFileDocMergeUpdate( + Y.encodeStateAsUpdate(live), + '---\ntitle: X\n---\n\n# Title\n\nBody rewritten.' + ) + Y.applyUpdate(live, update) + const md = yDocToMarkdown(live) + expect(md).not.toContain('title: X') + expect(md).toBe(serializeMarkdownBody('# Title\n\nBody rewritten.')) + }) + it('returns an empty (no-op) diff when the markdown already matches', () => { const live = markdownToYDoc('# Same\n\nBody.') const before = Y.encodeStateVector(live) diff --git a/apps/sim/lib/collab-doc/merge.ts b/apps/sim/lib/collab-doc/merge.ts index 0715fb80f15..d6bd4369c1b 100644 --- a/apps/sim/lib/collab-doc/merge.ts +++ b/apps/sim/lib/collab-doc/merge.ts @@ -1,4 +1,5 @@ import * as Y from 'yjs' +import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { applyMarkdownToYDoc } from './converter' /** @@ -18,7 +19,10 @@ export function buildFileDocMergeUpdate(docState: Uint8Array, markdown: string): try { Y.applyUpdate(doc, docState) const before = Y.encodeStateVector(doc) - applyMarkdownToYDoc(doc, markdown) + // Strip frontmatter exactly as the seed does — the collaborative body never includes it. Callers + // pass full file content (copilot's `finalContent`), so merging it verbatim would inject the YAML + // frontmatter as editor content, which the editor's autosave would then write back over the file. + applyMarkdownToYDoc(doc, splitFrontmatter(markdown).body) return Y.encodeStateAsUpdate(doc, before) } finally { doc.destroy() diff --git a/apps/sim/lib/copilot/tools/server/files/edit-content.ts b/apps/sim/lib/copilot/tools/server/files/edit-content.ts index 7eac749a63f..b8e6e3fd91b 100644 --- a/apps/sim/lib/copilot/tools/server/files/edit-content.ts +++ b/apps/sim/lib/copilot/tools/server/files/edit-content.ts @@ -8,6 +8,7 @@ import { import { isDocSandboxEnabled } from '@/lib/core/config/env-flags' import { mergeEditIntoLiveFileDoc } from '@/lib/realtime/notify' import { updateWorkspaceFileContent } from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { isMarkdownFileName } from '@/lib/uploads/utils/file-utils' import { getE2BDocFormat } from './doc-compile' import { buildEmbeddedImageRefWarning } from './embedded-image-refs' import { consumeLatestFileIntent } from './file-intent-store' @@ -244,9 +245,10 @@ export const editContentServerTool: BaseServerTool { + it('is true for .md and .markdown (case-insensitive)', () => { + expect(isMarkdownFileName('notes.md')).toBe(true) + expect(isMarkdownFileName('README.MD')).toBe(true) + expect(isMarkdownFileName('doc.markdown')).toBe(true) + }) + + it('is false for non-markdown files', () => { + expect(isMarkdownFileName('script.js')).toBe(false) + expect(isMarkdownFileName('report.docx')).toBe(false) + expect(isMarkdownFileName('notes.txt')).toBe(false) + expect(isMarkdownFileName('noext')).toBe(false) + }) +}) + describe('extractStorageKey', () => { it('strips every provider serve prefix', () => { expect(extractStorageKey('/api/files/serve/s3/workspace%2Fws-1%2Ffile.txt')).toBe( diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index cd36052fdbe..4f255b1e816 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -210,6 +210,16 @@ export function getFileExtension(filename: string): string { return lastDot !== -1 ? filename.slice(lastDot + 1).toLowerCase() : '' } +/** + * Whether a file is markdown by extension (`.md` / `.markdown`) — the format that renders in the + * collaborative rich editor. Server-safe (no client `resolvePreviewType` dependency), for gating + * server work like the live-doc merge to files that can actually be open in that editor. + */ +export function isMarkdownFileName(filename: string): boolean { + const ext = getFileExtension(filename) + return ext === 'md' || ext === 'markdown' +} + /** * Extensions whose stored bytes may be a generation source that renders to a larger * binary. Everything else stores exactly what it serves, so its declared size is From a7a55c9010a5d26f1189fd260744a6d31150e080 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 15:17:55 -0700 Subject: [PATCH 35/53] fix(collab-doc): make frontmatter collaborative so a merge can't revert it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The editor re-attaches its open-time frontmatter on every autosave, so the Stage C merge (which triggers an autosave with no user action) could write stale YAML back over a copilot frontmatter change — silently dropping it. Carry the file's frontmatter in the doc's config map instead of locking it at open: the seed stores it, the merge updates it (only when it actually changed, preserving the no-op diff), and the editor re-attaches THAT value on save — falling back to the locked copy before the seed lands and for non-collaborative docs. A server-side frontmatter change is now reflected rather than reverted. New FILE_DOC_SEED.frontmatterKey; seed/merge tests cover it. --- .../rich-markdown-editor.tsx | 20 ++++++++++++++++--- apps/sim/lib/collab-doc/merge.test.ts | 8 +++++++- apps/sim/lib/collab-doc/merge.ts | 17 ++++++++++++---- apps/sim/lib/collab-doc/seed.test.ts | 12 +++++++++++ apps/sim/lib/collab-doc/seed.ts | 8 ++++++-- packages/realtime-protocol/src/file-doc.ts | 8 ++++++++ 6 files changed, 63 insertions(+), 10 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 5717ed80c6a..fb6331323de 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -1,8 +1,8 @@ 'use client' -import { memo, useEffect, useRef, useState } from 'react' +import { memo, useCallback, useEffect, useRef, useState } from 'react' import { cn, toast } from '@sim/emcn' -import type { JoinFileDocError } from '@sim/realtime-protocol/file-doc' +import { FILE_DOC_SEED, type JoinFileDocError } from '@sim/realtime-protocol/file-doc' import type { Extensions, JSONContent } from '@tiptap/core' import { isChangeOrigin } from '@tiptap/extension-collaboration' import { Fragment, Slice } from '@tiptap/pm/model' @@ -306,6 +306,20 @@ export function LoadedRichMarkdownEditor({ const onSaveShortcutRef = useRef(onSaveShortcut) onSaveShortcutRef.current = onSaveShortcut + /** + * The frontmatter to re-attach to the body on save. For a collaborative doc it lives in the CRDT + * (config map, seeded/updated server-side), so a server edit that changes it is reflected rather + * than reverted by this editor's stale open-time copy; falls back to the locked `settledRef` copy + * before the seed lands and for non-collaborative documents. + */ + const resolveSaveFrontmatter = useCallback((): string => { + const fromDoc = collaboration?.doc + .getMap(FILE_DOC_SEED.configMap) + .get(FILE_DOC_SEED.frontmatterKey) + if (typeof fromDoc === 'string') return fromDoc + return settledRef.current?.frontmatter ?? '' + }, [collaboration]) + /** * While the file is still unnamed, name it after its leading heading: `onDeriveTitleFromHeading` is * called (debounced) so the caller can rename the file, and `fileNameRef` lets the onUpdate handler @@ -573,7 +587,7 @@ export function LoadedRichMarkdownEditor({ onUpdate: ({ editor, transaction }) => { const md = postProcessSerializedMarkdown(editor.getMarkdown()) lastSyncedBodyRef.current = md - onChangeRef.current(applyFrontmatter(settledRef.current?.frontmatter ?? '', md)) + onChangeRef.current(applyFrontmatter(resolveSaveFrontmatter(), md)) // While the file is still untitled, name it after its leading heading once typing settles — but // only for the LOCAL user's own edits. `isChangeOrigin` is true for a remote Yjs change (a peer // typing); bail BEFORE touching the timer so a remote edit never cancels or reschedules the local diff --git a/apps/sim/lib/collab-doc/merge.test.ts b/apps/sim/lib/collab-doc/merge.test.ts index 58602f56235..bb4b28196c5 100644 --- a/apps/sim/lib/collab-doc/merge.test.ts +++ b/apps/sim/lib/collab-doc/merge.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment jsdom */ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' import { describe, expect, it } from 'vitest' import * as Y from 'yjs' import { serializeMarkdownBody } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' @@ -15,7 +16,7 @@ describe('buildFileDocMergeUpdate', () => { expect(yDocToMarkdown(live)).toBe(serializeMarkdownBody('# Title\n\nRewritten.')) }) - it('strips frontmatter — merges only the body, never the YAML', () => { + it('strips frontmatter from the body but stores it in the config for the editor to re-attach', () => { const live = markdownToYDoc('# Title\n\nBody.') const update = buildFileDocMergeUpdate( Y.encodeStateAsUpdate(live), @@ -25,6 +26,11 @@ describe('buildFileDocMergeUpdate', () => { const md = yDocToMarkdown(live) expect(md).not.toContain('title: X') expect(md).toBe(serializeMarkdownBody('# Title\n\nBody rewritten.')) + // The updated frontmatter is carried in the config map, so an open editor re-attaches the NEW + // value on autosave instead of reverting to its stale open-time copy. + expect(live.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.frontmatterKey)).toContain( + 'title: X' + ) }) it('returns an empty (no-op) diff when the markdown already matches', () => { diff --git a/apps/sim/lib/collab-doc/merge.ts b/apps/sim/lib/collab-doc/merge.ts index d6bd4369c1b..3db9c73365e 100644 --- a/apps/sim/lib/collab-doc/merge.ts +++ b/apps/sim/lib/collab-doc/merge.ts @@ -1,3 +1,4 @@ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' import * as Y from 'yjs' import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { applyMarkdownToYDoc } from './converter' @@ -19,10 +20,18 @@ export function buildFileDocMergeUpdate(docState: Uint8Array, markdown: string): try { Y.applyUpdate(doc, docState) const before = Y.encodeStateVector(doc) - // Strip frontmatter exactly as the seed does — the collaborative body never includes it. Callers - // pass full file content (copilot's `finalContent`), so merging it verbatim would inject the YAML - // frontmatter as editor content, which the editor's autosave would then write back over the file. - applyMarkdownToYDoc(doc, splitFrontmatter(markdown).body) + // The collaborative body never includes frontmatter (callers pass full file content, e.g. + // copilot's `finalContent`), so merge only the body, and update the frontmatter in the config map + // — the editor re-attaches THAT on autosave, so a frontmatter change is reflected rather than + // reverted by an open editor's stale, open-time copy. + const { frontmatter, body } = splitFrontmatter(markdown) + applyMarkdownToYDoc(doc, body) + // Only write when it actually changed (treating "unset" as "") so an unchanged edit stays a true + // no-op diff rather than churning the config map on every merge. + const config = doc.getMap(FILE_DOC_SEED.configMap) + if ((config.get(FILE_DOC_SEED.frontmatterKey) ?? '') !== frontmatter) { + config.set(FILE_DOC_SEED.frontmatterKey, frontmatter) + } return Y.encodeStateAsUpdate(doc, before) } finally { doc.destroy() diff --git a/apps/sim/lib/collab-doc/seed.test.ts b/apps/sim/lib/collab-doc/seed.test.ts index c3ba8bf9bcf..8dcd660bb90 100644 --- a/apps/sim/lib/collab-doc/seed.test.ts +++ b/apps/sim/lib/collab-doc/seed.test.ts @@ -60,6 +60,18 @@ describe('buildFileDocSeed', () => { expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) }) + it('carries the frontmatter in the config map (not the body)', async () => { + mockFetchBuffer.mockResolvedValue(Buffer.from('---\ntitle: X\n---\n\n# Body', 'utf-8')) + const seed = await buildFileDocSeed('ws-1', 'file-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.frontmatterKey)).toContain( + 'title: X' + ) + // …and the frontmatter is NOT in the collaborative body. + expect(yDocToMarkdown(doc)).not.toContain('title: X') + }) + it('returns null for a missing file', async () => { mockGetWorkspaceFile.mockResolvedValue(null) expect(await buildFileDocSeed('ws-1', 'missing')).toBeNull() diff --git a/apps/sim/lib/collab-doc/seed.ts b/apps/sim/lib/collab-doc/seed.ts index 38c37c690b1..d83653cc0da 100644 --- a/apps/sim/lib/collab-doc/seed.ts +++ b/apps/sim/lib/collab-doc/seed.ts @@ -38,14 +38,18 @@ export async function buildFileDocSeed( const buffer = await fetchWorkspaceFileBuffer(record, { maxBytes: MAX_SEED_BYTES }) const markdown = buffer.toString('utf-8') - const { body } = splitFrontmatter(markdown) + const { frontmatter, body } = splitFrontmatter(markdown) const ydoc = markdownToYDoc(body) try { + const config = ydoc.getMap(FILE_DOC_SEED.configMap) // Mark the document seeded IN the same doc, so the client's readiness gate // (`synced && initialContentLoaded === true`) recognizes a server-seeded doc without any // client-seeder handshake, and a stray re-election can never seed on top of it. - ydoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + config.set(FILE_DOC_SEED.flag, true) + // Carry the frontmatter in the doc (not the body) so it merges across clients and a later + // server-side edit can update it — the editor re-attaches this on autosave. + config.set(FILE_DOC_SEED.frontmatterKey, frontmatter) return { update: Y.encodeStateAsUpdate(ydoc) } } finally { ydoc.destroy() diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index 5590d7988b6..319e70ad0de 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -71,6 +71,14 @@ export type FileDocMessageType = (typeof FILE_DOC_MESSAGE_TYPE)[keyof typeof FIL export const FILE_DOC_SEED = { configMap: 'config', flag: 'initialContentLoaded', + /** + * The file's raw YAML frontmatter string, stored in the config map so it lives in the CRDT and + * merges across clients. The collaborative document body never contains frontmatter (it is stripped + * on seed); the editor re-attaches THIS value on autosave. Keeping it in the doc — rather than + * locked at open time — is what lets a server-side edit (e.g. copilot) that changes the frontmatter + * be reflected instead of reverted by an open editor's autosave. + */ + frontmatterKey: 'frontmatter', } as const /** Client → server join request. `fileId` is the `workspace_files.id`. */ From 43726ed7105974daeee4f38daa97a538361fac0e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 15:26:01 -0700 Subject: [PATCH 36/53] fix(collab-doc): re-sync draft on a frontmatter-only merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A server edit that changes only the frontmatter updates the config map but not the body fragment, so TipTap's onUpdate never fires — the autosave draft kept the stale open-time frontmatter, and an explicit save could revert the live change. Observe the config map and, on a frontmatter-only change, re-attach the new frontmatter to the current body and push a fresh draft (guarded on a synced body so it never races the seed's own onUpdate). --- .../rich-markdown-editor.tsx | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index fb6331323de..554f95ba73e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -691,9 +691,28 @@ export function LoadedRichMarkdownEditor({ if (error.retryable === false) seedFromLoaded() } + // A server edit that changes ONLY the frontmatter (e.g. copilot) updates the config map but not + // the body fragment, so TipTap's `onUpdate` never fires and the autosave draft would keep the + // stale open-time frontmatter — an explicit save could then revert the live change. Re-attach the + // new frontmatter to the current body and push a fresh draft whenever it changes on its own. + let lastFrontmatter = config.get(FILE_DOC_SEED.frontmatterKey) + const syncFrontmatter = () => { + const current = config.get(FILE_DOC_SEED.frontmatterKey) + if (current === lastFrontmatter) return + lastFrontmatter = current + // Null body ref ⇒ no body has synced yet (e.g. this fired before the seed's own `onUpdate`); + // that path re-attaches the frontmatter itself, so there is nothing to do here. + if (lastSyncedBodyRef.current !== null) { + onChangeRef.current( + applyFrontmatter(typeof current === 'string' ? current : '', lastSyncedBodyRef.current) + ) + } + } + provider.on('synced', report) provider.on('join-error', onJoinError) config.observe(report) + config.observe(syncFrontmatter) report() if (provider.joinError) onJoinError(provider.joinError) @@ -701,6 +720,7 @@ export function LoadedRichMarkdownEditor({ provider.off('synced', report) provider.off('join-error', onJoinError) config.unobserve(report) + config.unobserve(syncFrontmatter) // Report NOT ready on teardown — the safe direction. If this effect ever re-runs while mounted // (a future dep change), briefly gating autosave off is harmless; reporting `true` here could // ungate it while the doc is unready. From 6cf455ba84f6df947b544a152b9b0ce52e1580ab Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 15:39:47 -0700 Subject: [PATCH 37/53] fix(collab-doc): order the apply-edit/merge timeouts (outer > inner) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Sim->realtime apply-edit timeout (4s) was shorter than the nested realtime->Sim merge timeout (8s), so the outer call could abort while the relay was still merging — the relay then applied the merge after edit_content had returned, racing a follow-on edit. Split the shared realtime->app timeout: the seed keeps 8s (it reads a cold blob), the merge gets a tight 3s (it is a pure conversion, no I/O), and the outer apply-edit is raised to 6s so it always outlives the inner merge. Cross-referenced in comments to prevent drift. --- apps/realtime/src/handlers/file-doc-app.ts | 40 ++++++++++++++-------- apps/sim/lib/realtime/notify.ts | 9 +++-- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-app.ts b/apps/realtime/src/handlers/file-doc-app.ts index 7ec74f6a61d..34bfb068ce1 100644 --- a/apps/realtime/src/handlers/file-doc-app.ts +++ b/apps/realtime/src/handlers/file-doc-app.ts @@ -7,20 +7,28 @@ import { env, getBaseUrl } from '@/env' */ /** - * Bound each request so a slow/unreachable app never wedges the relay. 8s is generous for the - * underlying markdown↔Yjs conversion — collab is gated client-side to ≤256 KB documents - * (`isRoundTripSafe`), which convert in well under a second. The seed additionally must stay under the - * client's readiness deadline (`READINESS_DEADLINE_MS`, 12s, in `file-doc-provider.ts`), which this - * satisfies. + * The seed reads a (possibly cold) blob + converts it, so give it a generous bound. Collab is gated + * client-side to ≤256 KB documents (`isRoundTripSafe`), so the conversion itself is well under a + * second; the headroom is for the blob read. Stays under the client's readiness deadline + * (`READINESS_DEADLINE_MS`, 12s, in `file-doc-provider.ts`), which it must. */ -const APP_REQUEST_TIMEOUT_MS = 8_000 +const SEED_REQUEST_TIMEOUT_MS = 8_000 -function postToApp(path: string, payload: unknown): Promise { +/** + * The merge is a PURE conversion (the caller supplies both the doc state and the markdown — no blob or + * DB I/O), so it is fast and gets a tight bound. Critically it must stay BELOW the app→relay apply-edit + * timeout (`APPLY_EDIT_TIMEOUT_MS`, in `apps/sim/lib/realtime/notify.ts`) that wraps this call — else + * that outer request aborts while this inner one is still running, and the relay could apply a merge + * after the caller has already moved on, racing a follow-on edit. + */ +const MERGE_REQUEST_TIMEOUT_MS = 3_000 + +function postToApp(path: string, payload: unknown, timeoutMs: number): Promise { return fetch(`${getBaseUrl()}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, body: JSON.stringify(payload), - signal: AbortSignal.timeout(APP_REQUEST_TIMEOUT_MS), + signal: AbortSignal.timeout(timeoutMs), }) } @@ -34,7 +42,11 @@ export async function fetchFileDocSeed( workspaceId: string, fileId: string ): Promise { - const response = await postToApp('/api/internal/file-doc/seed', { workspaceId, fileId }) + const response = await postToApp( + '/api/internal/file-doc/seed', + { workspaceId, fileId }, + SEED_REQUEST_TIMEOUT_MS + ) if (!response.ok) { throw new Error(`Seed fetch failed for file ${fileId}: ${response.status}`) } @@ -61,11 +73,11 @@ export async function fetchFileDocMerge( docState: Uint8Array, markdown: string ): Promise { - const response = await postToApp('/api/internal/file-doc/merge', { - fileId, - docState: Buffer.from(docState).toString('base64'), - markdown, - }) + const response = await postToApp( + '/api/internal/file-doc/merge', + { fileId, docState: Buffer.from(docState).toString('base64'), markdown }, + MERGE_REQUEST_TIMEOUT_MS + ) if (!response.ok) { throw new Error(`Merge fetch failed for file ${fileId}: ${response.status}`) } diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts index 4fb8afcb2b1..67e0b7e99cd 100644 --- a/apps/sim/lib/realtime/notify.ts +++ b/apps/sim/lib/realtime/notify.ts @@ -9,10 +9,13 @@ const logger = createLogger('RealtimeNotify') const NOTIFY_TIMEOUT_MS = 2000 /** - * Bound the wait on the live-doc merge. Larger than {@link NOTIFY_TIMEOUT_MS} because apply-edit is a - * round trip (relay → app `/merge` → relay); still tight, since the merge is a sub-second conversion. + * Bound the wait on the live-doc merge. This OUTER call wraps the relay's inner relay→app `/merge` + * request (`MERGE_REQUEST_TIMEOUT_MS`, 3s, in `apps/realtime/src/handlers/file-doc-app.ts`), so it + * must stay comfortably ABOVE that — otherwise this aborts while the relay is still merging, and the + * relay could apply the merge after we've returned, racing a follow-on edit. 6s leaves the inner 3s + * plus the two network hops and the relay's own work. */ -const APPLY_EDIT_TIMEOUT_MS = 4000 +const APPLY_EDIT_TIMEOUT_MS = 6000 /** * Best-effort fan-out to the realtime server that a workspace's file tree changed, From 3c580f3e05979501f1132d3ea7cb49e49c91743e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 15:59:35 -0700 Subject: [PATCH 38/53] fix(collab-doc): address deep-audit findings (jsdom trace, timeouts, gate, races) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 4-agent LOC audit + precedent research (TipTap/Hocuspocus/Yjs docs confirm the core patterns are idiomatic) surfaced these real issues: - The /merge route also lazy-requires jsdom but was missing its outputFileTracingIncludes entry — a Docker/standalone build would 500 with MODULE_NOT_FOUND. Add it. - The four conversion timeouts encoded an ordering invariant (merge { return fetch(`${getBaseUrl()}${path}`, { method: 'POST', @@ -45,7 +30,7 @@ export async function fetchFileDocSeed( const response = await postToApp( '/api/internal/file-doc/seed', { workspaceId, fileId }, - SEED_REQUEST_TIMEOUT_MS + FILE_DOC_TIMEOUTS.seedRequestMs ) if (!response.ok) { throw new Error(`Seed fetch failed for file ${fileId}: ${response.status}`) @@ -76,7 +61,7 @@ export async function fetchFileDocMerge( const response = await postToApp( '/api/internal/file-doc/merge', { fileId, docState: Buffer.from(docState).toString('base64'), markdown }, - MERGE_REQUEST_TIMEOUT_MS + FILE_DOC_TIMEOUTS.mergeRequestMs ) if (!response.ok) { throw new Error(`Merge fetch failed for file ${fileId}: ${response.status}`) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index cc6bd85d527..e4545cf3080 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -179,8 +179,9 @@ describe('setupWorkspaceFileDocHandlers', () => { // Default: the server seed builder returns no content (empty file). Tests that // exercise seeding override this per-case with an encoded Yjs update. mockFetchFileDocSeed.mockResolvedValue(null) - // Default: the merge builder returns a no-op update. Tests exercising copilot merges override it. - mockFetchFileDocMerge.mockResolvedValue(new Uint8Array()) + // Default: the merge builder returns a valid no-op (empty-doc) update. Tests exercising copilot + // merges override it. + mockFetchFileDocMerge.mockResolvedValue(Y.encodeStateAsUpdate(new Y.Doc())) }) afterEach(() => { @@ -440,6 +441,32 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(mockFetchFileDocMerge).not.toHaveBeenCalled() }) + it('serializes concurrent merges for the same file (second waits for the first)', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# Original')) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + + // First merge is left in flight; the second must not start its own fetch until the first finishes. + const noOpUpdate = Y.encodeStateAsUpdate(new Y.Doc()) + let resolveFirst: (v: Uint8Array) => void = () => {} + mockFetchFileDocMerge + .mockReturnValueOnce(new Promise((resolve) => (resolveFirst = resolve))) + .mockResolvedValueOnce(noOpUpdate) + + const first = applyMarkdownToLiveFileDoc('file-1', '# One') + const second = applyMarkdownToLiveFileDoc('file-1', '# Two') + await flushMicrotasks() + expect(mockFetchFileDocMerge).toHaveBeenCalledTimes(1) // second is queued behind the first + + resolveFirst(noOpUpdate) + await first + await second + // Only after the first resolved did the second run — and it snapshotted the post-first state. + expect(mockFetchFileDocMerge).toHaveBeenCalledTimes(2) + }) + it('relays a document update to the rest of the room, excluding the sender', async () => { const { io, sent } = createIo() const a = setup('socket-a', io) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 446cc1f9643..039f326c07f 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -205,21 +205,47 @@ async function ensureServerSeed( } } +/** Serializes live merges per file so overlapping calls never race the same doc (see below). */ +const fileDocMergeChains = new Map>() + /** * Apply new markdown into a file's LIVE collaborative document (Stage C — copilot writing into an open * doc). Ships the document's current state to the app to build a minimal Yjs diff, applies it — which * fires `doc.on('update')` and relays the merge to every connected editor, reconciled with any * concurrent user edits — and reports whether it landed. * + * Merges for the same file are SERIALIZED: each waits for any in-flight merge on that doc, so it + * snapshots the already-updated state and its diff is computed against current content — two + * overlapping full-document rewrites can never each diff the same stale snapshot and apply out of + * order. (The relay is single-writer per file — Helm pins one replica — so an in-process chain + * suffices.) + * * Returns `'no-live-room'` when the file has no seeded, occupied room: an unseeded/empty doc has no * authoritative content to merge against, so the caller (copilot) writes the file directly instead and * the seed picks up the new content on the next open. */ -export async function applyMarkdownToLiveFileDoc( +export function applyMarkdownToLiveFileDoc( fileId: string, markdown: string ): Promise<'applied' | 'no-live-room'> { const name = roomName(fileDocRoom(fileId)) + const prior = fileDocMergeChains.get(name) ?? Promise.resolve() + // `.catch` so a failed prior merge doesn't reject this one — each merge is independent. + const run = prior.catch(() => {}).then(() => mergeMarkdownIntoRoom(name, fileId, markdown)) + fileDocMergeChains.set( + name, + run.finally(() => { + if (fileDocMergeChains.get(name) === run) fileDocMergeChains.delete(name) + }) + ) + return run +} + +async function mergeMarkdownIntoRoom( + name: string, + fileId: string, + markdown: string +): Promise<'applied' | 'no-live-room'> { const room = fileDocRooms.get(name) if (!room || room.owners.size === 0 || !isDocSeeded(room.doc)) return 'no-live-room' diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index eda4308ed56..6049616f2ba 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -2,6 +2,7 @@ import { FILE_DOC_EVENTS, FILE_DOC_MESSAGE_TYPE, FILE_DOC_SEED, + FILE_DOC_TIMEOUTS, type JoinFileDocError, type JoinFileDocSuccess, toFileDocBytes, @@ -34,9 +35,10 @@ interface FileDocProviderEvents { * On the deadline the provider latches fatal and surfaces a non-retryable `join-error` — the exact * path a fatal rejection uses — so the editor falls back to showing the file's stored content * read-only instead of a permanently blank pane. Generous enough to clear a slow connect + seed - * round-trip; a healthy cold open reaches readiness well within it. + * round-trip; a healthy cold open reaches readiness well within it. Shared with (and must exceed) the + * relay's seed-fetch timeout — see `FILE_DOC_TIMEOUTS` and its ordering test. */ -const READINESS_DEADLINE_MS = 12_000 +const READINESS_DEADLINE_MS = FILE_DOC_TIMEOUTS.readinessDeadlineMs /** * The client half of the collaborative file-document protocol: a Yjs provider diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 554f95ba73e..8b10de6a5ce 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -664,16 +664,16 @@ export function LoadedRichMarkdownEditor({ setReady(false) return } - const config = doc.getMap('config') + const config = doc.getMap(FILE_DOC_SEED.configMap) const seedFromLoaded = () => { - if (config.get('initialContentLoaded') === true) return + if (config.get(FILE_DOC_SEED.flag) === true) return doc.transact(() => { editor.commands.setContent( parseMarkdownToDoc(splitFrontmatter(seedContentRef.current).body), { contentType: 'json', emitUpdate: false } ) - config.set('initialContentLoaded', true) + config.set(FILE_DOC_SEED.flag, true) }) } @@ -686,7 +686,7 @@ export function LoadedRichMarkdownEditor({ // simply "synced AND seeded" — no client-side seed import on the happy path. `seedFromLoaded` // remains only for the offline fallback below (a non-retryable join error / readiness timeout), // where it locally renders the file read-only since the server can't be reached. - const report = () => setReady(provider.synced && config.get('initialContentLoaded') === true) + const report = () => setReady(provider.synced && config.get(FILE_DOC_SEED.flag) === true) const onJoinError = (error: JoinFileDocError) => { if (error.retryable === false) seedFromLoaded() } diff --git a/apps/sim/lib/api/contracts/file-doc.ts b/apps/sim/lib/api/contracts/file-doc.ts index 24d03100f53..efb98168694 100644 --- a/apps/sim/lib/api/contracts/file-doc.ts +++ b/apps/sim/lib/api/contracts/file-doc.ts @@ -35,10 +35,17 @@ export const buildFileDocSeedContract = defineRouteContract({ export const mergeFileDocBodySchema = z.object({ /** For logging/observability only — the merge itself is stateless. */ fileId: z.string().min(1, 'fileId is required'), - /** Base64-encoded `Y.encodeStateAsUpdate` of the live document as the relay currently holds it. */ - docState: z.string().min(1, 'docState is required'), + /** + * Base64-encoded `Y.encodeStateAsUpdate` of the live document as the relay currently holds it. The + * bound is generous headroom over a max-size collab doc's Yjs state (base64-inflated), not a tuned + * limit — collab is gated to ≤256 KB documents client-side. + */ + docState: z + .string() + .min(1, 'docState is required') + .max(16 * 1024 * 1024, 'docState is too large'), /** The full markdown body the caller (e.g. copilot) wants the document to become. */ - markdown: z.string(), + markdown: z.string().max(8 * 1024 * 1024, 'markdown is too large'), }) export type MergeFileDocBody = z.input diff --git a/apps/sim/lib/copilot/tools/server/files/edit-content.ts b/apps/sim/lib/copilot/tools/server/files/edit-content.ts index b8e6e3fd91b..79ae7f9c23d 100644 --- a/apps/sim/lib/copilot/tools/server/files/edit-content.ts +++ b/apps/sim/lib/copilot/tools/server/files/edit-content.ts @@ -8,7 +8,7 @@ import { import { isDocSandboxEnabled } from '@/lib/core/config/env-flags' import { mergeEditIntoLiveFileDoc } from '@/lib/realtime/notify' import { updateWorkspaceFileContent } from '@/lib/uploads/contexts/workspace/workspace-file-manager' -import { isMarkdownFileName } from '@/lib/uploads/utils/file-utils' +import { isMarkdownFile } from '@/lib/uploads/utils/file-utils' import { getE2BDocFormat } from './doc-compile' import { buildEmbeddedImageRefWarning } from './embedded-image-refs' import { consumeLatestFileIntent } from './file-intent-store' @@ -248,7 +248,7 @@ export const editContentServerTool: BaseServerTool { +describe('isMarkdownFile', () => { it('is true for .md and .markdown (case-insensitive)', () => { - expect(isMarkdownFileName('notes.md')).toBe(true) - expect(isMarkdownFileName('README.MD')).toBe(true) - expect(isMarkdownFileName('doc.markdown')).toBe(true) + expect(isMarkdownFile({ name: 'notes.md' })).toBe(true) + expect(isMarkdownFile({ name: 'README.MD' })).toBe(true) + expect(isMarkdownFile({ name: 'doc.markdown' })).toBe(true) + }) + + it('is true for a text/markdown MIME even without a .md name', () => { + expect(isMarkdownFile({ type: 'text/markdown', name: 'notes' })).toBe(true) + expect(isMarkdownFile({ type: 'text/markdown', name: 'doc.txt' })).toBe(true) }) it('is false for non-markdown files', () => { - expect(isMarkdownFileName('script.js')).toBe(false) - expect(isMarkdownFileName('report.docx')).toBe(false) - expect(isMarkdownFileName('notes.txt')).toBe(false) - expect(isMarkdownFileName('noext')).toBe(false) + expect(isMarkdownFile({ type: 'text/javascript', name: 'script.js' })).toBe(false) + expect(isMarkdownFile({ name: 'report.docx' })).toBe(false) + expect(isMarkdownFile({ type: 'text/plain', name: 'notes.txt' })).toBe(false) + expect(isMarkdownFile({ name: 'noext' })).toBe(false) }) }) diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index 4f255b1e816..9e331c10854 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -211,12 +211,15 @@ export function getFileExtension(filename: string): string { } /** - * Whether a file is markdown by extension (`.md` / `.markdown`) — the format that renders in the - * collaborative rich editor. Server-safe (no client `resolvePreviewType` dependency), for gating - * server work like the live-doc merge to files that can actually be open in that editor. + * Whether a file renders in the collaborative rich markdown editor. Server-safe counterpart to the + * client's `isMarkdownFile` (which uses `resolvePreviewType`): the editor treats a file as markdown by + * its `text/markdown` MIME *or* a `.md`/`.markdown` extension — MIME first, matching the client — so a + * `text/markdown` file with a non-`.md` name still counts. Used to gate server work (e.g. the live-doc + * merge) to exactly the files that can be open in that editor. */ -export function isMarkdownFileName(filename: string): boolean { - const ext = getFileExtension(filename) +export function isMarkdownFile(file: { type?: string | null; name: string }): boolean { + if (file.type === 'text/markdown') return true + const ext = getFileExtension(file.name) return ext === 'md' || ext === 'markdown' } diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index de05dcbb585..dec2577d8eb 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -138,10 +138,12 @@ const nextConfig: NextConfig = { ], outputFileTracingIncludes: { '/api/tools/stagehand/*': ['./node_modules/ws/**/*'], - // The seed endpoint's lazy `require('jsdom')` is invisible to the standalone file tracer, so - // force jsdom (and its transitive deps, followed from its static requires) into the trace — - // otherwise a Docker/standalone build omits it and the endpoint 500s with MODULE_NOT_FOUND. + // The seed and merge endpoints both lazily `require('jsdom')` (via the collab-doc converter), which + // is invisible to the standalone file tracer, so force jsdom (and its transitive deps, followed + // from its static requires) into the trace — otherwise a Docker/standalone build omits it and the + // endpoint 500s with MODULE_NOT_FOUND. '/api/internal/file-doc/seed': ['./node_modules/jsdom/**/*'], + '/api/internal/file-doc/merge': ['./node_modules/jsdom/**/*'], '/*': [ './node_modules/sharp/**/*', './node_modules/@img/**/*', diff --git a/packages/realtime-protocol/src/file-doc.test.ts b/packages/realtime-protocol/src/file-doc.test.ts new file mode 100644 index 00000000000..48fd7db5b8a --- /dev/null +++ b/packages/realtime-protocol/src/file-doc.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from 'vitest' +import { FILE_DOC_TIMEOUTS } from './file-doc' + +describe('FILE_DOC_TIMEOUTS ordering invariants', () => { + it('bounds every nested call below the one that wraps it', () => { + // The relay's inner `/merge` fetch must finish before the app's outer `apply-edit` call gives up, + // or the relay applies a merge after the caller has already returned (racing a follow-on edit). + expect(FILE_DOC_TIMEOUTS.mergeRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.applyEditMs) + // The relay's `/seed` fetch must finish before the client's readiness deadline lapses into its + // read-only fallback, or a late-but-successful seed can never reach the client. + expect(FILE_DOC_TIMEOUTS.seedRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.readinessDeadlineMs) + }) +}) diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index 319e70ad0de..bba9db511bb 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -81,6 +81,27 @@ export const FILE_DOC_SEED = { frontmatterKey: 'frontmatter', } as const +/** + * The timeouts for the file-doc conversion round-trips, in ONE place because two of the pairs form an + * ordering invariant that would otherwise live only in prose comments across two apps. A nested call's + * bound must exceed the bound of the call it wraps, or the outer aborts while the inner is still + * running — orphaning work that lands late (see the assertion in the accompanying test): + * + * - `applyEditMs` (app → relay `apply-edit`, in `apps/sim`) wraps `mergeRequestMs` (relay → app + * `/merge`, in `apps/realtime`), so `mergeRequestMs < applyEditMs`. + * - `readinessDeadlineMs` (the client's give-up-and-fall-back-read-only deadline) must outlast + * `seedRequestMs` (relay → app `/seed`), so `seedRequestMs < readinessDeadlineMs`. + * + * The seed request gets more headroom than the merge because it reads a (possibly cold) blob before + * converting; the merge is a pure in-memory conversion the caller fully supplies. + */ +export const FILE_DOC_TIMEOUTS = { + seedRequestMs: 8_000, + mergeRequestMs: 3_000, + applyEditMs: 6_000, + readinessDeadlineMs: 12_000, +} as const + /** Client → server join request. `fileId` is the `workspace_files.id`. */ export interface JoinFileDocPayload { fileId: string From c8ecfc06b243fe88ac298fc5ccc3564faaed35f2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 17:50:23 -0700 Subject: [PATCH 39/53] feat(collab-doc): multi-replica shared Yjs backend + server-side markdown persistence Make collaborative file-doc editing correct across multiple ECS tasks (the per-process Y.Doc previously assumed one replica per file). - Shared Yjs backend over Redis Streams (apps/realtime file-doc-store): each file's stream is the ordered, replayable log of updates; a multiplexed XREAD tailer converges every task's in-memory doc. Coordinated single-seeder election (SET NX + empty-stream recheck) fixes split-brain seeding. - Doc-sync fans out to local clients + the stream; awareness stays on the Socket.IO adapter. Snapshot+XTRIM compaction trims only integrated entries. - Server-side persistence: project the live doc back to markdown via a new /api/internal/file-doc/persist endpoint, debounced during editing and flushed on last-disconnect, from the authoritative stream state. Collaborative editors no longer client-autosave, closing the copilot clobber-window. - Copilot merges apply through the stream (reach the live doc on any task) and serialize cross-task via a Redis merge lock. - Degrades to the original single-replica behavior when REDIS_URL is unset. --- apps/realtime/src/handlers/file-doc-app.ts | 23 + .../src/handlers/file-doc-store.test.ts | 229 ++++++++++ apps/realtime/src/handlers/file-doc-store.ts | 419 ++++++++++++++++++ apps/realtime/src/handlers/file-doc.test.ts | 18 +- apps/realtime/src/handlers/file-doc.ts | 224 +++++++++- apps/realtime/src/index.ts | 11 + .../api/internal/file-doc/persist/route.ts | 43 ++ .../rich-markdown-editor.tsx | 16 +- apps/sim/lib/api/contracts/file-doc.ts | 42 ++ apps/sim/lib/collab-doc/converter.ts | 15 +- apps/sim/lib/collab-doc/persist.ts | 42 ++ apps/sim/lib/realtime/notify.ts | 15 +- apps/sim/next.config.ts | 9 +- packages/realtime-protocol/src/file-doc.ts | 6 + scripts/check-api-validation-contracts.ts | 4 +- 15 files changed, 1067 insertions(+), 49 deletions(-) create mode 100644 apps/realtime/src/handlers/file-doc-store.test.ts create mode 100644 apps/realtime/src/handlers/file-doc-store.ts create mode 100644 apps/sim/app/api/internal/file-doc/persist/route.ts create mode 100644 apps/sim/lib/collab-doc/persist.ts diff --git a/apps/realtime/src/handlers/file-doc-app.ts b/apps/realtime/src/handlers/file-doc-app.ts index 050b7432cc9..8223a5eef74 100644 --- a/apps/realtime/src/handlers/file-doc-app.ts +++ b/apps/realtime/src/handlers/file-doc-app.ts @@ -72,3 +72,26 @@ export async function fetchFileDocMerge( } return new Uint8Array(Buffer.from(body.update, 'base64')) } + +/** + * Ask the app to project a live collaborative document back to durable markdown and write it to the + * file (Yjs → markdown, through the exact editor engine). This is the server-authoritative durable + * path — called debounced while the doc is edited and when the last collaborator leaves — that + * replaces the editor's client autosave, so a server/copilot edit can't be clobbered by a stale + * keystroke. THROWS on a transport failure so the caller can log/retry on the next debounce. + */ +export async function fetchFileDocPersist( + workspaceId: string, + fileId: string, + userId: string, + docState: Uint8Array +): Promise { + const response = await postToApp( + '/api/internal/file-doc/persist', + { workspaceId, fileId, userId, docState: Buffer.from(docState).toString('base64') }, + FILE_DOC_TIMEOUTS.persistRequestMs + ) + if (!response.ok) { + throw new Error(`Persist failed for file ${fileId}: ${response.status}`) + } +} diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts new file mode 100644 index 00000000000..75faf7f89c4 --- /dev/null +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -0,0 +1,229 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' + +/** + * One shared in-memory Redis backing per test, so several {@link FileDocStore} instances (modelling + * several ECS tasks) all talk to the "same Redis". A minimal fake of just the stream/lock ops the + * store uses. + */ +interface Backing { + streams: Map }[]> + kv: Map + seq: number +} + +const state = vi.hoisted(() => ({ backing: null as Backing | null })) + +const seqOf = (id: string) => Number(id.split('-')[0]) + +function makeClient(): any { + const b = () => { + if (!state.backing) throw new Error('backing not initialized') + return state.backing + } + const client: any = { + connect: async () => {}, + quit: async () => {}, + on: () => client, + duplicate: () => makeClient(), + xAdd: async (key: string, _star: string, fields: Record) => { + const id = `${++b().seq}-0` + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { ...fields } }) + b().streams.set(key, arr) + return id + }, + xRange: async (key: string) => (b().streams.get(key) ?? []).map((e) => ({ ...e })), + xLen: async (key: string) => (b().streams.get(key) ?? []).length, + xTrim: async (key: string, _strategy: string, minid: string) => { + const arr = b().streams.get(key) ?? [] + b().streams.set( + key, + arr.filter((e) => seqOf(e.id) >= seqOf(minid)) + ) + }, + xRead: async (streams: { key: string; id: string }[]) => { + const res: { name: string; messages: { id: string; message: Record }[] }[] = + [] + for (const { key, id } of streams) { + const after = (b().streams.get(key) ?? []).filter((e) => seqOf(e.id) > seqOf(id)) + if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) + } + if (res.length) return res + await new Promise((r) => setTimeout(r, 5)) + return null + }, + set: async (key: string, val: string, opts?: { NX?: boolean }) => { + if (opts?.NX && b().kv.has(key)) return null + b().kv.set(key, val) + return 'OK' + }, + del: async (key: string) => { + b().kv.delete(key) + return 1 + }, + expire: async () => 1, + } + return client +} + +vi.mock('redis', () => ({ createClient: () => makeClient() })) + +import { FileDocStore } from '@/handlers/file-doc-store' + +const REDIS_URL = 'redis://fake' +const NAME = 'workspace-file-doc:file-1' + +function docWithText(text: string): Y.Doc { + const doc = new Y.Doc() + doc.getText('body').insert(0, text) + return doc +} + +/** The delta a doc emits when `text` is inserted — what the relay would `publish`. */ +function updateFor(text: string): Uint8Array { + const doc = docWithText(text) + const update = Y.encodeStateAsUpdate(doc) + doc.destroy() + return update +} + +let stores: FileDocStore[] = [] +async function newStore(): Promise { + const store = new FileDocStore(REDIS_URL) + await store.init() + stores.push(store) + return store +} + +describe('FileDocStore', () => { + beforeEach(() => { + state.backing = { streams: new Map(), kv: new Map(), seq: 0 } + stores = [] + }) + + afterEach(async () => { + await Promise.all(stores.map((s) => s.shutdown())) + }) + + it('elects exactly one seeder across tasks (no split-brain seed)', async () => { + const a = await newStore() + const b = await newStore() + const [aWon, bWon] = await Promise.all([a.shouldSeed(NAME), b.shouldSeed(NAME)]) + expect([aWon, bWon].filter(Boolean)).toHaveLength(1) + }) + + it('does not re-seed once the stream already has content (stale lock)', async () => { + const a = await newStore() + expect(await a.shouldSeed(NAME)).toBe(true) + // A seeds and releases its lock. + a.publish(NAME, updateFor('hello')) + await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) + await a.releaseSeedLock(NAME) + // A different task must NOT seed again — the lock is free but the stream is non-empty. + const b = await newStore() + expect(await b.shouldSeed(NAME)).toBe(false) + }) + + it('getStreamState reconstructs the shared document from the stream', async () => { + const a = await newStore() + a.publish(NAME, updateFor('shared content')) + let state: Uint8Array | null = null + await vi.waitFor(async () => { + state = await a.getStreamState(NAME) + expect(state).not.toBeNull() + }) + const doc = new Y.Doc() + Y.applyUpdate(doc, state!) + expect(doc.getText('body').toString()).toBe('shared content') + doc.destroy() + }) + + it('attachRoom catches a fresh task up to the current shared state', async () => { + const a = await newStore() + a.publish(NAME, updateFor('already here')) + await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) + + // A second task opens the same file: its doc must load the existing content, not start empty. + const b = await newStore() + const doc = new Y.Doc() + await b.attachRoom(NAME, doc) + expect(doc.getText('body').toString()).toBe('already here') + doc.destroy() + }) + + it('converges a peer task via the tailer after attach', async () => { + const a = await newStore() + const b = await newStore() + const bDoc = new Y.Doc() + await b.attachRoom(NAME, bDoc) + + // A publishes an edit; B's multiplexed reader must apply it to B's attached doc. + a.publish(NAME, updateFor('from task A')) + await vi.waitFor(() => expect(bDoc.getText('body').toString()).toBe('from task A'), { + timeout: 2000, + }) + bDoc.destroy() + }) + + it('compaction never trims peer entries the compacting task has not yet integrated', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(Y.encodeStateAsUpdate(new Y.Doc())).toString('base64') + + // Two peer edits published by ANOTHER task that this task's tailer has not read yet. + const peerDoc = new Y.Doc() + const peerUpdates: Uint8Array[] = [] + peerDoc.on('update', (u: Uint8Array) => peerUpdates.push(u)) + peerDoc.getText('body').insert(0, 'PEER1') + peerDoc.getText('body').insert(5, 'PEER2') + + // Backing: 400 already-integrated (no-op) entries this task's doc reflects, then the 2 un-integrated + // peer entries. Enough entries to cross COMPACT_THRESHOLD. + const entries = Array.from({ length: 400 }, (_, i) => ({ + id: `${i + 1}-0`, + message: { u: noop }, + })) + entries.push({ id: '401-0', message: { u: Buffer.from(peerUpdates[0]).toString('base64') } }) + entries.push({ id: '402-0', message: { u: Buffer.from(peerUpdates[1]).toString('base64') } }) + state.backing!.streams.set(streamKey, entries) + state.backing!.seq = 402 + + const a = await newStore() + // This task has integrated only up to entry 400 (all no-ops) — its local doc is empty and lags the + // two peer entries. Inject that lagging room directly. + ;(a as any).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0 }) + await (a as any).maybeCompact(NAME) + + // A fresh catch-up must still reconstruct the peer content — compaction must not have trimmed 401/402. + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(doc.getText('body').toString()).toBe('PEER1PEER2') + doc.destroy() + }) + + it('serializes merges across tasks via the merge lock', async () => { + const a = await newStore() + const b = await newStore() + expect(await a.acquireMergeSlot(NAME, 5_000)).toBe(true) + // A holds it → B is refused until A releases. + expect(await b.acquireMergeSlot(NAME, 5_000)).toBe(false) + await a.releaseMergeSlot(NAME) + expect(await b.acquireMergeSlot(NAME, 5_000)).toBe(true) + await b.releaseMergeSlot(NAME) + }) + + it('is disabled without a REDIS_URL and behaves single-replica', async () => { + const store = new FileDocStore(undefined) + expect(store.enabled).toBe(false) + // Seeds locally (returns true), never touches a stream. + expect(await store.shouldSeed(NAME)).toBe(true) + expect(await store.getStreamState(NAME)).toBeNull() + const doc = new Y.Doc() + await store.attachRoom(NAME, doc) // no-op, no throw + expect(doc.getText('body').toString()).toBe('') + doc.destroy() + }) +}) diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts new file mode 100644 index 00000000000..aafcf0a7890 --- /dev/null +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -0,0 +1,419 @@ +/** + * Shared, multi-replica Yjs backend for the collaborative file-document relay, over Redis Streams. + * + * The relay keeps an in-memory {@link Y.Doc} per open file (for the sync handshake, awareness, and + * copilot merges), but on a horizontally-scaled deployment (multiple ECS tasks, autoscaling) that + * per-process doc is NOT authoritative on its own: two tasks each seeding the same file from markdown + * would mint independent Yjs client ids and union into duplicated content (split-brain), and a task + * only ever sees the edits of ITS OWN clients. This module makes every task converge on ONE CRDT per + * file by treating a Redis Stream as the shared, ordered, replayable log of Yjs updates — the union of + * a stream's entries IS the document. It is the "shared Yjs backend (y-redis / Hocuspocus)" the relay's + * single-replica model always deferred, built natively for our Socket.IO transport on the Redis the + * Socket.IO adapter already runs. + * + * How it fits the relay's message flow (see `file-doc.ts`): + * - Doc-sync messages no longer ride the Socket.IO Redis ADAPTER cross-pod. Instead each applied + * update is {@link publish}ed to the stream; every task's multiplexed reader + * applies it to its local doc (origin {@link REDIS_ORIGIN}) and fans it out to ITS OWN clients. So a + * client receives each update exactly once, from its own task's local broadcast — no adapter + * amplification, and every task's doc stays converged. (Awareness/presence stay on the adapter: they + * are ephemeral and need no convergence or replay.) + * - {@link attachRoom} does a synchronous catch-up read from the head of the stream when a task first + * opens a file, so a late-joining task (the normal case under autoscaling) loads the current shared + * state before its first client syncs. Catch-up + tail are seamless: the tailer resumes from the + * exact id catch-up stopped at. + * - {@link shouldSeed} coordinates the one-time seed with a Redis lock AND an empty-stream check, so + * exactly one task ever writes the seed cluster-wide (the fix for split-brain). + * + * When `REDIS_URL` is unset (single-pod dev) the store is DISABLED and every method degrades to the + * relay's original single-replica behavior: seed locally, no stream, no tailer. + * + * @module + */ +import { createLogger } from '@sim/logger' +import { FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { createClient, type RedisClientType } from 'redis' +import * as Y from 'yjs' + +const logger = createLogger('FileDocStore') + +/** + * The transaction origin the store stamps on updates it applies from the stream. The relay's + * `doc.on('update')` handler uses it to distinguish an update that ARRIVED from a peer (fan out to + * local clients, but do NOT re-publish — it is already in the stream) from a local edit (fan out AND + * publish). It must be a non-string sentinel so it is never mistaken for a socket id. + */ +export const REDIS_ORIGIN = Symbol('file-doc-redis') + +const STREAM_PREFIX = 'filedoc:stream:' +const SEED_LOCK_PREFIX = 'filedoc:seedlock:' +const COMPACT_LOCK_PREFIX = 'filedoc:compactlock:' +const PERSIST_LOCK_PREFIX = 'filedoc:persistlock:' +const MERGE_LOCK_PREFIX = 'filedoc:mergelock:' + +/** The single field each stream entry carries — a base64 Yjs update. */ +const UPDATE_FIELD = 'u' + +/** How long a blocking multiplexed read waits before re-snapshotting the live room set. Also bounds + * how long a room attached mid-block waits for its first cross-task update (updates are not lost — the + * next read resumes from its last id — only briefly delayed). */ +const READ_BLOCK_MS = 1_000 +/** Idle poll cadence when NO room is open on this task, so a freshly-attached room is picked up fast + * without busy-spinning an empty task. */ +const IDLE_POLL_MS = 250 +/** Max entries drained per stream per read. */ +const READ_COUNT = 200 +/** Compact a stream once it exceeds this many entries (snapshot + trim). */ +const COMPACT_THRESHOLD = 400 +/** Check whether compaction is due only every Nth local publish, to avoid an XLEN per keystroke. */ +const COMPACT_CHECK_EVERY = 64 +/** The seed lock is held across the app seed fetch (hard-bounded at `seedRequestMs`) plus the apply + + * publish. The generous cushion over `seedRequestMs` means only a multi-second event-loop stall — not + * ordinary latency — could expire it mid-seed, so the single-seeder invariant does not hinge on a tight + * 2s margin coupled to an unrelated timeout. */ +const SEED_LOCK_TTL_MS = FILE_DOC_TIMEOUTS.seedRequestMs + 12_000 +/** How long a stream survives with no heartbeat — long enough that an occupied-but-idle doc never + * loses its shared state (the heartbeat refreshes it while any task holds the room). */ +const STREAM_TTL_SEC = 600 +/** Refresh every occupied stream's TTL on this cadence, so a live doc's stream never expires. */ +const HEARTBEAT_MS = 60_000 + +const streamKey = (name: string) => `${STREAM_PREFIX}${name}` + +/** + * Decode one stream entry's base64 Yjs update and apply it to `doc`. A malformed entry is logged and + * SKIPPED — never thrown — so one bad frame can neither wedge the tailer nor abort a headless + * stream-fold. Shared by the tailer/catch-up (applies with {@link REDIS_ORIGIN}) and the merge-base + * reconstruction (no origin — a throwaway doc), so the two can never diverge on how an entry is read. + */ +function applyEntryToDoc( + doc: Y.Doc, + id: string, + message: Record, + origin?: unknown +): void { + const encoded = message[UPDATE_FIELD] + if (!encoded) return + try { + Y.applyUpdate(doc, new Uint8Array(Buffer.from(encoded, 'base64')), origin) + } catch (error) { + logger.warn('FileDocStore dropping malformed stream entry', { + id, + error: getErrorMessage(error), + }) + } +} + +/** One locally-open room the store tracks: its doc and the last stream id applied to it. */ +interface StoreRoom { + doc: Y.Doc + /** The id of the last stream entry applied to `doc`; the tailer resumes strictly after it. */ + lastId: string + /** Local publish count, to pace compaction checks. */ + publishes: number +} + +/** + * The Redis-Streams shared Yjs backend. A single instance per process. `enabled` is false when there + * is no `REDIS_URL`, in which case every method is a no-op and the relay runs single-replica. + */ +export class FileDocStore { + readonly enabled: boolean + /** Command connection: XADD / locks / XLEN / XTRIM / EXPIRE. */ + private write: RedisClientType | null = null + /** Dedicated connection for blocking XREAD (a blocking command monopolizes its connection). */ + private read: RedisClientType | null = null + private readonly rooms = new Map() + private running = false + private heartbeat: ReturnType | null = null + + constructor(private readonly redisUrl: string | undefined) { + this.enabled = Boolean(redisUrl) + } + + /** Connect the two Redis clients and start the multiplexed reader + TTL heartbeat. Idempotent. */ + async init(): Promise { + if (!this.enabled || this.running || !this.redisUrl) return + const options = { + url: this.redisUrl, + socket: { + reconnectStrategy: (retries: number) => { + if (retries > 10) return new Error('FileDocStore Redis reconnection failed') + return Math.min(retries * 100, 3000) + }, + }, + } + this.write = createClient(options) + this.read = this.write.duplicate() + this.write.on('error', (err) => logger.error('FileDocStore write client error:', err)) + this.read.on('error', (err) => logger.error('FileDocStore read client error:', err)) + await Promise.all([this.write.connect(), this.read.connect()]) + this.running = true + void this.runReader() + this.heartbeat = setInterval(() => void this.refreshTtls(), HEARTBEAT_MS) + logger.info('FileDocStore ready — shared Yjs backend over Redis Streams enabled') + } + + /** Stop the reader/heartbeat and close both clients. */ + async shutdown(): Promise { + this.running = false + if (this.heartbeat) clearInterval(this.heartbeat) + this.heartbeat = null + await Promise.all([this.write?.quit().catch(() => {}), this.read?.quit().catch(() => {})]) + this.write = null + this.read = null + } + + /** + * Register a locally-opened room and load the shared state into its doc: read the whole stream from + * the head, apply every entry (origin {@link REDIS_ORIGIN}), and remember the last id so the tailer + * resumes exactly after it. A brand-new file has an empty stream and loads nothing (it is seeded + * shortly after, via {@link shouldSeed}). No-op when disabled. + */ + async attachRoom(name: string, doc: Y.Doc): Promise { + if (!this.enabled || !this.write) return + // Register BEFORE the async read so a concurrent publish/tailer for this room can't be missed — + // the tailer resumes from `lastId`, which the catch-up advances. + const room: StoreRoom = { doc, lastId: '0', publishes: 0 } + this.rooms.set(name, room) + try { + const entries = await this.write.xRange(streamKey(name), '-', '+') + for (const entry of entries) { + // The room can be detached + its doc destroyed while catch-up is in flight (a fast open→close); + // stop touching it the moment that happens. + if (this.rooms.get(name) !== room) return + this.applyEntry(room, entry.id, entry.message) + } + await this.write.expire(streamKey(name), STREAM_TTL_SEC) + } catch (error) { + logger.warn(`FileDocStore catch-up failed for ${name}`, { error: getErrorMessage(error) }) + } + } + + /** Deregister a room the relay is destroying, so the tailer stops touching its (about-to-be-destroyed) doc. */ + detachRoom(name: string): void { + this.rooms.delete(name) + } + + /** + * Append a locally-applied update to the shared stream so every task converges. Called from the + * relay's `doc.on('update')` for local edits only (never for {@link REDIS_ORIGIN} updates — those + * already came from the stream). No-op when disabled. + */ + publish(name: string, update: Uint8Array): void { + if (!this.enabled || !this.write) return + const room = this.rooms.get(name) + void this.write + .xAdd(streamKey(name), '*', { [UPDATE_FIELD]: Buffer.from(update).toString('base64') }) + .then(() => this.write?.expire(streamKey(name), STREAM_TTL_SEC)) + .then(() => { + if (room && ++room.publishes % COMPACT_CHECK_EVERY === 0) return this.maybeCompact(name) + }) + .catch((error) => + logger.warn(`FileDocStore publish failed for ${name}`, { error: getErrorMessage(error) }) + ) + } + + /** + * Decide whether THIS task should build and write the file's one-time seed. Returns true only when + * the shared stream is genuinely empty AND this task wins the seed lock — so exactly one task across + * the cluster ever seeds a file, even if several open it at once (the fix for split-brain seeding). + * When disabled, always true (single-replica: seed locally). + */ + async shouldSeed(name: string): Promise { + if (!this.enabled || !this.write) return true + try { + const won = await this.write.set(`${SEED_LOCK_PREFIX}${name}`, '1', { + NX: true, + PX: SEED_LOCK_TTL_MS, + }) + if (won !== 'OK') return false + // The lock could be free yet the stream already seeded (a prior holder seeded then its lock + // expired). Re-check under the lock so we never write a SECOND seed on top of an existing one. + if ((await this.write.xLen(streamKey(name))) > 0) { + await this.releaseSeedLock(name) + return false + } + return true + } catch (error) { + logger.warn(`FileDocStore shouldSeed failed for ${name}`, { error: getErrorMessage(error) }) + return false + } + } + + /** + * Build the file's current shared state from the stream, headless (no registered room), for a merge + * that must reach the live doc regardless of which task holds it. Returns the encoded Yjs state, or + * `null` when the stream is empty — i.e. no doc is (or was recently) live, so there is nothing to + * merge into and the caller should fall back to a direct file write. Disabled → always null. + */ + async getStreamState(name: string): Promise { + if (!this.enabled || !this.write) return null + const entries = await this.write.xRange(streamKey(name), '-', '+') + if (entries.length === 0) return null + const doc = new Y.Doc() + try { + for (const entry of entries) applyEntryToDoc(doc, entry.id, entry.message) + return Y.encodeStateAsUpdate(doc) + } finally { + doc.destroy() + } + } + + /** Release the seed lock (best-effort) once the seed has been published or a seed attempt failed. */ + async releaseSeedLock(name: string): Promise { + if (!this.enabled || !this.write) return + await this.write.del(`${SEED_LOCK_PREFIX}${name}`).catch(() => {}) + } + + /** + * Try to claim the right to persist this file for the current debounce window, so concurrent tasks + * editing the same file don't each write a redundant blob version. Returns true (proceed) when + * disabled, or when this task wins a short lock. The final last-collaborator flush does NOT gate on + * this — it must always write. + */ + async acquirePersistSlot(name: string, ttlMs: number): Promise { + if (!this.enabled || !this.write) return true + try { + const won = await this.write.set(`${PERSIST_LOCK_PREFIX}${name}`, '1', { + NX: true, + PX: ttlMs, + }) + return won === 'OK' + } catch { + return true + } + } + + /** + * Try to claim the cross-task right to merge new content into this file. The relay already serializes + * merges per task; this extends that across tasks so two copilot edits to the same file landing on + * different tasks don't each diff the SAME shared base and publish conflicting full-document rewrites. + * The loser waits and retries so it diffs against the winner's RESULT (correct sequential merge). + * Returns true (proceed) when disabled or once the lock is won. Release with {@link releaseMergeSlot}. + */ + async acquireMergeSlot(name: string, ttlMs: number): Promise { + if (!this.enabled || !this.write) return true + try { + return ( + (await this.write.set(`${MERGE_LOCK_PREFIX}${name}`, '1', { NX: true, PX: ttlMs })) === 'OK' + ) + } catch { + return true + } + } + + async releaseMergeSlot(name: string): Promise { + if (!this.enabled || !this.write) return + await this.write.del(`${MERGE_LOCK_PREFIX}${name}`).catch(() => {}) + } + + private applyEntry(room: StoreRoom, id: string, message: Record): void { + room.lastId = id + applyEntryToDoc(room.doc, id, message, REDIS_ORIGIN) + } + + /** + * The single multiplexed tail loop: block-read every locally-open room's stream from its last id and + * apply new entries. One blocking connection for the whole process regardless of open-file count. + */ + private async runReader(): Promise { + while (this.running && this.read) { + const snapshot = [...this.rooms.entries()] + if (snapshot.length === 0) { + await sleep(IDLE_POLL_MS) + continue + } + try { + const res = await this.read.xRead( + snapshot.map(([name, room]) => ({ key: streamKey(name), id: room.lastId })), + { BLOCK: READ_BLOCK_MS, COUNT: READ_COUNT } + ) + if (!res) continue + for (const stream of res) { + const name = stream.name.slice(STREAM_PREFIX.length) + const room = this.rooms.get(name) + if (!room) continue // detached mid-read; its doc is being destroyed + for (const entry of stream.messages) this.applyEntry(room, entry.id, entry.message) + } + } catch (error) { + if (!this.running) break + logger.warn('FileDocStore reader error; retrying', { error: getErrorMessage(error) }) + await sleep(500) + } + } + } + + /** + * Snapshot-then-trim compaction: append a full-state snapshot and drop the older deltas it subsumes, + * so the stream stays bounded while a fresh task can still catch up from the head. Lock-guarded so + * only one task compacts a given stream at a time (concurrent snapshot+trim would race). Trims only up + * to what the snapshot provably contains — never un-integrated peer entries (see below). + */ + private async maybeCompact(name: string): Promise { + if (!this.write) return + const room = this.rooms.get(name) + if (!room) return + try { + if ((await this.write.xLen(streamKey(name))) < COMPACT_THRESHOLD) return + const won = await this.write.set(`${COMPACT_LOCK_PREFIX}${name}`, '1', { + NX: true, + PX: 10_000, + }) + if (won !== 'OK') return + try { + // Capture the snapshot AND the id it covers in one synchronous step (no await between): the + // snapshot is `room.doc`, which holds exactly what this task's tailer has integrated — every + // entry up to `room.lastId`. Entries a peer task published AFTER that (id > lastId) are NOT in + // the snapshot and this task's blocking reader may not have seen them yet, so we must NOT trim + // them — only entries the snapshot provably subsumes (id <= lastId). Trimming to the freshly + // appended snapshot id instead would silently drop those un-integrated peer entries. + const upTo = room.lastId + const snapshot = Buffer.from(Y.encodeStateAsUpdate(room.doc)).toString('base64') + await this.write.xAdd(streamKey(name), '*', { [UPDATE_FIELD]: snapshot }) + // MINID keeps entries with id >= upTo: the snapshot, any un-integrated peer entries, and + // `upTo` itself (redundant with the snapshot, harmless); it drops only the folded older deltas. + await this.write.xTrim(streamKey(name), 'MINID', upTo) + } finally { + await this.write.del(`${COMPACT_LOCK_PREFIX}${name}`).catch(() => {}) + } + } catch (error) { + logger.warn(`FileDocStore compaction failed for ${name}`, { error: getErrorMessage(error) }) + } + } + + private async refreshTtls(): Promise { + if (!this.write) return + for (const name of this.rooms.keys()) { + await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + } + } +} + +let store: FileDocStore | null = null + +/** + * Initialize the process-wide store from the realtime server bootstrap (alongside the socket adapter). + * Authoritative: if a disabled placeholder was lazily created by an early {@link getFileDocStore} call, + * this REPLACES it with the real, connected store — so the bootstrap can never silently no-op. A second + * call once already initialized is a no-op. + */ +export async function initFileDocStore(redisUrl: string | undefined): Promise { + if (store?.enabled) return store + store = new FileDocStore(redisUrl) + await store.init() + return store +} + +/** The process-wide store. Returns a disabled instance if init was never called (e.g. in unit tests). */ +export function getFileDocStore(): FileDocStore { + if (!store) store = new FileDocStore(undefined) + return store +} + +/** Test-only: reset the singleton so a test can install its own instance. */ +export function __setFileDocStoreForTest(next: FileDocStore | null): void { + store = next +} diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index e4545cf3080..d5954b7d5f5 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -14,11 +14,13 @@ import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' import type { IRoomManager } from '@/rooms' -const { mockAuthorizeRoom, mockFetchFileDocSeed, mockFetchFileDocMerge } = vi.hoisted(() => ({ - mockAuthorizeRoom: vi.fn(), - mockFetchFileDocSeed: vi.fn(), - mockFetchFileDocMerge: vi.fn(), -})) +const { mockAuthorizeRoom, mockFetchFileDocSeed, mockFetchFileDocMerge, mockFetchFileDocPersist } = + vi.hoisted(() => ({ + mockAuthorizeRoom: vi.fn(), + mockFetchFileDocSeed: vi.fn(), + mockFetchFileDocMerge: vi.fn(), + mockFetchFileDocPersist: vi.fn(), + })) vi.mock('@sim/platform-authz/rooms', () => ({ authorizeRoom: mockAuthorizeRoom, @@ -27,6 +29,7 @@ vi.mock('@sim/platform-authz/rooms', () => ({ vi.mock('@/handlers/file-doc-app', () => ({ fetchFileDocSeed: mockFetchFileDocSeed, fetchFileDocMerge: mockFetchFileDocMerge, + fetchFileDocPersist: mockFetchFileDocPersist, })) import { @@ -63,7 +66,10 @@ function createIo() { left.push({ socketId, room }) }, })) - return { io: { to, in: inFn } as unknown as IRoomManager['io'], sent, left } + // Doc-sync frames fan out via `io.local.to(...)` (cross-task delivery rides the Redis stream, not the + // adapter). With the store disabled in tests, `local` is the whole room — mirror `to` so those emits + // are recorded identically. Awareness/presence still use `io.to(...)`. + return { io: { to, in: inFn, local: { to } } as unknown as IRoomManager['io'], sent, left } } /** Every socket id a test created, so `afterEach` can drop their rooms without a diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 039f326c07f..e4ff1823f0d 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -6,15 +6,20 @@ * and the room abstraction, rather than a separate ws server. Clients speak the * `y-protocols` sync + awareness protocols; the server applies and relays them. * - * No durable Yjs state is kept yet: the document lives only while at least one - * collaborator is connected, and is re-seeded from the file's stored markdown on - * the next cold open (the markdown, saved by a client through the content API, is - * the durable source of truth). Durable Yjs snapshots are a separate follow-up. + * Multi-replica safe. The in-memory {@link Y.Doc} is NOT authoritative on its own — every task + * converges on one CRDT per file through the shared Redis-Streams backend in {@link file-doc-store}: + * each applied update is published to the file's stream and every task's tailer applies it to its own + * doc and fans it out to its own clients, so two tasks can never split-brain. Consequently doc-sync + * messages are broadcast LOCALLY ({@link io.local}) and cross-task delivery rides the stream — NOT the + * Socket.IO adapter (that would double-deliver). Awareness/presence stay on the adapter: they are + * ephemeral and need neither convergence nor replay. When `REDIS_URL` is unset the store is disabled + * and this falls back to the original single-replica behavior (local doc, local seed, local fan-out). * - * Single-writer assumption: the authoritative {@link Y.Doc} is held in this - * process's memory, so correctness assumes one realtime replica per file (Helm - * pins `realtime.replicaCount: 1`). Horizontal scaling would need a shared Yjs - * backend (y-redis / Hocuspocus) — out of scope here. + * Durability: the live doc is projected back to the file's markdown server-side — debounced while it is + * edited and flushed when the last collaborator leaves — via the app's `/persist` endpoint (the app + * owns the conversion engine). This replaces the editor's client autosave, closing the copilot + * clobber-window, and the Redis stream is the crash buffer between flushes. The markdown file remains + * the long-term source of truth; the Redis stream is ephemeral (TTL'd, heartbeat-refreshed while live). * * @module */ @@ -23,12 +28,15 @@ import { FILE_DOC_EVENTS, FILE_DOC_MESSAGE_TYPE, FILE_DOC_SEED, + FILE_DOC_TIMEOUTS, type FileDocPresenceUser, type JoinFileDocPayload, type LeaveFileDocPayload, toFileDocBytes, } from '@sim/realtime-protocol/file-doc' import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' import * as decoding from 'lib0/decoding' import * as encoding from 'lib0/encoding' import type { Server } from 'socket.io' @@ -36,13 +44,29 @@ import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' import { resolveAvatarUrl } from '@/handlers/avatar' -import { fetchFileDocMerge, fetchFileDocSeed } from '@/handlers/file-doc-app' +import { fetchFileDocMerge, fetchFileDocPersist, fetchFileDocSeed } from '@/handlers/file-doc-app' +import { getFileDocStore, REDIS_ORIGIN } from '@/handlers/file-doc-store' import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' const logger = createLogger('FileDocHandlers') +/** + * The transaction origin the server stamps on a SEED apply, so `doc.on('update')` can broadcast + share + * it (peers still need the seed) but skip the debounced persist — the seed IS the file's current + * content, so writing it straight back would only churn a redundant blob version. + */ +const SEED_ORIGIN = Symbol('file-doc-seed') + +/** Debounce window for the server-side project-to-markdown persist while a doc is actively edited. */ +const PERSIST_DEBOUNCE_MS = 5_000 + +/** Cross-task merge-lock wait: while a peer task is merging the same file, retry at this cadence for up + * to ~`mergeRequestMs` so we diff against the peer's RESULT rather than racing it; then proceed. */ +const MERGE_LOCK_RETRY_MS = 200 +const MERGE_LOCK_RETRIES = Math.ceil(FILE_DOC_TIMEOUTS.mergeRequestMs / MERGE_LOCK_RETRY_MS) + /** A socket's presence ownership within a room. */ interface FileDocOwner { /** @@ -70,6 +94,12 @@ interface FileDocRoom { /** True once the server-side seed fetch has started, so concurrent joins don't each fetch. * Reset on a fetch FAILURE so a later join can retry (a genuinely empty file stays empty). */ serverSeedStarted: boolean + /** The workspace this file belongs to, captured at join — needed to persist back to markdown. */ + workspaceId: string | null + /** The last collaborator to edit here, for persist attribution (blob metadata) only. */ + lastEditorUserId: string | null + /** The pending debounced persist timer, if any. */ + persistTimer: ReturnType | null } /** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */ @@ -108,11 +138,75 @@ function originSocketId(origin: unknown): string | null { return typeof origin === 'string' ? origin : null } +/** + * Broadcast an AWARENESS frame to the room ACROSS tasks via the Socket.IO Redis adapter. Awareness + * (cursors/selection) is ephemeral and needs no convergence or replay, so the adapter's cross-task + * fan-out is exactly right for it. + */ function broadcast(io: Server, name: string, payload: Uint8Array, exceptSocketId: string | null) { const channel = exceptSocketId ? io.to(name).except(exceptSocketId) : io.to(name) channel.emit(FILE_DOC_EVENTS.MESSAGE, payload) } +/** + * Broadcast a DOC-SYNC frame to this task's LOCAL clients only. Cross-task delivery rides the shared + * Redis stream (each task's tailer applies the update and runs its OWN local fan-out), so using the + * adapter here would double-deliver and amplify. With no adapter (single-pod dev) `io.local` is the + * whole room, so behavior is unchanged. + */ +function broadcastLocal( + io: Server, + name: string, + payload: Uint8Array, + exceptSocketId: string | null +) { + const channel = exceptSocketId ? io.local.to(name).except(exceptSocketId) : io.local.to(name) + channel.emit(FILE_DOC_EVENTS.MESSAGE, payload) +} + +/** + * Schedule a debounced server-side persist of the live doc back to durable markdown. Coalesces rapid + * edits; a no-op until the room knows its workspace (set at join). The final flush on last-disconnect + * is separate ({@link flushPersist} with `final`). + */ +function schedulePersist(name: string, room: FileDocRoom): void { + if (!room.workspaceId || !room.lastEditorUserId) return + if (room.persistTimer) clearTimeout(room.persistTimer) + room.persistTimer = setTimeout(() => { + room.persistTimer = null + void flushPersist(name, room, false) + }, PERSIST_DEBOUNCE_MS) +} + +/** + * Project the live doc to markdown and write it durably via the app. `final` (last collaborator + * leaving) always writes; a debounced mid-edit flush first claims a cross-task slot (held for the whole + * write, so a concurrent task can't issue an overlapping blob write) so tasks don't each write a + * redundant version. Best-effort: never throws (a failure is retried on the next debounce; the stream + * holds the state meanwhile). + * + * Persists the AUTHORITATIVE shared state (the stream), not this task's local doc: a copilot merge — or + * a peer's edit — published by another task may not be integrated into `room.doc` yet, and a + * last-disconnect flush of that lagging local doc would clobber the durable file (the exact + * copilot-write regression a naive local-doc flush reintroduces). Reading from the stream also means + * the enabled path never touches `room.doc`, so `void flushPersist(name, room, true)` is safe to fire + * immediately before the caller destroys it. When disabled the local doc IS authoritative, and is + * encoded synchronously (before the first await) so the same fire-then-destroy ordering holds. + */ +async function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise { + if (!isDocSeeded(room.doc) || !room.workspaceId || !room.lastEditorUserId) return + const store = getFileDocStore() + try { + if (!final && !(await store.acquirePersistSlot(name, FILE_DOC_TIMEOUTS.persistRequestMs))) + return + const shared = store.enabled ? await store.getStreamState(name) : null + const docState = shared ?? Y.encodeStateAsUpdate(room.doc) + await fetchFileDocPersist(room.workspaceId, room.fileId, room.lastEditorUserId, docState) + } catch (error) { + logger.warn(`Persist failed for file ${room.fileId}`, { error: getErrorMessage(error) }) + } +} + /** * Broadcast the room's collaborator roster to everyone in it, for the avatar stack. One entry * PER SESSION (socket) — the client excludes its own socket and dedupes the remainder per user @@ -159,12 +253,22 @@ function awarenessUpdateClientIds(update: Uint8Array): number[] { } /** - * Drop a room's document + awareness once it has no owners, so an idle file holds no memory. - * A later joiner re-creates and re-seeds it from the file's current markdown. + * Drop a room's document + awareness once it has no owners on THIS task, so an idle file holds no + * memory. Before dropping, flush the converged doc back to durable markdown (the last collaborator on + * this task leaving) and detach from the shared stream. A later joiner re-creates it — catching up + * from the stream if the doc is still live on another task, or re-seeding from markdown otherwise. */ function destroyRoomIfIdle(name: string) { const room = fileDocRooms.get(name) if (!room || room.owners.size > 0) return + if (room.persistTimer) { + clearTimeout(room.persistTimer) + room.persistTimer = null + } + // Final durable flush BEFORE teardown — `flushPersist` encodes the doc synchronously (before the + // destroy below) and awaits the write in the background. Best-effort; never throws. + void flushPersist(name, room, true) + getFileDocStore().detachRoom(name) room.awareness.destroy() room.doc.destroy() fileDocRooms.delete(name) @@ -194,14 +298,32 @@ async function ensureServerSeed( ): Promise { if (room.serverSeedStarted || isDocSeeded(room.doc)) return room.serverSeedStarted = true + const store = getFileDocStore() + // Exactly one task across the cluster builds the seed; the others receive it via the stream (the fix + // for split-brain seeding). `shouldSeed` is true here on a single-pod deployment. + if (!(await store.shouldSeed(name))) { + // A peer is seeding (or already did). Release our guard so a later join can retry if the seed never + // arrives (e.g. the seeder died); the stream / this doc being seeded makes a retry safe. + room.serverSeedStarted = false + return + } + // We hold the seed lock — release it on EVERY exit from here (one `finally`, impossible to leak). try { const update = await fetchFileDocSeed(workspaceId, room.fileId) if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return - if (update) Y.applyUpdate(room.doc, update) - else room.doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + // SEED_ORIGIN → `doc.on('update')` shares the seed to the stream (peers need it) but skips the + // persist (the seed is the file's current content, so a persist would only churn a blob version). + if (update) Y.applyUpdate(room.doc, update, SEED_ORIGIN) + else + room.doc.transact( + () => room.doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true), + SEED_ORIGIN + ) } catch (error) { logger.warn(`Server seed failed for file ${room.fileId} (workspace ${workspaceId})`, error) room.serverSeedStarted = false + } finally { + await store.releaseSeedLock(name) } } @@ -214,15 +336,18 @@ const fileDocMergeChains = new Map>() * fires `doc.on('update')` and relays the merge to every connected editor, reconciled with any * concurrent user edits — and reports whether it landed. * - * Merges for the same file are SERIALIZED: each waits for any in-flight merge on that doc, so it - * snapshots the already-updated state and its diff is computed against current content — two - * overlapping full-document rewrites can never each diff the same stale snapshot and apply out of - * order. (The relay is single-writer per file — Helm pins one replica — so an in-process chain - * suffices.) + * Merges for the same file are SERIALIZED — within a task by the {@link fileDocMergeChains} promise + * chain, and ACROSS tasks by a Redis merge lock (below) — so each diff is computed against the previous + * merge's result, never the same stale base concurrently. * - * Returns `'no-live-room'` when the file has no seeded, occupied room: an unseeded/empty doc has no - * authoritative content to merge against, so the caller (copilot) writes the file directly instead and - * the seed picks up the new content on the next open. + * Multi-task: the diff is computed against, and published to, the file's SHARED stream state — so the + * merge reaches the live doc no matter which task holds it (the apply-edit HTTP call can land on any + * task). Every task's tailer then applies it and fans it out to its own clients. Because the merge + * always lands in the stream while the stream exists, the stream can never go stale relative to a + * copilot direct file write. + * + * Returns `'no-live-room'` when there is no shared state to merge against (no doc is or was recently + * live): the caller (copilot) writes the file directly and the next open seeds from that markdown. */ export function applyMarkdownToLiveFileDoc( fileId: string, @@ -246,9 +371,38 @@ async function mergeMarkdownIntoRoom( fileId: string, markdown: string ): Promise<'applied' | 'no-live-room'> { + const store = getFileDocStore() + + if (store.enabled) { + // Serialize merges to this file ACROSS tasks — the per-file chain above only covers this process. + // Two copilot edits to the same file landing on different tasks must not diff the SAME shared base + // and publish conflicting full-document rewrites; wait briefly for a peer's merge to finish so we + // diff against its RESULT. If the holder is stuck past the wait (likely dead; its lock TTL will + // lapse), proceed anyway rather than drop the edit. + const lockTtl = FILE_DOC_TIMEOUTS.mergeRequestMs + 2_000 + let acquired = await store.acquireMergeSlot(name, lockTtl) + for (let i = 0; !acquired && i < MERGE_LOCK_RETRIES; i++) { + await sleep(MERGE_LOCK_RETRY_MS) + acquired = await store.acquireMergeSlot(name, lockTtl) + } + try { + // Compute the diff against the committed SHARED state and PUBLISH it — every task with the doc + // live (including this one, via its own tailer) applies it and fans it out to its clients, so the + // merge reaches the live doc no matter which task the apply-edit call landed on. An empty stream + // means no doc is (or was recently) live → nothing to merge into. + const base = await store.getStreamState(name) + if (!base) return 'no-live-room' + const diff = await fetchFileDocMerge(fileId, base, markdown) + store.publish(name, diff) + return 'applied' + } finally { + if (acquired) await store.releaseMergeSlot(name) + } + } + + // Single-replica fallback: apply straight to the local authoritative doc. const room = fileDocRooms.get(name) if (!room || room.owners.size === 0 || !isDocSeeded(room.doc)) return 'no-live-room' - const update = await fetchFileDocMerge(fileId, Y.encodeStateAsUpdate(room.doc), markdown) // The room may have been dropped while the diff was being built; never touch a destroyed doc. if (fileDocRooms.get(name) !== room) return 'no-live-room' @@ -278,14 +432,26 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { awareness, owners: new Map(), serverSeedStarted: false, + workspaceId: null, + lastEditorUserId: null, + persistTimer: null, } + // Register synchronously BEFORE the async catch-up so a concurrent join sees this room, not a second. fileDocRooms.set(name, room) doc.on('update', (update: Uint8Array, origin: unknown) => { const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) syncProtocol.writeUpdate(encoder, update) - broadcast(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) + // Fan out to THIS task's clients only (excluding the origin socket if local). Cross-task delivery + // rides the shared stream — every task's tailer applies + runs its own local fan-out. + broadcastLocal(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) + // Share every locally-originated update to the stream so peers converge; an update that ARRIVED + // from the stream (REDIS_ORIGIN) is already there — never re-publish it. + if (origin !== REDIS_ORIGIN) getFileDocStore().publish(name, update) + // Persist real edits (user edits + copilot merges) back to markdown, debounced. Skip the seed (it + // is the file's current content) and stream-relayed updates (their originating task persists them). + if (origin !== REDIS_ORIGIN && origin !== SEED_ORIGIN) schedulePersist(name, room) }) awareness.on('update', ({ added, updated, removed }: AwarenessChange, origin: unknown) => { @@ -300,6 +466,10 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { broadcast(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) }) + // Load the shared state into the doc and start tailing the stream (fire-and-forget: content streams + // in via `doc.on('update')` as it lands, mirroring the fire-and-forget seed below). Disabled → no-op. + void getFileDocStore().attachRoom(name, doc) + return room } @@ -335,6 +505,9 @@ function handleMessage(socket: AuthenticatedSocket, data: unknown) { switch (messageType) { case FILE_DOC_MESSAGE_TYPE.SYNC: { + // Attribute a server-side persist of the resulting edit to the actual editor (blob metadata). + const editor = room.owners.get(socket.id)?.userId + if (editor) room.lastEditorUserId = editor const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) // `socket.id` is the transaction origin, so the doc's `update` handler @@ -534,6 +707,11 @@ export function setupWorkspaceFileDocHandlers( socketToRoomName.set(socket.id, name) socket.join(name) + // Capture what the server-side persist needs: the workspace to write back to, and the current + // user for attribution (refreshed to the actual editor on each edit in `handleMessage`). + if (authorized.workspaceId) entry.workspaceId = authorized.workspaceId + entry.lastEditorUserId = userId + socket.emit(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId }) // Server-authenticated roster → everyone in the room, including this joiner. broadcastFileDocPresence(io, name, entry) diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index 3f7f7eb22d0..910e92a75c9 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -6,6 +6,7 @@ import { createSocketIOServer, shutdownSocketIOAdapter } from '@/config/socket' import { assertSchemaCompatibility } from '@/database/preflight' import { env } from '@/env' import { setupAllHandlers } from '@/handlers' +import { getFileDocStore, initFileDocStore } from '@/handlers/file-doc-store' import { type AuthenticatedSocket, authenticateSocket } from '@/middleware/auth' import { type IRoomManager, MemoryRoomManager, RedisRoomManager } from '@/rooms' import { createHttpHandler } from '@/routes/http' @@ -55,6 +56,10 @@ async function main() { // Initialize room manager (Redis or in-memory based on config) const roomManager = await createRoomManager(io) + // Initialize the shared Yjs backend for collaborative file docs (Redis Streams). Enabled only when + // REDIS_URL is set; otherwise the relay runs its original single-replica in-memory doc path. + await initFileDocStore(env.REDIS_URL) + // Set up authentication middleware io.use(authenticateSocket) @@ -124,6 +129,12 @@ async function main() { logger.error('Error during Socket.IO adapter shutdown:', error) } + try { + await getFileDocStore().shutdown() + } catch (error) { + logger.error('Error during FileDocStore shutdown:', error) + } + httpServer.close(() => { logger.info('Socket.IO server closed') process.exit(0) diff --git a/apps/sim/app/api/internal/file-doc/persist/route.ts b/apps/sim/app/api/internal/file-doc/persist/route.ts new file mode 100644 index 00000000000..9ef87415991 --- /dev/null +++ b/apps/sim/app/api/internal/file-doc/persist/route.ts @@ -0,0 +1,43 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { persistFileDocContract } from '@/lib/api/contracts/file-doc' +import { parseRequest } from '@/lib/api/server' +import { persistFileDoc } from '@/lib/collab-doc/persist' +import { checkInternalApiKey, createUnauthorizedResponse } from '@/lib/copilot/request/http' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + +const logger = createLogger('FileDocPersistAPI') + +/** + * POST /api/internal/file-doc/persist — project a live collaborative document back to durable markdown + * (Yjs → markdown, through the exact editor engine) and write it to the file. Internal only: gated on + * the shared `x-api-key: INTERNAL_API_SECRET` secret, matching the header the realtime relay sends. + * The relay owns the live doc but not the conversion engine or blob/DB access, so it ships the current + * doc state here — the server-authoritative durable path that replaces the editor's client autosave. + */ +export const POST = withRouteHandler(async (request: NextRequest) => { + const auth = checkInternalApiKey(request) + if (!auth.success) return createUnauthorizedResponse() + + const parsed = await parseRequest(persistFileDocContract, request, {}) + if (!parsed.success) return parsed.response + const { workspaceId, fileId, userId, docState } = parsed.data.body + + try { + const persisted = await persistFileDoc( + workspaceId, + fileId, + userId, + new Uint8Array(Buffer.from(docState, 'base64')) + ) + return NextResponse.json({ persisted }) + } catch (error) { + logger.error('Failed to persist file-doc', { workspaceId, fileId, error }) + return NextResponse.json( + { error: getErrorMessage(error, 'Failed to persist document') }, + { status: 500 } + ) + } +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 8b10de6a5ce..21cbbe72ed2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -121,10 +121,11 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ const userName = session?.user?.name?.trim() || 'Collaborator' /** - * Autosave gate for the collaborative path: the child reports `false` while its - * shared document is still syncing/seeding and `true` once it is safe to persist - * the markdown mirror — so an empty or partially-synced doc can never overwrite - * the real file. `true` for non-collaborative files (never gated). + * Client-autosave gate. For a NON-collaborative file this is `true` (the client owns durability and + * autosaves the markdown). For a collaborative file it stays `false`: the realtime relay persists the + * shared document to markdown server-side, so the client must never also autosave — a stale keystroke + * saving over a server/copilot edit is exactly the clobber the server path closes. The child reports + * the right value up via `onCollabReadyChange`. */ const [collabReady, setCollabReady] = useState(true) @@ -652,8 +653,13 @@ export function LoadedRichMarkdownEditor({ */ useEffect(() => { const setReady = (ready: boolean) => { + // Child-local: gates editability (a user must never type into an unsynced/unseeded doc). setCollabReady(ready) - onCollabReadyChange(ready) + // Parent: gates CLIENT autosave. In a collaborative session the relay persists the doc to + // markdown server-side (debounced + on last-disconnect), so the client must NOT also autosave — + // a stale keystroke saving over a server/copilot edit is the clobber the server path closes. + // Only the non-collaborative (solo) path client-autosaves. + onCollabReadyChange(collaboration ? false : ready) } if (!collaboration) { setReady(true) diff --git a/apps/sim/lib/api/contracts/file-doc.ts b/apps/sim/lib/api/contracts/file-doc.ts index efb98168694..a46bce7f5d3 100644 --- a/apps/sim/lib/api/contracts/file-doc.ts +++ b/apps/sim/lib/api/contracts/file-doc.ts @@ -75,3 +75,45 @@ export const mergeFileDocContract = defineRouteContract({ schema: mergeFileDocResponseSchema, }, }) + +export const persistFileDocBodySchema = z.object({ + workspaceId: z.string().min(1, 'workspaceId is required'), + fileId: z.string().min(1, 'fileId is required'), + /** Attribution only (blob metadata) — the last collaborator to touch the live doc. */ + userId: z.string().min(1, 'userId is required'), + /** + * Base64-encoded `Y.encodeStateAsUpdate` of the live document as the relay currently holds it — the + * bound matches the merge contract (generous base64 headroom over a ≤256 KB collab doc's Yjs state). + */ + docState: z + .string() + .min(1, 'docState is required') + .max(16 * 1024 * 1024, 'docState is too large'), +}) +export type PersistFileDocBody = z.input + +export const persistFileDocResponseSchema = z.object({ + /** + * `true` when the document was projected to markdown and written durably to the file; `false` when + * the file was missing/deleted (nothing to write). A transport/conversion failure is a non-2xx. + */ + persisted: z.boolean(), +}) +export type PersistFileDocResponse = z.output + +/** + * Internal, `x-api-key`-gated: the realtime relay asks the app to project a live collaborative + * document back to durable markdown (Yjs → markdown, through the exact editor engine) and write it to + * the file. The relay owns the live doc but not the conversion engine or blob/DB access, so it ships + * the current doc state here. Called debounced during editing and when the last collaborator leaves — + * the server-authoritative durable path that replaces the editor's client-side autosave. + */ +export const persistFileDocContract = defineRouteContract({ + method: 'POST', + path: '/api/internal/file-doc/persist', + body: persistFileDocBodySchema, + response: { + mode: 'json', + schema: persistFileDocResponseSchema, + }, +}) diff --git a/apps/sim/lib/collab-doc/converter.ts b/apps/sim/lib/collab-doc/converter.ts index e18c1ca8ffc..5557ffb2439 100644 --- a/apps/sim/lib/collab-doc/converter.ts +++ b/apps/sim/lib/collab-doc/converter.ts @@ -1,3 +1,4 @@ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' import { getSchema } from '@tiptap/core' import { Node as ProseMirrorNode, type Schema } from '@tiptap/pm/model' import { @@ -8,6 +9,7 @@ import { } from '@tiptap/y-tiptap' import type * as Y from 'yjs' import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { applyFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { parseMarkdownToDoc, serializeDocToMarkdown, @@ -76,13 +78,24 @@ export function markdownToYDoc(markdown: string): Y.Doc { return prosemirrorJSONToYDoc(markdownSchema(), json, COLLAB_DOC_FIELD) } -/** Project a collaborative {@link Y.Doc} back to the file's canonical markdown. */ +/** Project a collaborative {@link Y.Doc}'s BODY back to markdown (no frontmatter). */ export function yDocToMarkdown(ydoc: Y.Doc): string { ensureDomForTipTap() const json = yDocToProsemirrorJSON(ydoc, COLLAB_DOC_FIELD) return serializeDocToMarkdown(json) } +/** + * Project a collaborative {@link Y.Doc} back to the file's FULL canonical markdown — the body from the + * CRDT re-joined with the frontmatter carried in the config map. This is exactly what the editor + * writes on save (`applyFrontmatter(resolveSaveFrontmatter(), body)`), so a server-side persist of the + * live doc is byte-identical to a client save — no spurious churn on the round-trip. + */ +export function yDocToFileMarkdown(ydoc: Y.Doc): string { + const frontmatter = ydoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.frontmatterKey) + return applyFrontmatter(typeof frontmatter === 'string' ? frontmatter : '', yDocToMarkdown(ydoc)) +} + /** * Apply new markdown content into an EXISTING collaborative {@link Y.Doc} as a minimal CRDT diff, * merging with any concurrent user edits rather than replacing the document. This is how the agent diff --git a/apps/sim/lib/collab-doc/persist.ts b/apps/sim/lib/collab-doc/persist.ts new file mode 100644 index 00000000000..2a0a2393292 --- /dev/null +++ b/apps/sim/lib/collab-doc/persist.ts @@ -0,0 +1,42 @@ +import { createLogger } from '@sim/logger' +import * as Y from 'yjs' +import { getWorkspaceFile, updateWorkspaceFileContent } from '@/lib/uploads/contexts/workspace' +import { yDocToFileMarkdown } from './converter' + +const logger = createLogger('FileDocPersist') + +/** + * Project a live collaborative document back to durable markdown and write it to the file. This is the + * server-authoritative durable path — the realtime relay owns the live Yjs doc but not the conversion + * engine or blob/DB access, so it ships the doc state here and the app persists it. Called debounced + * while the doc is being edited and when the last collaborator leaves; it replaces the editor's + * client-side autosave, so a copilot (or any server) edit can never be clobbered by a stale keystroke + * saving over it. + * + * Returns `false` when the file is genuinely absent (deleted) — nothing to write. Any other failure + * (conversion / blob / DB) THROWS so the caller surfaces a non-2xx and can retry on the next debounce. + * + * `userId` is attribution only (blob metadata) — the last collaborator to touch the doc; the caller is + * already trusted via the internal `x-api-key` gate, so this does not re-authorize. + */ +export async function persistFileDoc( + workspaceId: string, + fileId: string, + userId: string, + docState: Uint8Array +): Promise { + const record = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) + if (!record) return false + + const ydoc = new Y.Doc() + try { + Y.applyUpdate(ydoc, docState) + const markdown = yDocToFileMarkdown(ydoc) + await updateWorkspaceFileContent(workspaceId, fileId, userId, Buffer.from(markdown, 'utf-8')) + } finally { + ydoc.destroy() + } + + logger.info(`Persisted live collaborative document to file ${fileId} (workspace ${workspaceId})`) + return true +} diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts index 7d86138b330..474383a433f 100644 --- a/apps/sim/lib/realtime/notify.ts +++ b/apps/sim/lib/realtime/notify.ts @@ -53,15 +53,14 @@ export async function notifyWorkspaceFilesChanged(workspaceId: string): Promise< /** * Best-effort: ask the realtime relay to merge a copilot edit into a file's LIVE collaborative * document, so open editors see it stream in as a CRDT merge (Stage C) rather than the file changing - * underneath them. No-op when no editor is connected (the relay reports `applied: false`). The file - * itself is written durably by the caller regardless — this only drives the live view. Never throws. + * underneath them. No-op when no doc is (or was recently) live (the relay reports `applied: false`). + * The file itself is written durably by the caller regardless — this only drives the live view. + * Never throws. * - * KNOWN GAP (narrow): if an editor IS open but this merge fails (socket pod slow/down), the open - * editor keeps the pre-edit doc; the user's next keystroke autosaves that stale doc over the durable - * write, dropping the copilot edit until a reload. This is the interim cost of "durable file write + - * best-effort live merge + editor autosave reconciles" and is closed by the deferred move to a - * durable server-authoritative doc (copilot writing THROUGH the document rather than the file). Rare - * — it needs the socket pod unreachable exactly while the file is open — and non-corrupting. + * The former clobber gap — an open editor's autosave dropping this edit — is now closed: a + * collaborative editor no longer client-autosaves (the relay persists the shared doc to markdown + * server-side), and the relay applies this merge THROUGH the shared Redis stream, so it reaches the + * live doc on whichever task holds it and can't go stale relative to this direct write. * * Awaited (not fire-and-forget) so the fetch dispatches before the route handler returns; bounded to * {@link APPLY_EDIT_TIMEOUT_MS}, so it adds latency only when the socket pod is unreachable. diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index dec2577d8eb..506720b51d5 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -138,12 +138,13 @@ const nextConfig: NextConfig = { ], outputFileTracingIncludes: { '/api/tools/stagehand/*': ['./node_modules/ws/**/*'], - // The seed and merge endpoints both lazily `require('jsdom')` (via the collab-doc converter), which - // is invisible to the standalone file tracer, so force jsdom (and its transitive deps, followed - // from its static requires) into the trace — otherwise a Docker/standalone build omits it and the - // endpoint 500s with MODULE_NOT_FOUND. + // The seed, merge, and persist endpoints all lazily `require('jsdom')` (via the collab-doc + // converter), which is invisible to the standalone file tracer, so force jsdom (and its transitive + // deps, followed from its static requires) into the trace — otherwise a Docker/standalone build + // omits it and the endpoint 500s with MODULE_NOT_FOUND. '/api/internal/file-doc/seed': ['./node_modules/jsdom/**/*'], '/api/internal/file-doc/merge': ['./node_modules/jsdom/**/*'], + '/api/internal/file-doc/persist': ['./node_modules/jsdom/**/*'], '/*': [ './node_modules/sharp/**/*', './node_modules/@img/**/*', diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index bba9db511bb..e58ceb477a3 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -94,12 +94,18 @@ export const FILE_DOC_SEED = { * * The seed request gets more headroom than the merge because it reads a (possibly cold) blob before * converting; the merge is a pure in-memory conversion the caller fully supplies. + * + * `persistRequestMs` (relay → app `/persist`) stands alone — no client waits on it (the relay flushes + * the live doc to durable markdown debounced during editing and on the last collaborator leaving), so + * it forms no ordering invariant. It gets seed-level headroom because, like the seed, it crosses a + * durable blob write (Yjs → markdown → storage), not just an in-memory conversion. */ export const FILE_DOC_TIMEOUTS = { seedRequestMs: 8_000, mergeRequestMs: 3_000, applyEditMs: 6_000, readinessDeadlineMs: 12_000, + persistRequestMs: 8_000, } as const /** Client → server join request. `fileId` is the `workspace_files.id`. */ diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 2c49dcf26ca..7d364ffe8d8 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 983, - zodRoutes: 983, + totalRoutes: 984, + zodRoutes: 984, nonZodRoutes: 0, } as const From 161b0312c36690d162dc7b59fa2a26239db2180d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 18:18:47 -0700 Subject: [PATCH 40/53] fix(collab-doc): harden distributed locks and durability from review Address Greptile + Cursor review of the multi-replica backend: - Merge lock: retry LONGER than the lock TTL (guaranteed acquisition, never merges against a shared base while a peer holds the lock) and AWAIT the stream write before releasing, so the next task never diffs a stale base. - Distributed locks (seed/merge/compact) now use ownership tokens + a compare-and-delete release (Lua), so a lock that expired and was re-acquired by another task is never stolen; acquisition fails CLOSED on Redis error. - Seed: publish the seed to the stream AWAITED under the lock before releasing, so a later seeder's empty-stream fence always sees it (closes the fence's publish-after-release gap); TTL kept at the readiness deadline. - Persist: persist the AUTHORITATIVE stream state even when this task's local doc was never seeded, and capture the local fallback synchronously so a last-disconnect flush never encodes an already-destroyed doc. - Publish: retry a transient xAdd failure so a Redis blip can't silently drop an edit from the shared log. --- .../src/handlers/file-doc-store.test.ts | 72 +++++-- apps/realtime/src/handlers/file-doc-store.ts | 191 ++++++++++++------ apps/realtime/src/handlers/file-doc.ts | 83 +++++--- 3 files changed, 242 insertions(+), 104 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index 75faf7f89c4..b38fb5cdfce 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -13,6 +13,8 @@ interface Backing { streams: Map }[]> kv: Map seq: number + /** Number of upcoming xAdd calls to fail with a transient error (to exercise publish retry). */ + failXAdd: number } const state = vi.hoisted(() => ({ backing: null as Backing | null })) @@ -30,6 +32,10 @@ function makeClient(): any { on: () => client, duplicate: () => makeClient(), xAdd: async (key: string, _star: string, fields: Record) => { + if (b().failXAdd > 0) { + b().failXAdd-- + throw new Error('transient xAdd failure') + } const id = `${++b().seq}-0` const arr = b().streams.get(key) ?? [] arr.push({ id, message: { ...fields } }) @@ -65,6 +71,16 @@ function makeClient(): any { b().kv.delete(key) return 1 }, + // Compare-and-delete Lua (RELEASE_LOCK_SCRIPT): del only if the stored value matches the token. + eval: async (_script: string, opts: { keys: string[]; arguments: string[] }) => { + const [key] = opts.keys + const [token] = opts.arguments + if (b().kv.get(key) === token) { + b().kv.delete(key) + return 1 + } + return 0 + }, expire: async () => 1, } return client @@ -101,7 +117,7 @@ async function newStore(): Promise { describe('FileDocStore', () => { beforeEach(() => { - state.backing = { streams: new Map(), kv: new Map(), seq: 0 } + state.backing = { streams: new Map(), kv: new Map(), seq: 0, failXAdd: 0 } stores = [] }) @@ -112,20 +128,22 @@ describe('FileDocStore', () => { it('elects exactly one seeder across tasks (no split-brain seed)', async () => { const a = await newStore() const b = await newStore() - const [aWon, bWon] = await Promise.all([a.shouldSeed(NAME), b.shouldSeed(NAME)]) - expect([aWon, bWon].filter(Boolean)).toHaveLength(1) + // shouldSeed returns a lock token (truthy) for the winner, null for the loser. + const [aTok, bTok] = await Promise.all([a.shouldSeed(NAME), b.shouldSeed(NAME)]) + expect([aTok, bTok].filter(Boolean)).toHaveLength(1) }) it('does not re-seed once the stream already has content (stale lock)', async () => { const a = await newStore() - expect(await a.shouldSeed(NAME)).toBe(true) + const token = await a.shouldSeed(NAME) + expect(token).toBeTruthy() // A seeds and releases its lock. a.publish(NAME, updateFor('hello')) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) - await a.releaseSeedLock(NAME) + await a.releaseSeedLock(NAME, token as string) // A different task must NOT seed again — the lock is free but the stream is non-empty. const b = await newStore() - expect(await b.shouldSeed(NAME)).toBe(false) + expect(await b.shouldSeed(NAME)).toBeNull() }) it('getStreamState reconstructs the shared document from the stream', async () => { @@ -204,22 +222,50 @@ describe('FileDocStore', () => { doc.destroy() }) + it('retries a transient append failure so the edit is not lost from the shared log', async () => { + const a = await newStore() + state.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed + a.publish(NAME, updateFor('resilient')) + await vi.waitFor( + async () => { + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(doc.getText('body').toString()).toBe('resilient') + doc.destroy() + }, + { timeout: 2000 } + ) + }) + + it('streamHasContent fences a seed apply against an already-seeded stream', async () => { + const a = await newStore() + expect(await a.streamHasContent(NAME)).toBe(false) + a.publish(NAME, updateFor('seeded')) + await vi.waitFor(async () => expect(await a.streamHasContent(NAME)).toBe(true)) + }) + it('serializes merges across tasks via the merge lock', async () => { const a = await newStore() const b = await newStore() - expect(await a.acquireMergeSlot(NAME, 5_000)).toBe(true) + const aTok = await a.acquireMergeSlot(NAME, 5_000) + expect(aTok).toBeTruthy() // A holds it → B is refused until A releases. - expect(await b.acquireMergeSlot(NAME, 5_000)).toBe(false) - await a.releaseMergeSlot(NAME) - expect(await b.acquireMergeSlot(NAME, 5_000)).toBe(true) - await b.releaseMergeSlot(NAME) + expect(await b.acquireMergeSlot(NAME, 5_000)).toBeNull() + // A stale-holder release with the WRONG token must NOT free A's lock (compare-and-delete). + await b.releaseMergeSlot(NAME, 'wrong-token') + expect(await b.acquireMergeSlot(NAME, 5_000)).toBeNull() + // A releases with its real token → B can now acquire. + await a.releaseMergeSlot(NAME, aTok as string) + const bTok = await b.acquireMergeSlot(NAME, 5_000) + expect(bTok).toBeTruthy() + await b.releaseMergeSlot(NAME, bTok as string) }) it('is disabled without a REDIS_URL and behaves single-replica', async () => { const store = new FileDocStore(undefined) expect(store.enabled).toBe(false) - // Seeds locally (returns true), never touches a stream. - expect(await store.shouldSeed(NAME)).toBe(true) + // Seeds locally (returns a sentinel token), never touches a stream. + expect(await store.shouldSeed(NAME)).toBeTruthy() expect(await store.getStreamState(NAME)).toBeNull() const doc = new Y.Doc() await store.attachRoom(NAME, doc) // no-op, no throw diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index aafcf0a7890..dde15206eda 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -34,11 +34,20 @@ import { createLogger } from '@sim/logger' import { FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { backoffWithJitter } from '@sim/utils/retry' import { createClient, type RedisClientType } from 'redis' import * as Y from 'yjs' const logger = createLogger('FileDocStore') +/** + * Compare-and-delete: release a lock ONLY if this task still holds it (its token still the value), so a + * lock that expired and was re-acquired by another task is never stolen by the original holder's release. + */ +const RELEASE_LOCK_SCRIPT = + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end" + /** * The transaction origin the store stamps on updates it applies from the stream. The relay's * `doc.on('update')` handler uses it to distinguish an update that ARRIVED from a peer (fan out to @@ -56,6 +65,11 @@ const MERGE_LOCK_PREFIX = 'filedoc:mergelock:' /** The single field each stream entry carries — a base64 Yjs update. */ const UPDATE_FIELD = 'u' +/** Sentinel token a DISABLED store returns from a lock acquire, so single-replica callers proceed + * without special-casing; {@link FileDocStore.releaseLock} treats it as a no-op. Not a real UUID, so it + * can never collide with a {@link generateId} token. */ +const DISABLED_LOCK_TOKEN = '__disabled__' + /** How long a blocking multiplexed read waits before re-snapshotting the live room set. Also bounds * how long a room attached mid-block waits for its first cross-task update (updates are not lost — the * next read resumes from its last id — only briefly delayed). */ @@ -69,11 +83,19 @@ const READ_COUNT = 200 const COMPACT_THRESHOLD = 400 /** Check whether compaction is due only every Nth local publish, to avoid an XLEN per keystroke. */ const COMPACT_CHECK_EVERY = 64 -/** The seed lock is held across the app seed fetch (hard-bounded at `seedRequestMs`) plus the apply + - * publish. The generous cushion over `seedRequestMs` means only a multi-second event-loop stall — not - * ordinary latency — could expire it mid-seed, so the single-seeder invariant does not hinge on a tight - * 2s margin coupled to an unrelated timeout. */ -const SEED_LOCK_TTL_MS = FILE_DOC_TIMEOUTS.seedRequestMs + 12_000 +/** Compaction critical section (snapshot + xAdd + xTrim) is fast; a generous TTL covers a slow Redis + * round-trip without risking expiry mid-compact. Released via compare-and-delete regardless. */ +const COMPACT_LOCK_TTL_MS = 10_000 +/** Retry a failed stream append this many times before giving up, so a transient Redis blip doesn't + * silently drop an edit from the shared log (which no peer would then ever see). */ +const PUBLISH_MAX_RETRIES = 3 +/** The seed lock spans the app seed fetch (hard-bounded at `seedRequestMs = 8s`) + the apply + the + * AWAITED seed publish. The margin comfortably exceeds the fetch bound so the lock does not expire + * mid-seed, while staying at the client readiness deadline (12s) so a dead seeder's lock frees when + * clients would recover anyway. Double-seed is prevented regardless of the margin: the seeder publishes + * the seed to the stream BEFORE releasing the lock, so any later seeder's {@link streamHasContent} fence + * sees it. */ +const SEED_LOCK_TTL_MS = FILE_DOC_TIMEOUTS.seedRequestMs + 4_000 /** How long a stream survives with no heartbeat — long enough that an occupied-but-idle doc never * loses its shared state (the heartbeat refreshes it while any task holds the room). */ const STREAM_TTL_SEC = 600 @@ -198,51 +220,108 @@ export class FileDocStore { } /** - * Append a locally-applied update to the shared stream so every task converges. Called from the - * relay's `doc.on('update')` for local edits only (never for {@link REDIS_ORIGIN} updates — those - * already came from the stream). No-op when disabled. + * Append a locally-applied update to the shared stream so every task converges, AWAITING the write + * and retrying a transient failure ({@link PUBLISH_MAX_RETRIES}) so a Redis blip can't silently drop + * an edit from the shared log. Only the `xAdd` is retried; the TTL refresh + compaction check are + * post-write best-effort and never re-trigger the append. Throws if the append ultimately fails. + */ + private async appendUpdate(name: string, update: Uint8Array): Promise { + if (!this.write) return + const encoded = Buffer.from(update).toString('base64') + for (let attempt = 0; ; attempt++) { + try { + await this.write.xAdd(streamKey(name), '*', { [UPDATE_FIELD]: encoded }) + break + } catch (error) { + if (attempt >= PUBLISH_MAX_RETRIES) { + logger.error(`FileDocStore append failed for ${name}`, { error: getErrorMessage(error) }) + throw error + } + // Snappy backoff — a stream append is a fast op; a transient blip clears in tens of ms. + // `backoffWithJitter` is 1-indexed, so pass the 1-based attempt number. + await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) + } + } + await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + const room = this.rooms.get(name) + if (room && ++room.publishes % COMPACT_CHECK_EVERY === 0) void this.maybeCompact(name) + } + + /** + * Fire-and-forget append for the hot keystroke path (`doc.on('update')`): converges peers without + * blocking the relay. Retries internally; never throws. No-op when disabled. */ publish(name: string, update: Uint8Array): void { if (!this.enabled || !this.write) return - const room = this.rooms.get(name) - void this.write - .xAdd(streamKey(name), '*', { [UPDATE_FIELD]: Buffer.from(update).toString('base64') }) - .then(() => this.write?.expire(streamKey(name), STREAM_TTL_SEC)) - .then(() => { - if (room && ++room.publishes % COMPACT_CHECK_EVERY === 0) return this.maybeCompact(name) - }) - .catch((error) => - logger.warn(`FileDocStore publish failed for ${name}`, { error: getErrorMessage(error) }) - ) + void this.appendUpdate(name, update).catch(() => {}) // already logged inside appendUpdate } /** - * Decide whether THIS task should build and write the file's one-time seed. Returns true only when - * the shared stream is genuinely empty AND this task wins the seed lock — so exactly one task across - * the cluster ever seeds a file, even if several open it at once (the fix for split-brain seeding). - * When disabled, always true (single-replica: seed locally). + * Awaitable append for callers that must know the update is durably in the stream before proceeding + * — the copilot merge, so the cross-task merge lock is not released before the diff is committed + * (else the next task would diff a stale base). Throws on ultimate failure. No-op when disabled. */ - async shouldSeed(name: string): Promise { - if (!this.enabled || !this.write) return true + async publishAndWait(name: string, update: Uint8Array): Promise { + if (!this.enabled || !this.write) return + await this.appendUpdate(name, update) + } + + /** + * Whether the file's stream already holds content — fences a seed apply against a peer that seeded + * while this task held a (possibly stale) lock, so we never write a second seed (split-brain). False + * when disabled or on error (the caller then proceeds under its lock, as before). + */ + async streamHasContent(name: string): Promise { + if (!this.enabled || !this.write) return false try { - const won = await this.write.set(`${SEED_LOCK_PREFIX}${name}`, '1', { - NX: true, - PX: SEED_LOCK_TTL_MS, - }) - if (won !== 'OK') return false - // The lock could be free yet the stream already seeded (a prior holder seeded then its lock - // expired). Re-check under the lock so we never write a SECOND seed on top of an existing one. - if ((await this.write.xLen(streamKey(name))) > 0) { - await this.releaseSeedLock(name) - return false - } - return true - } catch (error) { - logger.warn(`FileDocStore shouldSeed failed for ${name}`, { error: getErrorMessage(error) }) + return (await this.write.xLen(streamKey(name))) > 0 + } catch { return false } } + /** + * Acquire a distributed lock with a unique ownership TOKEN (`SET key NX PX`). Returns the + * token to release with, or `null` if not won. Fails CLOSED (null) on a Redis error — a lock we can't + * prove we hold must not be treated as held. The special sentinel {@link DISABLED_LOCK_TOKEN} lets a + * disabled store return a truthy token so callers proceed single-replica without special-casing. + */ + private async acquireLock(key: string, ttlMs: number): Promise { + if (!this.enabled || !this.write) return DISABLED_LOCK_TOKEN + const token = generateId() + try { + return (await this.write.set(key, token, { NX: true, PX: ttlMs })) === 'OK' ? token : null + } catch (error) { + logger.warn(`FileDocStore lock ${key} failed`, { error: getErrorMessage(error) }) + return null + } + } + + /** Release a lock via compare-and-delete, so it is only dropped if we still hold our token. */ + private async releaseLock(key: string, token: string): Promise { + if (!this.write || token === DISABLED_LOCK_TOKEN) return + await this.write.eval(RELEASE_LOCK_SCRIPT, { keys: [key], arguments: [token] }).catch(() => {}) + } + + /** + * Decide whether THIS task should build and write the file's one-time seed. Returns a lock TOKEN only + * when the shared stream is genuinely empty AND this task wins the seed lock — so exactly one task + * across the cluster ever seeds a file, even if several open it at once (the fix for split-brain + * seeding). `null` otherwise. Release the token with {@link releaseSeedLock}. Disabled → always a + * token (single-replica: seed locally). + */ + async shouldSeed(name: string): Promise { + const token = await this.acquireLock(`${SEED_LOCK_PREFIX}${name}`, SEED_LOCK_TTL_MS) + if (!token || token === DISABLED_LOCK_TOKEN) return token + // The lock could be free yet the stream already seeded (a prior holder seeded then its lock + // expired). Re-check under the lock so we never write a SECOND seed on top of an existing one. + if (await this.streamHasContent(name)) { + await this.releaseSeedLock(name, token) + return null + } + return token + } + /** * Build the file's current shared state from the stream, headless (no registered room), for a merge * that must reach the live doc regardless of which task holds it. Returns the encoded Yjs state, or @@ -262,10 +341,9 @@ export class FileDocStore { } } - /** Release the seed lock (best-effort) once the seed has been published or a seed attempt failed. */ - async releaseSeedLock(name: string): Promise { - if (!this.enabled || !this.write) return - await this.write.del(`${SEED_LOCK_PREFIX}${name}`).catch(() => {}) + /** Release the seed lock (compare-and-delete) once the seed has been published or a seed attempt failed. */ + async releaseSeedLock(name: string, token: string): Promise { + await this.releaseLock(`${SEED_LOCK_PREFIX}${name}`, token) } /** @@ -292,22 +370,15 @@ export class FileDocStore { * merges per task; this extends that across tasks so two copilot edits to the same file landing on * different tasks don't each diff the SAME shared base and publish conflicting full-document rewrites. * The loser waits and retries so it diffs against the winner's RESULT (correct sequential merge). - * Returns true (proceed) when disabled or once the lock is won. Release with {@link releaseMergeSlot}. + * Returns a lock TOKEN (proceed) when disabled or once won; `null` otherwise (fails CLOSED on error, so + * a merge never races when exclusivity can't be proven). Release with {@link releaseMergeSlot}. */ - async acquireMergeSlot(name: string, ttlMs: number): Promise { - if (!this.enabled || !this.write) return true - try { - return ( - (await this.write.set(`${MERGE_LOCK_PREFIX}${name}`, '1', { NX: true, PX: ttlMs })) === 'OK' - ) - } catch { - return true - } + async acquireMergeSlot(name: string, ttlMs: number): Promise { + return this.acquireLock(`${MERGE_LOCK_PREFIX}${name}`, ttlMs) } - async releaseMergeSlot(name: string): Promise { - if (!this.enabled || !this.write) return - await this.write.del(`${MERGE_LOCK_PREFIX}${name}`).catch(() => {}) + async releaseMergeSlot(name: string, token: string): Promise { + await this.releaseLock(`${MERGE_LOCK_PREFIX}${name}`, token) } private applyEntry(room: StoreRoom, id: string, message: Record): void { @@ -358,11 +429,9 @@ export class FileDocStore { if (!room) return try { if ((await this.write.xLen(streamKey(name))) < COMPACT_THRESHOLD) return - const won = await this.write.set(`${COMPACT_LOCK_PREFIX}${name}`, '1', { - NX: true, - PX: 10_000, - }) - if (won !== 'OK') return + const key = `${COMPACT_LOCK_PREFIX}${name}` + const token = await this.acquireLock(key, COMPACT_LOCK_TTL_MS) + if (!token) return try { // Capture the snapshot AND the id it covers in one synchronous step (no await between): the // snapshot is `room.doc`, which holds exactly what this task's tailer has integrated — every @@ -377,7 +446,7 @@ export class FileDocStore { // `upTo` itself (redundant with the snapshot, harmless); it drops only the folded older deltas. await this.write.xTrim(streamKey(name), 'MINID', upTo) } finally { - await this.write.del(`${COMPACT_LOCK_PREFIX}${name}`).catch(() => {}) + await this.releaseLock(key, token) } } catch (error) { logger.warn(`FileDocStore compaction failed for ${name}`, { error: getErrorMessage(error) }) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index e4ff1823f0d..596edb0e05e 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -62,10 +62,17 @@ const SEED_ORIGIN = Symbol('file-doc-seed') /** Debounce window for the server-side project-to-markdown persist while a doc is actively edited. */ const PERSIST_DEBOUNCE_MS = 5_000 -/** Cross-task merge-lock wait: while a peer task is merging the same file, retry at this cadence for up - * to ~`mergeRequestMs` so we diff against the peer's RESULT rather than racing it; then proceed. */ +/** Cross-task merge lock. The TTL must exceed the whole critical section it guards — stream fold + + * `fetchFileDocMerge` (bounded at `mergeRequestMs`) + the awaited publish — so the lock never expires + * mid-merge and lets a second task race the same base; hence `mergeRequestMs` plus generous headroom. + * The waiter retries at {@link MERGE_LOCK_RETRY_MS} for LONGER than the TTL, so a live holder always + * releases (or a dead holder's lock expires) within the window. Release is compare-and-delete on an + * ownership token, so even a pathological over-TTL hold never deletes another task's re-acquired lock. */ +const MERGE_LOCK_TTL_MS = FILE_DOC_TIMEOUTS.mergeRequestMs + 5_000 const MERGE_LOCK_RETRY_MS = 200 -const MERGE_LOCK_RETRIES = Math.ceil(FILE_DOC_TIMEOUTS.mergeRequestMs / MERGE_LOCK_RETRY_MS) +const MERGE_LOCK_RETRIES = Math.ceil( + (MERGE_LOCK_TTL_MS + FILE_DOC_TIMEOUTS.mergeRequestMs) / MERGE_LOCK_RETRY_MS +) /** A socket's presence ownership within a room. */ interface FileDocOwner { @@ -186,21 +193,23 @@ function schedulePersist(name: string, room: FileDocRoom): void { * holds the state meanwhile). * * Persists the AUTHORITATIVE shared state (the stream), not this task's local doc: a copilot merge — or - * a peer's edit — published by another task may not be integrated into `room.doc` yet, and a - * last-disconnect flush of that lagging local doc would clobber the durable file (the exact - * copilot-write regression a naive local-doc flush reintroduces). Reading from the stream also means - * the enabled path never touches `room.doc`, so `void flushPersist(name, room, true)` is safe to fire - * immediately before the caller destroys it. When disabled the local doc IS authoritative, and is - * encoded synchronously (before the first await) so the same fire-then-destroy ordering holds. + * a peer's edit — published by another task may not be integrated into `room.doc` yet (and the stream + * holds content even when THIS task's doc was never locally seeded), so a last-disconnect flush can't + * clobber the durable file with a lagging projection. The local doc is captured SYNCHRONOUSLY as a + * fallback before any await, so a `void flushPersist(name, room, true)` fired immediately before the + * caller destroys `room.doc` never encodes a destroyed doc, and the disabled path stays authoritative. */ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise { - if (!isDocSeeded(room.doc) || !room.workspaceId || !room.lastEditorUserId) return + if (!room.workspaceId || !room.lastEditorUserId) return const store = getFileDocStore() + // Synchronous fallback capture — before any await, since the caller may destroy `room.doc` the moment + // this yields. Only meaningful once seeded; used only when the authoritative stream state is absent. + const localState = isDocSeeded(room.doc) ? Y.encodeStateAsUpdate(room.doc) : null try { if (!final && !(await store.acquirePersistSlot(name, FILE_DOC_TIMEOUTS.persistRequestMs))) return - const shared = store.enabled ? await store.getStreamState(name) : null - const docState = shared ?? Y.encodeStateAsUpdate(room.doc) + const docState = (store.enabled ? await store.getStreamState(name) : null) ?? localState + if (!docState) return // nothing seeded/authoritative to persist yet await fetchFileDocPersist(room.workspaceId, room.fileId, room.lastEditorUserId, docState) } catch (error) { logger.warn(`Persist failed for file ${room.fileId}`, { error: getErrorMessage(error) }) @@ -300,8 +309,9 @@ async function ensureServerSeed( room.serverSeedStarted = true const store = getFileDocStore() // Exactly one task across the cluster builds the seed; the others receive it via the stream (the fix - // for split-brain seeding). `shouldSeed` is true here on a single-pod deployment. - if (!(await store.shouldSeed(name))) { + // for split-brain seeding). Returns a lock token here (single-pod: a sentinel token). + const token = await store.shouldSeed(name) + if (!token) { // A peer is seeding (or already did). Release our guard so a later join can retry if the seed never // arrives (e.g. the seeder died); the stream / this doc being seeded makes a retry safe. room.serverSeedStarted = false @@ -311,19 +321,26 @@ async function ensureServerSeed( try { const update = await fetchFileDocSeed(workspaceId, room.fileId) if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return - // SEED_ORIGIN → `doc.on('update')` shares the seed to the stream (peers need it) but skips the - // persist (the seed is the file's current content, so a persist would only churn a blob version). + // Fence against a peer that seeded while we held a stale lock (a rare long stall): if the stream + // already has content it is seeded, so never write a SECOND seed on top — that would split-brain. + if (await store.streamHasContent(name)) return + // SEED_ORIGIN → `doc.on('update')` fans the seed to THIS task's clients but does NOT publish it + // (nor persist it — the seed is the file's current content). We publish it EXPLICITLY and AWAIT the + // stream write below, so the seed is durably in the stream BEFORE the lock releases — then any later + // seeder's `streamHasContent` fence is guaranteed to see it. Closes the fence's publish-after-release + // gap that a fire-and-forget publish left open. if (update) Y.applyUpdate(room.doc, update, SEED_ORIGIN) else room.doc.transact( () => room.doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true), SEED_ORIGIN ) + await store.publishAndWait(name, Y.encodeStateAsUpdate(room.doc)) } catch (error) { logger.warn(`Server seed failed for file ${room.fileId} (workspace ${workspaceId})`, error) room.serverSeedStarted = false } finally { - await store.releaseSeedLock(name) + await store.releaseSeedLock(name, token) } } @@ -376,27 +393,32 @@ async function mergeMarkdownIntoRoom( if (store.enabled) { // Serialize merges to this file ACROSS tasks — the per-file chain above only covers this process. // Two copilot edits to the same file landing on different tasks must not diff the SAME shared base - // and publish conflicting full-document rewrites; wait briefly for a peer's merge to finish so we - // diff against its RESULT. If the holder is stuck past the wait (likely dead; its lock TTL will - // lapse), proceed anyway rather than drop the edit. - const lockTtl = FILE_DOC_TIMEOUTS.mergeRequestMs + 2_000 - let acquired = await store.acquireMergeSlot(name, lockTtl) - for (let i = 0; !acquired && i < MERGE_LOCK_RETRIES; i++) { + // and publish conflicting full-document rewrites. Retry LONGER than the lock TTL, so a live holder + // always releases (or its lock expires) first and we acquire — never merging against a shared base + // while a peer holds the lock. If somehow still unavailable, skip the live merge (copilot's durable + // file write stands) rather than race. + let token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + for (let i = 0; !token && i < MERGE_LOCK_RETRIES; i++) { await sleep(MERGE_LOCK_RETRY_MS) - acquired = await store.acquireMergeSlot(name, lockTtl) + token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + } + if (!token) { + logger.warn(`Merge lock unavailable for file ${fileId}; skipping live merge`) + return 'no-live-room' } try { // Compute the diff against the committed SHARED state and PUBLISH it — every task with the doc // live (including this one, via its own tailer) applies it and fans it out to its clients, so the // merge reaches the live doc no matter which task the apply-edit call landed on. An empty stream - // means no doc is (or was recently) live → nothing to merge into. + // means no doc is (or was recently) live → nothing to merge into. AWAIT the publish so the diff is + // durably in the stream before we release the lock (else the next task would diff a stale base). const base = await store.getStreamState(name) if (!base) return 'no-live-room' const diff = await fetchFileDocMerge(fileId, base, markdown) - store.publish(name, diff) + await store.publishAndWait(name, diff) return 'applied' } finally { - if (acquired) await store.releaseMergeSlot(name) + await store.releaseMergeSlot(name, token) } } @@ -446,9 +468,10 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { // Fan out to THIS task's clients only (excluding the origin socket if local). Cross-task delivery // rides the shared stream — every task's tailer applies + runs its own local fan-out. broadcastLocal(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) - // Share every locally-originated update to the stream so peers converge; an update that ARRIVED - // from the stream (REDIS_ORIGIN) is already there — never re-publish it. - if (origin !== REDIS_ORIGIN) getFileDocStore().publish(name, update) + // Share every locally-originated update to the stream so peers converge. Skip REDIS_ORIGIN (already + // in the stream) and SEED_ORIGIN — the seed is published EXPLICITLY and AWAITED under the seed lock + // (so it lands before the lock releases), which a fire-and-forget publish here couldn't guarantee. + if (origin !== REDIS_ORIGIN && origin !== SEED_ORIGIN) getFileDocStore().publish(name, update) // Persist real edits (user edits + copilot merges) back to markdown, debounced. Skip the seed (it // is the file's current content) and stream-relayed updates (their originating task persists them). if (origin !== REDIS_ORIGIN && origin !== SEED_ORIGIN) schedulePersist(name, room) From b52850969f1739076dc53213a31bf788eeef87dc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 18:35:09 -0700 Subject: [PATCH 41/53] fix(collab-doc): only persist a doc a user actually edited Cursor review: a copilot durable write landing while a doc is being seeded could have the stale seed projected back over it on last-disconnect, clobbering the copilot edit even with no user changes. Gate server-side persistence on a genuine user edit (socket-origin update): a seed-only or merge-only doc is never projected back to the file (copilot writes the file durably itself), so it can't clobber a concurrent external write. --- apps/realtime/src/handlers/file-doc.test.ts | 36 +++++++++++++++++++++ apps/realtime/src/handlers/file-doc.ts | 21 +++++++++--- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index d5954b7d5f5..b483841bc58 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -260,6 +260,42 @@ describe('setupWorkspaceFileDocHandlers', () => { ) }) + it('does NOT persist a seeded-but-unedited doc on last disconnect (no clobber of a concurrent write)', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() // let the seed apply + + // Last collaborator leaves without ever editing — projecting this seed back over the file could + // clobber a concurrent copilot write, so the final flush must NOT persist. + cleanupFileDocForSocket('socket-1', io, true) + await flushMicrotasks() + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + }) + + it('persists on last disconnect once a genuine user edit has landed', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + + // A real user edit (socket-origin sync update) marks the doc dirty. + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'user typed this') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(edit)) + ) + ) + await flushMicrotasks() + + cleanupFileDocForSocket('socket-1', io, true) + await flushMicrotasks() + expect(mockFetchFileDocPersist).toHaveBeenCalled() + }) + it('joins the room, sends sync step 1, and seeds the document from the server', async () => { mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) const { io } = createIo() diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 596edb0e05e..415209097bc 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -105,6 +105,13 @@ interface FileDocRoom { workspaceId: string | null /** The last collaborator to edit here, for persist attribution (blob metadata) only. */ lastEditorUserId: string | null + /** + * True once a genuine USER edit has been applied here. Persistence is gated on it so a doc that was + * only seeded (or only received a copilot merge) is NEVER projected back over the file: copilot writes + * the file durably itself, and a seed captured from possibly-stale markdown must not clobber a + * concurrent external write. + */ + edited: boolean /** The pending debounced persist timer, if any. */ persistTimer: ReturnType | null } @@ -200,7 +207,8 @@ function schedulePersist(name: string, room: FileDocRoom): void { * caller destroys `room.doc` never encodes a destroyed doc, and the disabled path stays authoritative. */ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise { - if (!room.workspaceId || !room.lastEditorUserId) return + // Never project a doc no user actually edited back over the file (see {@link FileDocRoom.edited}). + if (!room.edited || !room.workspaceId || !room.lastEditorUserId) return const store = getFileDocStore() // Synchronous fallback capture — before any await, since the caller may destroy `room.doc` the moment // this yields. Only meaningful once seeded; used only when the authoritative stream state is absent. @@ -456,6 +464,7 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { serverSeedStarted: false, workspaceId: null, lastEditorUserId: null, + edited: false, persistTimer: null, } // Register synchronously BEFORE the async catch-up so a concurrent join sees this room, not a second. @@ -472,9 +481,13 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { // in the stream) and SEED_ORIGIN — the seed is published EXPLICITLY and AWAITED under the seed lock // (so it lands before the lock releases), which a fire-and-forget publish here couldn't guarantee. if (origin !== REDIS_ORIGIN && origin !== SEED_ORIGIN) getFileDocStore().publish(name, update) - // Persist real edits (user edits + copilot merges) back to markdown, debounced. Skip the seed (it - // is the file's current content) and stream-relayed updates (their originating task persists them). - if (origin !== REDIS_ORIGIN && origin !== SEED_ORIGIN) schedulePersist(name, room) + // Persist ONLY genuine USER edits (socket origin), debounced. A seed or a bare copilot merge must + // NOT project back over the file — copilot writes the file durably itself, so persisting a + // seeded-but-unedited doc (built from possibly-stale markdown) could clobber that concurrent write. + if (originSocketId(origin)) { + room.edited = true + schedulePersist(name, room) + } }) awareness.on('update', ({ added, updated, removed }: AwarenessChange, origin: unknown) => { From be2c9ad64a84d2cd8548605ab847ab1532d19834 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 18:47:10 -0700 Subject: [PATCH 42/53] fix(collab-doc): close review-round race/durability gaps Address Greptile P1 + Cursor findings: - Seed publishes to the shared stream AWAITED *before* seeding the local doc, so a publish failure leaves the doc unseeded and the stream empty for a clean retry rather than serving an unpublished local seed a peer would re-seed over (split-brain). - flushPersist falls back to the synchronously-captured local snapshot when getStreamState throws (not only when it returns null), so a transient Redis read no longer drops the final durable write as the room is torn down. - streamHasContent fails CLOSED (returns true on xLen error): a Redis blip can no longer let the seed fence pass and double-seed. - Client collabReady initializes from the collaborative prop, so a collaborative editor never has a mount-window where client autosave could clobber the server write. --- apps/realtime/src/handlers/file-doc-store.ts | 13 +++-- apps/realtime/src/handlers/file-doc.test.ts | 4 +- apps/realtime/src/handlers/file-doc.ts | 47 ++++++++++++++----- .../rich-markdown-editor.tsx | 6 ++- 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index dde15206eda..46138441e4b 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -268,15 +268,20 @@ export class FileDocStore { /** * Whether the file's stream already holds content — fences a seed apply against a peer that seeded - * while this task held a (possibly stale) lock, so we never write a second seed (split-brain). False - * when disabled or on error (the caller then proceeds under its lock, as before). + * while this task held a (possibly stale) lock, so we never write a second seed (split-brain). Both + * callers treat `true` as "do not seed", so this fails CLOSED: a Redis `xLen` error returns `true` + * (cannot confirm the stream is empty → do not risk a double-seed). `false` only when genuinely empty, + * or when disabled (single-replica, where seeding locally is always correct). */ async streamHasContent(name: string): Promise { if (!this.enabled || !this.write) return false try { return (await this.write.xLen(streamKey(name))) > 0 - } catch { - return false + } catch (error) { + logger.warn(`FileDocStore streamHasContent failed for ${name}`, { + error: getErrorMessage(error), + }) + return true } } diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index b483841bc58..8608d2691c7 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -121,8 +121,8 @@ const FILE_DOC_FIELD = 'default' /** Let a fire-and-forget `void ensureServerSeed(...)` chain settle (mock resolves synchronously). */ async function flushMicrotasks(): Promise { - await Promise.resolve() - await Promise.resolve() + // Enough to drain the fire-and-forget seed chain (shouldSeed → fetch → fence → publish → apply). + for (let i = 0; i < 8; i++) await Promise.resolve() } /** diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 415209097bc..e17e1e5879c 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -216,7 +216,19 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr try { if (!final && !(await store.acquirePersistSlot(name, FILE_DOC_TIMEOUTS.persistRequestMs))) return - const docState = (store.enabled ? await store.getStreamState(name) : null) ?? localState + let docState = localState + if (store.enabled) { + try { + docState = (await store.getStreamState(name)) ?? localState + } catch (streamError) { + // A transient Redis read must NOT drop the write when we already hold a valid local snapshot — + // else the last-disconnect flush loses the session's edits as the room is torn down. + if (!localState) throw streamError + logger.warn(`Stream state unavailable for file ${room.fileId}; persisting local snapshot`, { + error: getErrorMessage(streamError), + }) + } + } if (!docState) return // nothing seeded/authoritative to persist yet await fetchFileDocPersist(room.workspaceId, room.fileId, room.lastEditorUserId, docState) } catch (error) { @@ -332,18 +344,15 @@ async function ensureServerSeed( // Fence against a peer that seeded while we held a stale lock (a rare long stall): if the stream // already has content it is seeded, so never write a SECOND seed on top — that would split-brain. if (await store.streamHasContent(name)) return - // SEED_ORIGIN → `doc.on('update')` fans the seed to THIS task's clients but does NOT publish it - // (nor persist it — the seed is the file's current content). We publish it EXPLICITLY and AWAIT the - // stream write below, so the seed is durably in the stream BEFORE the lock releases — then any later - // seeder's `streamHasContent` fence is guaranteed to see it. Closes the fence's publish-after-release - // gap that a fire-and-forget publish left open. - if (update) Y.applyUpdate(room.doc, update, SEED_ORIGIN) - else - room.doc.transact( - () => room.doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true), - SEED_ORIGIN - ) - await store.publishAndWait(name, Y.encodeStateAsUpdate(room.doc)) + // Build the seed (file content + seed flag, or just the flag for an empty/missing file), then + // PUBLISH it to the shared stream AWAITED *before* seeding the local doc. The doc is marked seeded + // only once the seed is durably shared — so if the publish fails we leave the doc unseeded and the + // stream empty for a clean retry, rather than serving an unpublished local seed that a peer would + // re-seed on top of (split-brain). SEED_ORIGIN keeps `doc.on('update')` from re-publishing it. + const seedUpdate = update ?? emptySeedUpdate() + await store.publishAndWait(name, seedUpdate) + if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return + Y.applyUpdate(room.doc, seedUpdate, SEED_ORIGIN) } catch (error) { logger.warn(`Server seed failed for file ${room.fileId} (workspace ${workspaceId})`, error) room.serverSeedStarted = false @@ -352,6 +361,18 @@ async function ensureServerSeed( } } +/** The seed update for an empty/missing file: just the `initialContentLoaded` flag, so an empty doc + * still reaches readiness (and its emptiness is durably shared like any seed). */ +function emptySeedUpdate(): Uint8Array { + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + try { + return Y.encodeStateAsUpdate(doc) + } finally { + doc.destroy() + } +} + /** Serializes live merges per file so overlapping calls never race the same doc (see below). */ const fileDocMergeChains = new Map>() diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 21cbbe72ed2..0f3ac812476 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -126,8 +126,12 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ * shared document to markdown server-side, so the client must never also autosave — a stale keystroke * saving over a server/copilot edit is exactly the clobber the server path closes. The child reports * the right value up via `onCollabReadyChange`. + * + * Initialize from the `collaborative` prop (NOT unconditionally `true`): a collaborative file must + * start with autosave OFF, or a save could fire in the window before the child mounts and reports — + * re-clobbering exactly what this closes. The child turns it on for the non-collaborative fallback. */ - const [collabReady, setCollabReady] = useState(true) + const [collabReady, setCollabReady] = useState(!collaborative) const { content, From 2f1933cb3ce8a6edc859f7e94ac0ea1c2ea6d595 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 18:55:51 -0700 Subject: [PATCH 43/53] fix(collab-doc): unstick seed retry and persist tailed edits Cursor review: - ensureServerSeed clears serverSeedStarted when it aborts at the streamHasContent fence, so a fail-closed Redis xLen (or a genuine peer-seed) no longer strands the room unseeded with no retry. - Persistence dirty-tracking now marks a doc edited on any post-seed update, including a peer's edit relayed via the tailer (REDIS_ORIGIN), tracked via a seededObserved flag. The last task to leave persists real edits even if it only tailed them; the seed transition itself is still never counted, so a seeded-but-unedited doc is never projected back over the file. --- apps/realtime/src/handlers/file-doc.ts | 38 ++++++++++++++++++-------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index e17e1e5879c..c6765c3eeeb 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -106,12 +106,16 @@ interface FileDocRoom { /** The last collaborator to edit here, for persist attribution (blob metadata) only. */ lastEditorUserId: string | null /** - * True once a genuine USER edit has been applied here. Persistence is gated on it so a doc that was - * only seeded (or only received a copilot merge) is NEVER projected back over the file: copilot writes - * the file durably itself, and a seed captured from possibly-stale markdown must not clobber a - * concurrent external write. + * True once a genuine edit (a user edit — local OR a peer's, relayed via the tailer) has been applied + * here AFTER seeding. Persistence is gated on it so a doc that was only seeded is NEVER projected back + * over the file: the seed is captured from possibly-stale markdown and must not clobber a concurrent + * external write, whereas real edits are not otherwise durable and must be persisted by whichever task + * is last to leave — even one that only tailed the edits. */ edited: boolean + /** Whether this room has observed its doc become seeded — so a post-seed update counts as an edit but + * the seed transition itself does not. See the `doc.on('update')` edit-tracking below. */ + seededObserved: boolean /** The pending debounced persist timer, if any. */ persistTimer: ReturnType | null } @@ -343,7 +347,13 @@ async function ensureServerSeed( if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return // Fence against a peer that seeded while we held a stale lock (a rare long stall): if the stream // already has content it is seeded, so never write a SECOND seed on top — that would split-brain. - if (await store.streamHasContent(name)) return + if (await store.streamHasContent(name)) { + // Clear the guard so a later join can retry. Safe in both cases: a genuine peer-seed arrives via + // the tailer (and shouldSeed/this fence then no-op), and a fail-closed Redis `xLen` error must not + // strand the room unseeded forever. + room.serverSeedStarted = false + return + } // Build the seed (file content + seed flag, or just the flag for an empty/missing file), then // PUBLISH it to the shared stream AWAITED *before* seeding the local doc. The doc is marked seeded // only once the seed is durably shared — so if the publish fails we leave the doc unseeded and the @@ -486,6 +496,7 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { workspaceId: null, lastEditorUserId: null, edited: false, + seededObserved: false, persistTimer: null, } // Register synchronously BEFORE the async catch-up so a concurrent join sees this room, not a second. @@ -502,13 +513,16 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { // in the stream) and SEED_ORIGIN — the seed is published EXPLICITLY and AWAITED under the seed lock // (so it lands before the lock releases), which a fire-and-forget publish here couldn't guarantee. if (origin !== REDIS_ORIGIN && origin !== SEED_ORIGIN) getFileDocStore().publish(name, update) - // Persist ONLY genuine USER edits (socket origin), debounced. A seed or a bare copilot merge must - // NOT project back over the file — copilot writes the file durably itself, so persisting a - // seeded-but-unedited doc (built from possibly-stale markdown) could clobber that concurrent write. - if (originSocketId(origin)) { - room.edited = true - schedulePersist(name, room) - } + // Edit tracking for persistence. Mark the doc dirty on any update applied AFTER it was seeded — a + // local user edit (socket origin) OR a peer's edit relayed via the tailer (REDIS_ORIGIN) — so + // whichever task is last to leave persists real edits, even one that only tailed them. The seed + // transition itself is never counted (nor a purely-local copilot merge, already durable via + // copilot's direct file write), so a seeded-but-unedited doc is never projected back over the file. + const seededBefore = room.seededObserved + if (isDocSeeded(room.doc)) room.seededObserved = true + if (originSocketId(origin) || (seededBefore && origin === REDIS_ORIGIN)) room.edited = true + // Debounce a persist for LOCAL user edits only (peers debounce their own). + if (originSocketId(origin)) schedulePersist(name, room) }) awareness.on('update', ({ added, updated, removed }: AwarenessChange, origin: unknown) => { From 9476b21022e2ef641c308448b2876fe99e49cbf8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 19:07:50 -0700 Subject: [PATCH 44/53] fix(collab-doc): match client markdown post-processing on server persist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor review: yDocToFileMarkdown serialized the body with yDocToMarkdown only, but the editor save path runs postProcessSerializedMarkdown before applyFrontmatter. Server persist could therefore write markdown differing from a client save (empty list markers, callout un-escaping) — spurious blob churn / round-trip drift despite the byte-identical claim. Apply postProcessSerializedMarkdown in yDocToFileMarkdown so a server persist is byte-identical to a client save and the client's dirty-check baseline. Add a regression test guarding the composition. --- apps/sim/lib/collab-doc/converter.test.ts | 33 ++++++++++++++++++++++- apps/sim/lib/collab-doc/converter.ts | 18 +++++++++---- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/apps/sim/lib/collab-doc/converter.test.ts b/apps/sim/lib/collab-doc/converter.test.ts index 7ce8899057f..ba5606600b6 100644 --- a/apps/sim/lib/collab-doc/converter.test.ts +++ b/apps/sim/lib/collab-doc/converter.test.ts @@ -1,10 +1,20 @@ /** * @vitest-environment jsdom */ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' import { describe, expect, it } from 'vitest' import * as Y from 'yjs' +import { + applyFrontmatter, + postProcessSerializedMarkdown, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { serializeMarkdownBody } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' -import { applyMarkdownToYDoc, markdownToYDoc, yDocToMarkdown } from './converter' +import { + applyMarkdownToYDoc, + markdownToYDoc, + yDocToFileMarkdown, + yDocToMarkdown, +} from './converter' /** Representative markdown covering the custom-fidelity constructs (tables, code, lists, marks). */ const SAMPLES = [ @@ -74,4 +84,25 @@ describe('collab-doc converter', () => { expect(merged).toContain('Alpha paragraph. EDITED') expect(merged).toContain('expanded by the agent') }) + + it('yDocToFileMarkdown matches the client save composition (frontmatter + postProcess body pass)', () => { + // A server-side persist must be byte-identical to the editor save — `applyFrontmatter(frontmatter, + // postProcessSerializedMarkdown(body))` — or it drifts from a client save / the dirty-check baseline + // and churns the blob. This guards against the postProcess pass being dropped from the server path. + const frontmatter = '---\ntitle: Doc\n---\n' + const doc = markdownToYDoc('- one\n- two\n\n> [!NOTE]\n> hi') + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.frontmatterKey, frontmatter) + const expected = applyFrontmatter( + frontmatter, + postProcessSerializedMarkdown(yDocToMarkdown(doc)) + ) + expect(yDocToFileMarkdown(doc)).toBe(expected) + doc.destroy() + }) + + it('yDocToFileMarkdown re-attaches empty frontmatter when the config map has none', () => { + const doc = markdownToYDoc('plain body\n') + expect(yDocToFileMarkdown(doc)).toBe(postProcessSerializedMarkdown(yDocToMarkdown(doc))) + doc.destroy() + }) }) diff --git a/apps/sim/lib/collab-doc/converter.ts b/apps/sim/lib/collab-doc/converter.ts index 5557ffb2439..c3c5b9cc69c 100644 --- a/apps/sim/lib/collab-doc/converter.ts +++ b/apps/sim/lib/collab-doc/converter.ts @@ -9,7 +9,10 @@ import { } from '@tiptap/y-tiptap' import type * as Y from 'yjs' import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' -import { applyFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { + applyFrontmatter, + postProcessSerializedMarkdown, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { parseMarkdownToDoc, serializeDocToMarkdown, @@ -87,13 +90,18 @@ export function yDocToMarkdown(ydoc: Y.Doc): string { /** * Project a collaborative {@link Y.Doc} back to the file's FULL canonical markdown — the body from the - * CRDT re-joined with the frontmatter carried in the config map. This is exactly what the editor - * writes on save (`applyFrontmatter(resolveSaveFrontmatter(), body)`), so a server-side persist of the - * live doc is byte-identical to a client save — no spurious churn on the round-trip. + * CRDT re-joined with the frontmatter carried in the config map. Mirrors the editor's save path EXACTLY + * (`applyFrontmatter(resolveSaveFrontmatter(), postProcessSerializedMarkdown(editor.getMarkdown()))`), + * INCLUDING the `postProcessSerializedMarkdown` body fidelity pass (empty list markers, callout + * un-escaping, trailing whitespace) — so a server-side persist is byte-identical to a client save and + * matches the client's dirty-check baseline, with no spurious blob churn on the round-trip. */ export function yDocToFileMarkdown(ydoc: Y.Doc): string { const frontmatter = ydoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.frontmatterKey) - return applyFrontmatter(typeof frontmatter === 'string' ? frontmatter : '', yDocToMarkdown(ydoc)) + return applyFrontmatter( + typeof frontmatter === 'string' ? frontmatter : '', + postProcessSerializedMarkdown(yDocToMarkdown(ydoc)) + ) } /** From f62e99f293a7574466cee368fb3a63625239300c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 19:32:58 -0700 Subject: [PATCH 45/53] fix(collab-doc): close durability gaps at deploy boundaries + audit polish From a comprehensive from-scratch audit (correctness, SOTA, cleanliness, feature-completeness): - Persist max-wait: a continuous edit burst kept resetting the 5s debounce and never persisted; cap it so a burst flushes at least every 20s, bounding unpersisted edits. - Graceful-shutdown flush: flushAllFileDocRooms awaited in shutdown so a rolling deploy / scale-in secures open edited rooms to durable markdown before exit, instead of relying on the stream + a surviving task. - Compacted-snapshot catch-up now marks the doc edited (REDIS_SNAPSHOT_ORIGIN): a snapshot folds seed+edits into one frame, so a task catching up purely from it no longer treats real edits as an unedited seed and skips persisting. - Polish: delete dead __setFileDocStoreForTest; bounded retry loop; rename acquirePersistSlot -> tryClaimPersistWindow with accurate docs; tailer object-identity guard; fix stale comments (seed route, edit-content autosave). --- .../src/handlers/file-doc-store.test.ts | 5 ++ apps/realtime/src/handlers/file-doc-store.ts | 57 ++++++++++----- apps/realtime/src/handlers/file-doc.test.ts | 27 +++++++ apps/realtime/src/handlers/file-doc.ts | 72 ++++++++++++++----- apps/realtime/src/index.ts | 10 +++ .../app/api/internal/file-doc/seed/route.ts | 2 +- .../tools/server/files/edit-content.ts | 5 +- 7 files changed, 139 insertions(+), 39 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index b38fb5cdfce..64ebff4adc0 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -220,6 +220,11 @@ describe('FileDocStore', () => { Y.applyUpdate(doc, (await a.getStreamState(NAME))!) expect(doc.getText('body').toString()).toBe('PEER1PEER2') doc.destroy() + + // The appended snapshot entry must carry the snapshot marker, so a fresh catch-up task treats it as + // edited content (not a bare seed) and persists on last-disconnect. + const stream = state.backing!.streams.get(streamKey)! + expect(stream[stream.length - 1].message.s).toBe('1') }) it('retries a transient append failure so the edit is not lost from the shared log', async () => { diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 46138441e4b..39d2f4ff51e 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -56,14 +56,26 @@ const RELEASE_LOCK_SCRIPT = */ export const REDIS_ORIGIN = Symbol('file-doc-redis') +/** + * Origin for a COMPACTED SNAPSHOT applied from the stream. A snapshot folds the seed + all prior edits + * into one entry, so a fresh task catching up from it would otherwise never see a separate post-seed + * edit frame and would treat the doc as unedited. The relay's edit-tracker uses this origin to mark the + * doc edited (a snapshot only exists after the stream crossed the compaction threshold, i.e. real edits + * happened). Behaves like {@link REDIS_ORIGIN} otherwise (already in the stream — never re-published). + */ +export const REDIS_SNAPSHOT_ORIGIN = Symbol('file-doc-redis-snapshot') + const STREAM_PREFIX = 'filedoc:stream:' const SEED_LOCK_PREFIX = 'filedoc:seedlock:' const COMPACT_LOCK_PREFIX = 'filedoc:compactlock:' const PERSIST_LOCK_PREFIX = 'filedoc:persistlock:' const MERGE_LOCK_PREFIX = 'filedoc:mergelock:' -/** The single field each stream entry carries — a base64 Yjs update. */ +/** The field each stream entry carries — a base64 Yjs update. */ const UPDATE_FIELD = 'u' +/** Marks a stream entry as a compaction SNAPSHOT (folds seed + edits), so the tailer applies it with + * {@link REDIS_SNAPSHOT_ORIGIN}. Present only on snapshot entries. */ +const SNAPSHOT_FIELD = 's' /** Sentinel token a DISABLED store returns from a lock acquire, so single-replica callers proceed * without special-casing; {@link FileDocStore.releaseLock} treats it as a no-op. Not a real UUID, so it @@ -228,12 +240,12 @@ export class FileDocStore { private async appendUpdate(name: string, update: Uint8Array): Promise { if (!this.write) return const encoded = Buffer.from(update).toString('base64') - for (let attempt = 0; ; attempt++) { + for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { await this.write.xAdd(streamKey(name), '*', { [UPDATE_FIELD]: encoded }) break } catch (error) { - if (attempt >= PUBLISH_MAX_RETRIES) { + if (attempt === PUBLISH_MAX_RETRIES) { logger.error(`FileDocStore append failed for ${name}`, { error: getErrorMessage(error) }) throw error } @@ -352,12 +364,14 @@ export class FileDocStore { } /** - * Try to claim the right to persist this file for the current debounce window, so concurrent tasks - * editing the same file don't each write a redundant blob version. Returns true (proceed) when - * disabled, or when this task wins a short lock. The final last-collaborator flush does NOT gate on - * this — it must always write. + * A best-effort TTL dedup WINDOW (NOT a lock): claim the right to run a debounced persist for the next + * `ttlMs`, so concurrent tasks editing the same file don't each write a redundant blob version. It is + * never released — it simply expires after `ttlMs`, gating the debounced persist to ~once per window + * cluster-wide. Fails OPEN (returns true on a Redis error): a redundant persist is a harmless + * idempotent write, so it must never block a real one. The final last-collaborator flush does NOT gate + * on this — it must always write. */ - async acquirePersistSlot(name: string, ttlMs: number): Promise { + async tryClaimPersistWindow(name: string, ttlMs: number): Promise { if (!this.enabled || !this.write) return true try { const won = await this.write.set(`${PERSIST_LOCK_PREFIX}${name}`, '1', { @@ -388,7 +402,10 @@ export class FileDocStore { private applyEntry(room: StoreRoom, id: string, message: Record): void { room.lastId = id - applyEntryToDoc(room.doc, id, message, REDIS_ORIGIN) + // A compaction snapshot folds seed + edits into one frame; stamp it so the relay's edit-tracker + // treats a fresh catch-up from it as edited (a snapshot only exists once real edits accumulated). + const origin = message[SNAPSHOT_FIELD] ? REDIS_SNAPSHOT_ORIGIN : REDIS_ORIGIN + applyEntryToDoc(room.doc, id, message, origin) } /** @@ -397,21 +414,24 @@ export class FileDocStore { */ private async runReader(): Promise { while (this.running && this.read) { - const snapshot = [...this.rooms.entries()] - if (snapshot.length === 0) { + const snapshot = new Map(this.rooms) + if (snapshot.size === 0) { await sleep(IDLE_POLL_MS) continue } try { const res = await this.read.xRead( - snapshot.map(([name, room]) => ({ key: streamKey(name), id: room.lastId })), + [...snapshot].map(([name, room]) => ({ key: streamKey(name), id: room.lastId })), { BLOCK: READ_BLOCK_MS, COUNT: READ_COUNT } ) if (!res) continue for (const stream of res) { const name = stream.name.slice(STREAM_PREFIX.length) const room = this.rooms.get(name) - if (!room) continue // detached mid-read; its doc is being destroyed + // Skip if detached mid-read, OR replaced by a close→reopen (a DIFFERENT StoreRoom): applying + // entries read against the OLD room's lastId to the new one could regress its lastId (harmless + // but wasteful re-delivery). The new room caught itself up via xRange already. + if (!room || room !== snapshot.get(name)) continue for (const entry of stream.messages) this.applyEntry(room, entry.id, entry.message) } } catch (error) { @@ -446,7 +466,11 @@ export class FileDocStore { // appended snapshot id instead would silently drop those un-integrated peer entries. const upTo = room.lastId const snapshot = Buffer.from(Y.encodeStateAsUpdate(room.doc)).toString('base64') - await this.write.xAdd(streamKey(name), '*', { [UPDATE_FIELD]: snapshot }) + // Mark it a snapshot so a fresh catch-up task treats it as edited content, not a bare seed. + await this.write.xAdd(streamKey(name), '*', { + [UPDATE_FIELD]: snapshot, + [SNAPSHOT_FIELD]: '1', + }) // MINID keeps entries with id >= upTo: the snapshot, any un-integrated peer entries, and // `upTo` itself (redundant with the snapshot, harmless); it drops only the folded older deltas. await this.write.xTrim(streamKey(name), 'MINID', upTo) @@ -486,8 +510,3 @@ export function getFileDocStore(): FileDocStore { if (!store) store = new FileDocStore(undefined) return store } - -/** Test-only: reset the singleton so a test can install its own instance. */ -export function __setFileDocStoreForTest(next: FileDocStore | null): void { - store = next -} diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 8608d2691c7..83c4c0e144a 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -35,6 +35,7 @@ vi.mock('@/handlers/file-doc-app', () => ({ import { applyMarkdownToLiveFileDoc, cleanupFileDocForSocket, + flushAllFileDocRooms, setupWorkspaceFileDocHandlers, } from '@/handlers/file-doc' @@ -296,6 +297,32 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(mockFetchFileDocPersist).toHaveBeenCalled() }) + it('flushAllFileDocRooms persists open EDITED rooms (graceful shutdown), skips unedited', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + + // Seed-only room: a graceful-shutdown flush must NOT persist it. + mockFetchFileDocPersist.mockClear() + await flushAllFileDocRooms() + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + + // After a real user edit, the same flush persists (edits would otherwise be lost on deploy). + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'typed') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(edit)) + ) + ) + await flushMicrotasks() + mockFetchFileDocPersist.mockClear() + await flushAllFileDocRooms() + expect(mockFetchFileDocPersist).toHaveBeenCalled() + }) + it('joins the room, sends sync step 1, and seeds the document from the server', async () => { mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) const { io } = createIo() diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index c6765c3eeeb..127f748dd86 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -45,7 +45,7 @@ import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' import { resolveAvatarUrl } from '@/handlers/avatar' import { fetchFileDocMerge, fetchFileDocPersist, fetchFileDocSeed } from '@/handlers/file-doc-app' -import { getFileDocStore, REDIS_ORIGIN } from '@/handlers/file-doc-store' +import { getFileDocStore, REDIS_ORIGIN, REDIS_SNAPSHOT_ORIGIN } from '@/handlers/file-doc-store' import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -61,6 +61,10 @@ const SEED_ORIGIN = Symbol('file-doc-seed') /** Debounce window for the server-side project-to-markdown persist while a doc is actively edited. */ const PERSIST_DEBOUNCE_MS = 5_000 +/** Max-wait cap on the persist debounce: a CONTINUOUS edit burst keeps resetting the 5s debounce and + * would otherwise never persist until an idle pause, so force a flush at least this often — bounding how + * many edits are unpersisted (in the stream only) if the task dies mid-burst. */ +const PERSIST_MAX_WAIT_MS = 20_000 /** Cross-task merge lock. The TTL must exceed the whole critical section it guards — stream fold + * `fetchFileDocMerge` (bounded at `mergeRequestMs`) + the awaited publish — so the lock never expires @@ -118,6 +122,9 @@ interface FileDocRoom { seededObserved: boolean /** The pending debounced persist timer, if any. */ persistTimer: ReturnType | null + /** Absolute time (ms) by which a debounced persist must fire even under continuous editing (the + * max-wait cap); null when no persist is pending. */ + persistDeadline: number | null } /** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */ @@ -184,24 +191,29 @@ function broadcastLocal( /** * Schedule a debounced server-side persist of the live doc back to durable markdown. Coalesces rapid - * edits; a no-op until the room knows its workspace (set at join). The final flush on last-disconnect - * is separate ({@link flushPersist} with `final`). + * edits; a no-op until the room knows its workspace (set at join). A {@link PERSIST_MAX_WAIT_MS} + * max-wait caps the debounce so a continuous burst still persists periodically. The final flush on + * last-disconnect is separate ({@link flushPersist} with `final`). */ function schedulePersist(name: string, room: FileDocRoom): void { if (!room.workspaceId || !room.lastEditorUserId) return + const now = Date.now() + if (room.persistDeadline === null) room.persistDeadline = now + PERSIST_MAX_WAIT_MS if (room.persistTimer) clearTimeout(room.persistTimer) + const delay = Math.max(0, Math.min(PERSIST_DEBOUNCE_MS, room.persistDeadline - now)) room.persistTimer = setTimeout(() => { room.persistTimer = null + room.persistDeadline = null void flushPersist(name, room, false) - }, PERSIST_DEBOUNCE_MS) + }, delay) } /** * Project the live doc to markdown and write it durably via the app. `final` (last collaborator - * leaving) always writes; a debounced mid-edit flush first claims a cross-task slot (held for the whole - * write, so a concurrent task can't issue an overlapping blob write) so tasks don't each write a - * redundant version. Best-effort: never throws (a failure is retried on the next debounce; the stream - * holds the state meanwhile). + * leaving) always writes; a debounced mid-edit flush first claims a best-effort cross-task dedup WINDOW + * (a TTL key that just expires, so at most ~one persist per window cluster-wide) so concurrent tasks + * editing the same file don't each write a redundant blob version. Best-effort: never throws (a failure + * is retried on the next debounce; the stream holds the state meanwhile). * * Persists the AUTHORITATIVE shared state (the stream), not this task's local doc: a copilot merge — or * a peer's edit — published by another task may not be integrated into `room.doc` yet (and the stream @@ -218,7 +230,7 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr // this yields. Only meaningful once seeded; used only when the authoritative stream state is absent. const localState = isDocSeeded(room.doc) ? Y.encodeStateAsUpdate(room.doc) : null try { - if (!final && !(await store.acquirePersistSlot(name, FILE_DOC_TIMEOUTS.persistRequestMs))) + if (!final && !(await store.tryClaimPersistWindow(name, FILE_DOC_TIMEOUTS.persistRequestMs))) return let docState = localState if (store.enabled) { @@ -294,6 +306,7 @@ function awarenessUpdateClientIds(update: Uint8Array): number[] { function destroyRoomIfIdle(name: string) { const room = fileDocRooms.get(name) if (!room || room.owners.size > 0) return + room.persistDeadline = null if (room.persistTimer) { clearTimeout(room.persistTimer) room.persistTimer = null @@ -307,6 +320,21 @@ function destroyRoomIfIdle(name: string) { fileDocRooms.delete(name) } +/** + * Flush every open, edited room's converged doc to durable markdown, AWAITING the writes. Called on + * graceful shutdown (rolling deploy / scale-in) so edits since the last debounce aren't left only in the + * ephemeral stream — the per-socket disconnect flush is fire-and-forget and would race `process.exit`. + * Best-effort and bounded by each persist's own timeout; never throws. Rooms are NOT torn down here (the + * process is exiting); only their durable state is secured. + */ +export async function flushAllFileDocRooms(): Promise { + const flushes: Promise[] = [] + for (const [name, room] of fileDocRooms) { + if (room.edited) flushes.push(flushPersist(name, room, true)) + } + await Promise.all(flushes) +} + /** * Seed a room's document server-side, once, on the first join: ask the app to build the seed (the * file's current markdown → Yjs, through the exact editor engine) and apply it, which relays the @@ -498,6 +526,7 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { edited: false, seededObserved: false, persistTimer: null, + persistDeadline: null, } // Register synchronously BEFORE the async catch-up so a concurrent join sees this room, not a second. fileDocRooms.set(name, room) @@ -509,18 +538,27 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { // Fan out to THIS task's clients only (excluding the origin socket if local). Cross-task delivery // rides the shared stream — every task's tailer applies + runs its own local fan-out. broadcastLocal(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) - // Share every locally-originated update to the stream so peers converge. Skip REDIS_ORIGIN (already - // in the stream) and SEED_ORIGIN — the seed is published EXPLICITLY and AWAITED under the seed lock - // (so it lands before the lock releases), which a fire-and-forget publish here couldn't guarantee. - if (origin !== REDIS_ORIGIN && origin !== SEED_ORIGIN) getFileDocStore().publish(name, update) + // Share every locally-originated update to the stream so peers converge. Skip updates that already + // came FROM the stream (REDIS_ORIGIN / REDIS_SNAPSHOT_ORIGIN) and SEED_ORIGIN — the seed is published + // EXPLICITLY and AWAITED under the seed lock (so it lands before the lock releases), which a + // fire-and-forget publish here couldn't guarantee. + if (origin !== REDIS_ORIGIN && origin !== REDIS_SNAPSHOT_ORIGIN && origin !== SEED_ORIGIN) + getFileDocStore().publish(name, update) // Edit tracking for persistence. Mark the doc dirty on any update applied AFTER it was seeded — a // local user edit (socket origin) OR a peer's edit relayed via the tailer (REDIS_ORIGIN) — so - // whichever task is last to leave persists real edits, even one that only tailed them. The seed - // transition itself is never counted (nor a purely-local copilot merge, already durable via - // copilot's direct file write), so a seeded-but-unedited doc is never projected back over the file. + // whichever task is last to leave persists real edits, even one that only tailed them. A compaction + // snapshot on catch-up (REDIS_SNAPSHOT_ORIGIN) also counts: it folds real edits into one frame, so a + // fresh task catching up purely from it must not treat the doc as unedited. The seed transition + // itself is never counted (nor a purely-local copilot merge, already durable via copilot's direct + // file write), so a seeded-but-unedited doc is never projected back over the file. const seededBefore = room.seededObserved if (isDocSeeded(room.doc)) room.seededObserved = true - if (originSocketId(origin) || (seededBefore && origin === REDIS_ORIGIN)) room.edited = true + if ( + originSocketId(origin) || + origin === REDIS_SNAPSHOT_ORIGIN || + (seededBefore && origin === REDIS_ORIGIN) + ) + room.edited = true // Debounce a persist for LOCAL user edits only (peers debounce their own). if (originSocketId(origin)) schedulePersist(name, room) }) diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index 910e92a75c9..8a56dceb4eb 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -6,6 +6,7 @@ import { createSocketIOServer, shutdownSocketIOAdapter } from '@/config/socket' import { assertSchemaCompatibility } from '@/database/preflight' import { env } from '@/env' import { setupAllHandlers } from '@/handlers' +import { flushAllFileDocRooms } from '@/handlers/file-doc' import { getFileDocStore, initFileDocStore } from '@/handlers/file-doc-store' import { type AuthenticatedSocket, authenticateSocket } from '@/middleware/auth' import { type IRoomManager, MemoryRoomManager, RedisRoomManager } from '@/rooms' @@ -116,6 +117,15 @@ async function main() { accessRevalidation.stop() + // Flush open collaborative docs to durable markdown BEFORE tearing down Redis/the store — the + // per-socket disconnect flush is fire-and-forget and would race process exit. + try { + await flushAllFileDocRooms() + logger.info('Flushed open collaborative documents') + } catch (error) { + logger.error('Error flushing collaborative documents on shutdown:', error) + } + try { await roomManager.shutdown() logger.info('RoomManager shutdown complete') diff --git a/apps/sim/app/api/internal/file-doc/seed/route.ts b/apps/sim/app/api/internal/file-doc/seed/route.ts index b6cbc83bfd0..058e7a0003a 100644 --- a/apps/sim/app/api/internal/file-doc/seed/route.ts +++ b/apps/sim/app/api/internal/file-doc/seed/route.ts @@ -14,7 +14,7 @@ const logger = createLogger('FileDocSeedAPI') * POST /api/internal/file-doc/seed — build a server-authoritative collaborative-document seed * (markdown → Yjs) for the realtime relay to apply on room creation. Internal only: gated on the * shared `x-api-key: INTERNAL_API_SECRET` secret, matching the header the realtime relay sends - * (`apps/realtime/src/handlers/file-doc-seed.ts`) and the realtime server's own inbound validator. + * (`apps/realtime/src/handlers/file-doc-app.ts`) and the realtime server's own inbound validator. */ export const POST = withRouteHandler(async (request: NextRequest) => { const auth = checkInternalApiKey(request) diff --git a/apps/sim/lib/copilot/tools/server/files/edit-content.ts b/apps/sim/lib/copilot/tools/server/files/edit-content.ts index 79ae7f9c23d..6997c9ae262 100644 --- a/apps/sim/lib/copilot/tools/server/files/edit-content.ts +++ b/apps/sim/lib/copilot/tools/server/files/edit-content.ts @@ -246,8 +246,9 @@ export const editContentServerTool: BaseServerTool Date: Tue, 28 Jul 2026 20:07:13 -0700 Subject: [PATCH 46/53] improvement(realtime): post-review fixes for collab dirty-state + relay lifecycle - collab editor no longer latches a spurious 'Unsaved changes' prompt: report dirty only when the client owns durability (canAutosave), since in a collaborative session the relay persists the doc server-side - guard shutdown against a double SIGINT/SIGTERM running teardown twice - disconnect local sockets before httpServer.close so shutdown exits gracefully instead of hitting the forced-exit timer (local-only, deploy-safe) - return 400 (not silent 200) on an invalid workspaceId in the files-changed fanout - clear a pending join-retry timer on reconnect so it can't fire a duplicate join --- apps/realtime/src/index.ts | 15 +++++++++++++++ apps/realtime/src/routes/http.ts | 13 ++++++------- .../file-viewer/use-editable-file-content.ts | 13 ++++++++++--- .../files/hooks/use-workspace-files-room.ts | 7 ++++++- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index 8a56dceb4eb..80663141334 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -112,7 +112,13 @@ async function main() { logger.info(`Health check available at: http://localhost:${PORT}/health`) }) + let shuttingDown = false const shutdown = async () => { + // SIGINT and SIGTERM both bind this; a double signal (or SIGTERM then SIGINT during the drain) + // must not run the whole teardown twice — that means a second forced-exit timer and a second + // Redis quit (which throws "The client is closed"). + if (shuttingDown) return + shuttingDown = true logger.info('Shutting down Socket.IO server...') accessRevalidation.stop() @@ -145,6 +151,15 @@ async function main() { logger.error('Error during FileDocStore shutdown:', error) } + // Close local client connections so `httpServer.close()` can complete its callback and exit + // gracefully — otherwise open websockets keep it hanging until the forced-exit timer below. + // Local-only: a rolling deploy must not disconnect clients pinned to other pods. + try { + io.local.disconnectSockets(true) + } catch (error) { + logger.error('Error disconnecting sockets on shutdown:', error) + } + httpServer.close(() => { logger.info('Socket.IO server closed') process.exit(0) diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index dcde36a0609..971162951f0 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -169,13 +169,12 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workspaceId } = JSON.parse(body) - if (typeof workspaceId === 'string' && workspaceId.length > 0) { - roomManager.emitToRoom( - { type: ROOM_TYPES.WORKSPACE_FILES, id: workspaceId }, - 'workspace-files-changed', - { workspaceId, timestamp: Date.now() } - ) - } + if (!isNonEmptyString(workspaceId)) return sendError(res, 'Invalid workspaceId', 400) + roomManager.emitToRoom( + { type: ROOM_TYPES.WORKSPACE_FILES, id: workspaceId }, + 'workspace-files-changed', + { workspaceId, timestamp: Date.now() } + ) sendSuccess(res) } catch (error) { logger.error('Error handling workspace files changed notification:', error) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts index 5e1b07d63a5..f8dac76bbdd 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts @@ -257,9 +257,16 @@ export function useEditableFileContent({ ), }) + // When the client can't autosave it isn't the durability owner: the collaborative editor holds + // `canAutosave` permanently false because the relay persists the doc server-side (debounced + on + // last-disconnect), so `savedContent` never advances and raw `isDirty` would latch true after any + // local OR remote keystroke — surfacing a spurious "Unsaved changes" navigation prompt whose + // "Discard" discards nothing real. With nothing the user can save, there is nothing to warn about. + const isDirtyForCaller = canAutosave && isDirty + useEffect(() => { - onDirtyChangeRef.current?.(isDirty) - }, [isDirty]) + onDirtyChangeRef.current?.(isDirtyForCaller) + }, [isDirtyForCaller]) useEffect(() => { onSaveStatusChangeRef.current?.( @@ -307,6 +314,6 @@ export function useEditableFileContent({ hasContentError: streamingContent === undefined && Boolean(error) && !isInitialized, saveStatus, saveImmediately, - isDirty, + isDirty: isDirtyForCaller, } } diff --git a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts index 49ff31dcabb..34cda7dade0 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts @@ -41,9 +41,14 @@ export function useWorkspaceFilesRoom(workspaceId: string): void { const join = () => socket.emit('join-workspace-files', { workspaceId }) // A fresh (re)connect gets a fresh retry budget, so a prior full exhaustion doesn't leave the - // socket unable to retry a failed re-join until the next success. + // socket unable to retry a failed re-join until the next success. Cancel any retry still pending + // from before the reconnect so it can't fire a duplicate join after this immediate one. const handleConnect = () => { retries = 0 + if (retryTimer) { + clearTimeout(retryTimer) + retryTimer = null + } join() } From ced03b35dfc07a4247a4dde199b89a33d2dc61e2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 21:13:22 -0700 Subject: [PATCH 47/53] fix(realtime): make collab-doc seeding atomic to close split-brain window An adversarial concurrency audit found a split-brain double-seed vector: the seed used an advisory SET NX PX lock + a SEPARATE xLen fence + an unconditional xAdd (a non-atomic check-then-append). If the seed lock's TTL expired mid-seed (a >4s stall after the 8s seed fetch), a second task could acquire the freed lock over a still-empty stream, both fences read empty, and both append seeds with distinct Yjs client ids -> duplicated document content. - add an atomic SEED_IF_EMPTY_SCRIPT (append-iff-empty in one Redis step) + store.seedIfEmpty(); the emptiness check and append are now inseparable, so two tasks racing (even both past an expired lock) can never both seed - ensureServerSeed uses seedIfEmpty instead of streamHasContent-fence + publishAndWait; the seed lock is now purely an efficiency optimization (avoid a duplicate fetch), not a correctness dependency - fix the misleading comment that claimed a copilot merge is not counted as an edit in the multi-replica path (it round-trips as REDIS_ORIGIN and does count; a safe idempotent over-persist, never a lost edit) - add interleaving tests: seedIfEmpty atomicity/fence, the split-brain regression under an expired lock, a peer edit during attach catch-up, and concurrent two-task compaction --- .../src/handlers/file-doc-store.test.ts | 118 +++++++++++++++++- apps/realtime/src/handlers/file-doc-store.ts | 65 ++++++++-- apps/realtime/src/handlers/file-doc.ts | 40 +++--- 3 files changed, 192 insertions(+), 31 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index 64ebff4adc0..7a6aba13015 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -71,9 +71,21 @@ function makeClient(): any { b().kv.delete(key) return 1 }, - // Compare-and-delete Lua (RELEASE_LOCK_SCRIPT): del only if the stored value matches the token. - eval: async (_script: string, opts: { keys: string[]; arguments: string[] }) => { + eval: async (script: string, opts: { keys: string[]; arguments: string[] }) => { const [key] = opts.keys + // Atomic seed-if-empty (SEED_IF_EMPTY_SCRIPT): append the entry iff the stream is empty, in one + // synchronous step — mirroring Redis's atomic Lua execution, so two concurrent evals can never both + // append (the second sees a non-empty stream). + if (script.includes('xlen')) { + const [field, value] = opts.arguments + const arr = b().streams.get(key) ?? [] + if (arr.length > 0) return 0 + const id = `${++b().seq}-0` + arr.push({ id, message: { [field]: value } }) + b().streams.set(key, arr) + return 1 + } + // Compare-and-delete Lua (RELEASE_LOCK_SCRIPT): del only if the stored value matches the token. const [token] = opts.arguments if (b().kv.get(key) === token) { b().kv.delete(key) @@ -277,4 +289,106 @@ describe('FileDocStore', () => { expect(doc.getText('body').toString()).toBe('') doc.destroy() }) + + it('seedIfEmpty writes the seed once and reports it, then refuses a non-empty stream', async () => { + const a = await newStore() + expect(await a.seedIfEmpty(NAME, updateFor('first'))).toBe(true) + // A second seed attempt (any task) must be refused — the stream already holds content. + const b = await newStore() + expect(await b.seedIfEmpty(NAME, updateFor('second'))).toBe(false) + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(doc.getText('body').toString()).toBe('first') + doc.destroy() + }) + + it('atomic seed prevents split-brain even when the seed lock expired mid-seed', async () => { + // The exact split-brain precondition from the concurrency audit: the seed lock is only an efficiency + // optimization, so if it lapses (TTL) while the stream is still empty, TWO tasks can both hold a + // token and both try to seed with DIFFERENT docs (distinct Yjs client ids). The atomic seedIfEmpty + // must still let only one land — otherwise the union duplicates content. + const a = await newStore() + const b = await newStore() + const tokenA = await a.shouldSeed(NAME) + expect(tokenA).toBeTruthy() + // Simulate A's lock expiring mid-seed so B also wins the freed lock over a still-empty stream. + state.backing!.kv.delete(`filedoc:seedlock:${NAME}`) + const tokenB = await b.shouldSeed(NAME) + expect(tokenB).toBeTruthy() + // Both tasks now race to seed with distinct client ids. + const [seededA, seededB] = await Promise.all([ + a.seedIfEmpty(NAME, updateFor('SEED-A')), + b.seedIfEmpty(NAME, updateFor('SEED-B')), + ]) + expect([seededA, seededB].filter(Boolean)).toHaveLength(1) + // Exactly one seed is in the stream — the reconstructed text is a single seed, never a duplicated + // union of both (e.g. 'SEED-ASEED-B'). + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(['SEED-A', 'SEED-B']).toContain(doc.getText('body').toString()) + doc.destroy() + }) + + it('a peer edit published during attachRoom catch-up is not lost', async () => { + // Author two INCREMENTAL edits from one doc so they converge to 'basepeer' (not an independent union). + const author = new Y.Doc() + const updates: Uint8Array[] = [] + author.on('update', (u: Uint8Array) => updates.push(u)) + author.getText('body').insert(0, 'base') + author.getText('body').insert(4, 'peer') + + const a = await newStore() + a.publish(NAME, updates[0]) // 'base' + await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) + + // Task B attaches; while its synchronous catch-up runs, task A publishes the second edit. The tailer + // resumes from the id catch-up stopped at, so the edit converges rather than falling into a gap. + const b = await newStore() + const bDoc = new Y.Doc() + const attach = b.attachRoom(NAME, bDoc) + a.publish(NAME, updates[1]) // 'peer' appended + await attach + await vi.waitFor(() => expect(bDoc.getText('body').toString()).toBe('basepeer'), { + timeout: 2000, + }) + bDoc.destroy() + author.destroy() + }) + + it('concurrent compaction on two tasks preserves the full document', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(Y.encodeStateAsUpdate(new Y.Doc())).toString('base64') + + // Two peer edits neither compacting task has integrated (id > each task's lastId). + const peerDoc = new Y.Doc() + const peerUpdates: Uint8Array[] = [] + peerDoc.on('update', (u: Uint8Array) => peerUpdates.push(u)) + peerDoc.getText('body').insert(0, 'PEER1') + peerDoc.getText('body').insert(5, 'PEER2') + + const entries = Array.from({ length: 400 }, (_, i) => ({ + id: `${i + 1}-0`, + message: { u: noop }, + })) + entries.push({ id: '401-0', message: { u: Buffer.from(peerUpdates[0]).toString('base64') } }) + entries.push({ id: '402-0', message: { u: Buffer.from(peerUpdates[1]).toString('base64') } }) + state.backing!.streams.set(streamKey, entries) + state.backing!.seq = 402 + + // Two tasks whose local docs lag at DIFFERENT points (400 and 401): both cross the threshold and + // compact concurrently. Each must only trim what its own snapshot subsumes, so the union of both + // snapshots plus the un-integrated peer entries still reconstructs the whole doc. + const a = await newStore() + const b = await newStore() + const docA = new Y.Doc() + Y.applyUpdate(docA, peerUpdates[0]) // A integrated up to 401 + ;(a as any).rooms.set(NAME, { doc: docA, lastId: '401-0', publishes: 0 }) + ;(b as any).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0 }) + await Promise.all([(a as any).maybeCompact(NAME), (b as any).maybeCompact(NAME)]) + + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(doc.getText('body').toString()).toBe('PEER1PEER2') + doc.destroy() + }) }) diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 39d2f4ff51e..27dbe8bfe01 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -48,6 +48,17 @@ const logger = createLogger('FileDocStore') const RELEASE_LOCK_SCRIPT = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end" +/** + * Atomic seed: append the seed entry ONLY if the stream is still empty, in one Redis-side step. This is + * the real split-brain guard — two tasks racing (even both past an expired seed lock) can never both + * write a seed (each would mint a distinct Yjs client id → duplicated content), because the emptiness + * check and the append happen atomically with no check-then-append window. The seed lock is only an + * efficiency optimization (avoid two seed fetches); correctness does not depend on it staying held. + * Returns 1 if THIS call wrote the seed, 0 if the stream already had content. + */ +const SEED_IF_EMPTY_SCRIPT = + "if redis.call('xlen', KEYS[1]) == 0 then redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]); return 1 else return 0 end" + /** * The transaction origin the store stamps on updates it applies from the stream. The relay's * `doc.on('update')` handler uses it to distinguish an update that ARRIVED from a peer (fan out to @@ -101,12 +112,12 @@ const COMPACT_LOCK_TTL_MS = 10_000 /** Retry a failed stream append this many times before giving up, so a transient Redis blip doesn't * silently drop an edit from the shared log (which no peer would then ever see). */ const PUBLISH_MAX_RETRIES = 3 -/** The seed lock spans the app seed fetch (hard-bounded at `seedRequestMs = 8s`) + the apply + the - * AWAITED seed publish. The margin comfortably exceeds the fetch bound so the lock does not expire - * mid-seed, while staying at the client readiness deadline (12s) so a dead seeder's lock frees when - * clients would recover anyway. Double-seed is prevented regardless of the margin: the seeder publishes - * the seed to the stream BEFORE releasing the lock, so any later seeder's {@link streamHasContent} fence - * sees it. */ +/** The seed lock spans the app seed fetch (hard-bounded at `seedRequestMs = 8s`) + the atomic seed + * append. It is only an EFFICIENCY optimization — it stops two tasks both running the seed fetch — and is + * sized to comfortably exceed the fetch bound while staying near the client readiness deadline (12s) so a + * dead seeder's lock frees when clients would recover anyway. Double-seed is prevented even if the lock + * expires mid-seed, because the seed is written via the atomic {@link SEED_IF_EMPTY_SCRIPT} + * (append-iff-empty), NOT the lock — correctness never depends on the lock staying held. */ const SEED_LOCK_TTL_MS = FILE_DOC_TIMEOUTS.seedRequestMs + 4_000 /** How long a stream survives with no heartbeat — long enough that an occupied-but-idle doc never * loses its shared state (the heartbeat refreshes it while any task holds the room). */ @@ -279,11 +290,43 @@ export class FileDocStore { } /** - * Whether the file's stream already holds content — fences a seed apply against a peer that seeded - * while this task held a (possibly stale) lock, so we never write a second seed (split-brain). Both - * callers treat `true` as "do not seed", so this fails CLOSED: a Redis `xLen` error returns `true` - * (cannot confirm the stream is empty → do not risk a double-seed). `false` only when genuinely empty, - * or when disabled (single-replica, where seeding locally is always correct). + * Atomically seed the stream iff it is still empty (see {@link SEED_IF_EMPTY_SCRIPT}). This is what + * actually prevents split-brain double-seeding: the emptiness check and the append happen in one + * Redis-side step, so — unlike a separate {@link streamHasContent} fence + {@link publishAndWait} — + * there is no check-then-append window, and two tasks racing (even both past an expired seed lock) can + * never both write a seed. Returns true if THIS call wrote the seed (apply it locally), false if the + * stream was already seeded (the tailer will deliver the peer's seed — do NOT apply a second one). + * Retries a transient Redis error like {@link appendUpdate}; throws if it ultimately fails. Disabled → + * true (single-replica: seed locally, no stream). + */ + async seedIfEmpty(name: string, update: Uint8Array): Promise { + if (!this.enabled || !this.write) return true + const encoded = Buffer.from(update).toString('base64') + for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { + try { + const wrote = await this.write.eval(SEED_IF_EMPTY_SCRIPT, { + keys: [streamKey(name)], + arguments: [UPDATE_FIELD, encoded], + }) + await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + return wrote === 1 + } catch (error) { + if (attempt === PUBLISH_MAX_RETRIES) { + logger.error(`FileDocStore seed failed for ${name}`, { error: getErrorMessage(error) }) + throw error + } + await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) + } + } + return false + } + + /** + * Whether the file's stream already holds content — an EFFICIENCY recheck in {@link shouldSeed} that + * skips the seed fetch when a prior holder already seeded (the split-brain guard itself is the atomic + * {@link SEED_IF_EMPTY_SCRIPT}, not this check). Treats `true` as "already seeded", so it fails CLOSED: + * a Redis `xLen` error returns `true` (cannot confirm empty → skip the redundant fetch; the atomic seed + * would no-op anyway). `false` only when genuinely empty, or when disabled (single-replica). */ async streamHasContent(name: string): Promise { if (!this.enabled || !this.write) return false diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 127f748dd86..509d312c338 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -373,24 +373,25 @@ async function ensureServerSeed( try { const update = await fetchFileDocSeed(workspaceId, room.fileId) if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return - // Fence against a peer that seeded while we held a stale lock (a rare long stall): if the stream - // already has content it is seeded, so never write a SECOND seed on top — that would split-brain. - if (await store.streamHasContent(name)) { - // Clear the guard so a later join can retry. Safe in both cases: a genuine peer-seed arrives via - // the tailer (and shouldSeed/this fence then no-op), and a fail-closed Redis `xLen` error must not - // strand the room unseeded forever. - room.serverSeedStarted = false - return - } - // Build the seed (file content + seed flag, or just the flag for an empty/missing file), then - // PUBLISH it to the shared stream AWAITED *before* seeding the local doc. The doc is marked seeded - // only once the seed is durably shared — so if the publish fails we leave the doc unseeded and the - // stream empty for a clean retry, rather than serving an unpublished local seed that a peer would - // re-seed on top of (split-brain). SEED_ORIGIN keeps `doc.on('update')` from re-publishing it. + // Build the seed (file content + seed flag, or just the flag for an empty/missing file) and write it + // to the shared stream ATOMICALLY, iff the stream is still empty. This — NOT the seed lock — is the + // split-brain guard: two tasks racing (even both past an expired lock) can never both seed, because + // the emptiness check and the append are one Redis-side step. Publish-before-apply: the doc is marked + // seeded (via the local apply) only once the seed is durably in the stream, so a failed write leaves + // the doc unseeded and the stream empty for a clean retry. SEED_ORIGIN keeps `doc.on('update')` from + // re-publishing it. const seedUpdate = update ?? emptySeedUpdate() - await store.publishAndWait(name, seedUpdate) + const didSeed = await store.seedIfEmpty(name, seedUpdate) if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return - Y.applyUpdate(room.doc, seedUpdate, SEED_ORIGIN) + if (didSeed) { + Y.applyUpdate(room.doc, seedUpdate, SEED_ORIGIN) + } else { + // A peer seeded first: its seed arrives via the tailer, so we must NOT apply our own — a second, + // different-client-id seed IS the split-brain. Clear the guard so a later join can retry if that + // peer seed somehow never lands (e.g. a fail-closed `xLen` error made `shouldSeed` skip a genuinely + // empty stream); a real peer-seed makes the retry a no-op. + room.serverSeedStarted = false + } } catch (error) { logger.warn(`Server seed failed for file ${room.fileId} (workspace ${workspaceId})`, error) room.serverSeedStarted = false @@ -549,8 +550,11 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { // whichever task is last to leave persists real edits, even one that only tailed them. A compaction // snapshot on catch-up (REDIS_SNAPSHOT_ORIGIN) also counts: it folds real edits into one frame, so a // fresh task catching up purely from it must not treat the doc as unedited. The seed transition - // itself is never counted (nor a purely-local copilot merge, already durable via copilot's direct - // file write), so a seeded-but-unedited doc is never projected back over the file. + // itself is never counted, so a seeded-but-unedited doc is never projected back over the file. NOTE: + // in the multi-replica path a copilot merge is NOT excluded — it round-trips through the stream as + // REDIS_ORIGIN, indistinguishable from a peer edit, so it marks `edited`. That only ever causes an + // extra idempotent persist of content copilot already wrote directly (safe over-persist, never a lost + // edit); the single-replica path applies the merge locally with no origin and does not count it. const seededBefore = room.seededObserved if (isDocSeeded(room.doc)) room.seededObserved = true if ( From f83af6e8ab2e01eb514bc0dd925100c28fcfed9b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 21:19:47 -0700 Subject: [PATCH 48/53] docs(realtime): align seed comments with the atomic-append correctness model Greptile flagged lingering doc drift: the module + shouldSeed comments still credited the seed lock + empty-stream check as the split-brain fix. Reframe them so the atomic seedIfEmpty is the exactly-once guarantee and shouldSeed is an efficiency gate only. --- apps/realtime/src/handlers/file-doc-store.ts | 21 ++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 27dbe8bfe01..e4bd138612c 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -22,8 +22,10 @@ * opens a file, so a late-joining task (the normal case under autoscaling) loads the current shared * state before its first client syncs. Catch-up + tail are seamless: the tailer resumes from the * exact id catch-up stopped at. - * - {@link shouldSeed} coordinates the one-time seed with a Redis lock AND an empty-stream check, so - * exactly one task ever writes the seed cluster-wide (the fix for split-brain). + * - The one-time seed is written via the atomic {@link seedIfEmpty} (append-iff-empty in one Redis + * step), so exactly one task ever writes the seed cluster-wide (the fix for split-brain) — even if two + * tasks race. {@link shouldSeed} is a Redis lock + empty-stream check layered on top ONLY as an + * efficiency gate (so tasks don't all run the seed fetch); correctness does not depend on it. * * When `REDIS_URL` is unset (single-pod dev) the store is DISABLED and every method degrades to the * relay's original single-replica behavior: seed locally, no stream, no tailer. @@ -364,17 +366,20 @@ export class FileDocStore { } /** - * Decide whether THIS task should build and write the file's one-time seed. Returns a lock TOKEN only - * when the shared stream is genuinely empty AND this task wins the seed lock — so exactly one task - * across the cluster ever seeds a file, even if several open it at once (the fix for split-brain - * seeding). `null` otherwise. Release the token with {@link releaseSeedLock}. Disabled → always a - * token (single-replica: seed locally). + * Decide whether THIS task should run the (expensive) seed fetch + write for a file. Returns a lock + * TOKEN when this task wins the seed lock and the stream still looks empty; `null` otherwise. This is an + * EFFICIENCY gate — it stops every task that opens the file at once from each fetching the seed. It does + * NOT by itself guarantee a single seed: exactly-once is enforced by the atomic {@link seedIfEmpty} the + * token-holder then calls (the split-brain guard), so a lock that expires mid-seed cannot cause a + * double-seed. Release the token with {@link releaseSeedLock}. Disabled → always a token (single-replica: + * seed locally). */ async shouldSeed(name: string): Promise { const token = await this.acquireLock(`${SEED_LOCK_PREFIX}${name}`, SEED_LOCK_TTL_MS) if (!token || token === DISABLED_LOCK_TOKEN) return token // The lock could be free yet the stream already seeded (a prior holder seeded then its lock - // expired). Re-check under the lock so we never write a SECOND seed on top of an existing one. + // expired). Re-check so we skip the redundant seed fetch — the atomic seedIfEmpty would no-op anyway, + // but this avoids the wasted app round-trip. if (await this.streamHasContent(name)) { await this.releaseSeedLock(name, token) return null From 4419951b3fe2f63e9acb704bc65a03deb02a3708 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 29 Jul 2026 10:46:40 -0700 Subject: [PATCH 49/53] feat(realtime): live workspace tables list, sharing one invalidation-room impl (#6053) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(realtime): live workspace tables list, sharing one invalidation-room impl Bring the tables list to parity with the files list: a create/rename/move/delete/ restore now propagates to every viewer live instead of waiting out the 30s staleTime. Following the files pattern, but factoring the two into one shared implementation rather than copy-pasting. - add ROOM_TYPES.WORKSPACE_TABLES + its authz resolver (workspace-id-addressed, reuses the workspace resolver like workspace-files) - extract setupWorkspaceInvalidationRoom (server) and useWorkspaceInvalidationRoom (client) — the presence-free, workspace-scoped live-list room; files and tables now both bind to it, so they can never drift. Event/room names derive from the room type. Replaces the standalone workspace-files handler + hook - notifyWorkspaceTablesChanged fanout fired from the table service (createTable, renameTable, moveTableToFolder, deleteTable, restoreTable) so it covers both the HTTP routes AND copilot, which call the service directly - relay /api/workspace-tables-changed endpoint; wire the hook into the tables page - consolidate the handler test into one suite run against both room types * feat(realtime): live tables list also covers table-folder mutations Fold in the follow-up: a table folder create/rename/move/delete/restore now propagates to the tables list live too, so the browser is fully consistent. - generic notifyFolderResourceChanged(resourceType, workspaceId) dispatches the workspace live-list signal by resource type (a map, not a special-case if), so file/knowledge_base/workflow are no-ops today and gain liveness by adding a map entry when they adopt an invalidation room - fired from the shared folder lifecycle (createFolder/updateFolder/deleteFolder/ restoreFolder), covering routes AND copilot - the tables room hook now invalidates the table folders query too, not just the tables list, since the page renders both * fix(realtime): skip per-table live-list notify during a folder cascade A folder delete/restore already fires one folder-level notifyFolderResourceChanged for the whole subtree, but the cascade also calls deleteTable/restoreTable per table — each awaiting its own notifyWorkspaceTablesChanged. A folder with many tables would run N+1 sequential relay calls (each bounded by NOTIFY_TIMEOUT_MS), blocking the mutation. Add a skipNotify option the cascade passes so only the one folder-level notify fires. --- apps/realtime/src/handlers/index.ts | 7 +- .../src/handlers/workspace-files.test.ts | 234 ------------------ .../workspace-invalidation-room.test.ts | 223 +++++++++++++++++ ...iles.ts => workspace-invalidation-room.ts} | 83 ++++--- apps/realtime/src/routes/http.ts | 20 ++ .../files/hooks/use-workspace-files-room.ts | 99 +------- .../hooks/use-workspace-invalidation-room.ts | 111 +++++++++ .../tables/hooks/use-workspace-tables-room.ts | 23 ++ .../workspace/[workspaceId]/tables/tables.tsx | 5 + apps/sim/lib/folders/config.ts | 8 +- apps/sim/lib/folders/lifecycle.ts | 10 + apps/sim/lib/realtime/notify.ts | 59 +++++ apps/sim/lib/table/service.ts | 26 +- packages/platform-authz/src/rooms.ts | 2 + packages/realtime-protocol/src/rooms.ts | 8 + 15 files changed, 548 insertions(+), 370 deletions(-) delete mode 100644 apps/realtime/src/handlers/workspace-files.test.ts create mode 100644 apps/realtime/src/handlers/workspace-invalidation-room.test.ts rename apps/realtime/src/handlers/{workspace-files.ts => workspace-invalidation-room.ts} (62%) create mode 100644 apps/sim/app/workspace/[workspaceId]/hooks/use-workspace-invalidation-room.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/hooks/use-workspace-tables-room.ts diff --git a/apps/realtime/src/handlers/index.ts b/apps/realtime/src/handlers/index.ts index 485e2853170..8dd71093673 100644 --- a/apps/realtime/src/handlers/index.ts +++ b/apps/realtime/src/handlers/index.ts @@ -1,3 +1,4 @@ +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { setupConnectionHandlers } from '@/handlers/connection' import { setupWorkspaceFileDocHandlers } from '@/handlers/file-doc' import { setupOperationsHandlers } from '@/handlers/operations' @@ -6,7 +7,7 @@ import { setupSubblocksHandlers } from '@/handlers/subblocks' import { setupTablesHandlers } from '@/handlers/tables' import { setupVariablesHandlers } from '@/handlers/variables' import { setupWorkflowHandlers } from '@/handlers/workflow' -import { setupWorkspaceFilesHandlers } from '@/handlers/workspace-files' +import { setupWorkspaceInvalidationRoom } from '@/handlers/workspace-invalidation-room' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -16,7 +17,9 @@ export function setupAllHandlers(socket: AuthenticatedSocket, roomManager: IRoom setupSubblocksHandlers(socket, roomManager) setupVariablesHandlers(socket, roomManager) setupPresenceHandlers(socket, roomManager) - setupWorkspaceFilesHandlers(socket, roomManager) + // Presence-free, workspace-scoped live-list rooms (share one implementation). + setupWorkspaceInvalidationRoom(socket, roomManager, ROOM_TYPES.WORKSPACE_FILES) + setupWorkspaceInvalidationRoom(socket, roomManager, ROOM_TYPES.WORKSPACE_TABLES) setupWorkspaceFileDocHandlers(socket, roomManager) setupTablesHandlers(socket, roomManager) setupConnectionHandlers(socket, roomManager) diff --git a/apps/realtime/src/handlers/workspace-files.test.ts b/apps/realtime/src/handlers/workspace-files.test.ts deleted file mode 100644 index 350651c94a9..00000000000 --- a/apps/realtime/src/handlers/workspace-files.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' -import type { IRoomManager } from '@/rooms' - -const { mockAuthorizeRoom } = vi.hoisted(() => ({ - mockAuthorizeRoom: vi.fn(), -})) - -vi.mock('@sim/db', () => ({ - db: { select: vi.fn() }, - user: { image: 'image' }, -})) - -vi.mock('@sim/platform-authz/rooms', () => ({ - authorizeRoom: mockAuthorizeRoom, -})) - -import { setupWorkspaceFilesHandlers } from '@/handlers/workspace-files' - -type Payload = { workspaceId?: string } - -function createSocket(overrides?: Record) { - const handlers: Record Promise | void> = {} - // Live Set so the handler's native `socket.rooms` membership tracking works in tests. - const rooms = new Set() - const socket = { - id: 'socket-1', - userId: 'user-1', - userName: 'Test User', - userImage: 'avatar.png', - rooms, - on: vi.fn((event: string, handler: (payload?: Payload) => Promise | void) => { - handlers[event] = handler - }), - emit: vi.fn(), - join: vi.fn((room: string) => rooms.add(room)), - leave: vi.fn((room: string) => rooms.delete(room)), - to: vi.fn().mockReturnValue({ emit: vi.fn() }), - ...overrides, - } - return { handlers, socket, rooms } -} - -function createRoomManager(overrides?: Partial): IRoomManager { - return { - isReady: vi.fn().mockReturnValue(true), - getRoomForSocket: vi.fn().mockResolvedValue(null), - getRoomsForSocket: vi.fn().mockResolvedValue([]), - removeUserFromRoom: vi.fn().mockResolvedValue(false), - removeSocketFromAllRooms: vi.fn().mockResolvedValue([]), - broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), - getRoomUsers: vi.fn().mockResolvedValue([]), - hasRoom: vi.fn().mockResolvedValue(false), - deleteRoom: vi.fn().mockResolvedValue(undefined), - addUserToRoom: vi.fn().mockResolvedValue(undefined), - getUserSession: vi.fn().mockResolvedValue(null), - updateUserActivity: vi.fn().mockResolvedValue(undefined), - updateRoomLastModified: vi.fn().mockResolvedValue(undefined), - emitToRoom: vi.fn(), - getUniqueUserCount: vi.fn().mockResolvedValue(1), - getTotalActiveConnections: vi.fn().mockResolvedValue(0), - shutdown: vi.fn().mockResolvedValue(undefined), - initialize: vi.fn().mockResolvedValue(undefined), - io: { - in: vi.fn().mockReturnValue({ socketsLeave: vi.fn().mockResolvedValue(undefined) }), - }, - ...overrides, - } as unknown as IRoomManager -} - -describe('setupWorkspaceFilesHandlers', () => { - beforeEach(() => { - vi.clearAllMocks() - mockAuthorizeRoom.mockResolvedValue({ - allowed: true, - status: 200, - workspaceId: 'ws-1', - workspacePermission: 'admin', - }) - }) - - it('rejects join when the socket is not authenticated', async () => { - const { socket, handlers } = createSocket({ userId: undefined, userName: undefined }) - setupWorkspaceFilesHandlers( - socket as unknown as Parameters[0], - createRoomManager() - ) - - await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) - - expect(socket.emit).toHaveBeenCalledWith('join-workspace-files-error', { - workspaceId: 'ws-1', - error: 'Authentication required', - code: 'AUTHENTICATION_REQUIRED', - retryable: false, - }) - }) - - it('rejects join with a retryable error when realtime is unavailable', async () => { - const { socket, handlers } = createSocket() - setupWorkspaceFilesHandlers( - socket as unknown as Parameters[0], - createRoomManager({ isReady: vi.fn().mockReturnValue(false) }) - ) - - await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) - - expect(socket.emit).toHaveBeenCalledWith( - 'join-workspace-files-error', - expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) - ) - }) - - it('rejects join when workspace access is denied', async () => { - mockAuthorizeRoom.mockResolvedValue({ - allowed: false, - status: 403, - workspaceId: 'ws-1', - workspacePermission: null, - }) - const { socket, handlers } = createSocket() - setupWorkspaceFilesHandlers( - socket as unknown as Parameters[0], - createRoomManager() - ) - - await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) - - expect(socket.emit).toHaveBeenCalledWith( - 'join-workspace-files-error', - expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) - ) - }) - - it('joins the workspace files room on success without any presence bookkeeping', async () => { - const { socket, handlers } = createSocket() - const roomManager = createRoomManager() - setupWorkspaceFilesHandlers( - socket as unknown as Parameters[0], - roomManager - ) - - await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) - - expect(socket.join).toHaveBeenCalledWith('workspace-files:ws-1') - expect(socket.emit).toHaveBeenCalledWith('join-workspace-files-success', { - workspaceId: 'ws-1', - }) - // The room is live-tree-only: no room-manager presence is tracked or broadcast. - expect(roomManager.addUserToRoom).not.toHaveBeenCalled() - expect(roomManager.broadcastPresenceUpdate).not.toHaveBeenCalled() - }) - - it('leaves a previously-joined files room when switching workspaces', async () => { - const { socket, handlers, rooms } = createSocket() - rooms.add('workspace-files:ws-old') - setupWorkspaceFilesHandlers( - socket as unknown as Parameters[0], - createRoomManager() - ) - - await handlers['join-workspace-files']({ workspaceId: 'ws-1' }) - - expect(socket.leave).toHaveBeenCalledWith('workspace-files:ws-old') - expect(socket.join).toHaveBeenCalledWith('workspace-files:ws-1') - }) - - it('leaves the scoped files room on leave', () => { - const { socket, handlers, rooms } = createSocket() - rooms.add('workspace-files:ws-1') - setupWorkspaceFilesHandlers( - socket as unknown as Parameters[0], - createRoomManager() - ) - - handlers['leave-workspace-files']({ workspaceId: 'ws-1' }) - - expect(socket.leave).toHaveBeenCalledWith('workspace-files:ws-1') - }) - - it('cancels an in-flight join when the user leaves that workspace mid-authorize', async () => { - const { socket, handlers } = createSocket() - let resolveAuth: (value: unknown) => void = () => {} - mockAuthorizeRoom.mockReturnValue( - new Promise((resolve) => { - resolveAuth = resolve - }) - ) - setupWorkspaceFilesHandlers( - socket as unknown as Parameters[0], - createRoomManager() - ) - - // Join ws-1 is awaiting authorization when the view unmounts and leaves ws-1. - const joinPromise = handlers['join-workspace-files']({ workspaceId: 'ws-1' }) - handlers['leave-workspace-files']({ workspaceId: 'ws-1' }) - resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) - await joinPromise - - // The stale join must NOT join the room the client has since left (no stranded membership). - expect(socket.join).not.toHaveBeenCalled() - expect(socket.emit).not.toHaveBeenCalledWith('join-workspace-files-success', { - workspaceId: 'ws-1', - }) - }) - - it('does not cancel an in-flight join when a deferred leave targets a different workspace', async () => { - const { socket, handlers } = createSocket() - let resolveAuth: (value: unknown) => void = () => {} - mockAuthorizeRoom.mockReturnValue( - new Promise((resolve) => { - resolveAuth = resolve - }) - ) - setupWorkspaceFilesHandlers( - socket as unknown as Parameters[0], - createRoomManager() - ) - - // The client has switched to ws-2 (join in-flight) when a stale leave for the prior ws-1 lands. - const joinPromise = handlers['join-workspace-files']({ workspaceId: 'ws-2' }) - handlers['leave-workspace-files']({ workspaceId: 'ws-1' }) - resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-2', workspacePermission: 'admin' }) - await joinPromise - - // The deferred leave for ws-1 must not abort the join the client actually wants (ws-2). - expect(socket.join).toHaveBeenCalledWith('workspace-files:ws-2') - expect(socket.emit).toHaveBeenCalledWith('join-workspace-files-success', { - workspaceId: 'ws-2', - }) - }) -}) diff --git a/apps/realtime/src/handlers/workspace-invalidation-room.test.ts b/apps/realtime/src/handlers/workspace-invalidation-room.test.ts new file mode 100644 index 00000000000..643fca696f0 --- /dev/null +++ b/apps/realtime/src/handlers/workspace-invalidation-room.test.ts @@ -0,0 +1,223 @@ +/** + * @vitest-environment node + */ +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { IRoomManager } from '@/rooms' + +const { mockAuthorizeRoom } = vi.hoisted(() => ({ + mockAuthorizeRoom: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { select: vi.fn() }, + user: { image: 'image' }, +})) + +vi.mock('@sim/platform-authz/rooms', () => ({ + authorizeRoom: mockAuthorizeRoom, +})) + +import { setupWorkspaceInvalidationRoom } from '@/handlers/workspace-invalidation-room' + +type Payload = { workspaceId?: string } + +function createSocket(overrides?: Record) { + const handlers: Record Promise | void> = {} + // Live Set so the handler's native `socket.rooms` membership tracking works in tests. + const rooms = new Set() + const socket = { + id: 'socket-1', + userId: 'user-1', + userName: 'Test User', + userImage: 'avatar.png', + rooms, + on: vi.fn((event: string, handler: (payload?: Payload) => Promise | void) => { + handlers[event] = handler + }), + emit: vi.fn(), + join: vi.fn((room: string) => rooms.add(room)), + leave: vi.fn((room: string) => rooms.delete(room)), + to: vi.fn().mockReturnValue({ emit: vi.fn() }), + ...overrides, + } + return { handlers, socket, rooms } +} + +function createRoomManager(overrides?: Partial): IRoomManager { + return { + isReady: vi.fn().mockReturnValue(true), + getRoomForSocket: vi.fn().mockResolvedValue(null), + getRoomsForSocket: vi.fn().mockResolvedValue([]), + removeUserFromRoom: vi.fn().mockResolvedValue(false), + removeSocketFromAllRooms: vi.fn().mockResolvedValue([]), + broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), + getRoomUsers: vi.fn().mockResolvedValue([]), + hasRoom: vi.fn().mockResolvedValue(false), + deleteRoom: vi.fn().mockResolvedValue(undefined), + addUserToRoom: vi.fn().mockResolvedValue(undefined), + getUserSession: vi.fn().mockResolvedValue(null), + updateUserActivity: vi.fn().mockResolvedValue(undefined), + updateRoomLastModified: vi.fn().mockResolvedValue(undefined), + emitToRoom: vi.fn(), + getUniqueUserCount: vi.fn().mockResolvedValue(1), + getTotalActiveConnections: vi.fn().mockResolvedValue(0), + shutdown: vi.fn().mockResolvedValue(undefined), + initialize: vi.fn().mockResolvedValue(undefined), + io: { + in: vi.fn().mockReturnValue({ socketsLeave: vi.fn().mockResolvedValue(undefined) }), + }, + ...overrides, + } as unknown as IRoomManager +} + +// The two presence-free live-list rooms share one implementation; run the whole suite against both +// so files and tables can never drift. Event names and room names derive from the room type. +describe.each([ROOM_TYPES.WORKSPACE_FILES, ROOM_TYPES.WORKSPACE_TABLES] as const)( + 'setupWorkspaceInvalidationRoom(%s)', + (roomType) => { + const joinEvent = `join-${roomType}` + const successEvent = `${joinEvent}-success` + const errorEvent = `${joinEvent}-error` + const leaveEvent = `leave-${roomType}` + const roomOf = (workspaceId: string) => `${roomType}:${workspaceId}` + + const setup = (socket: ReturnType['socket'], roomManager: IRoomManager) => + setupWorkspaceInvalidationRoom( + socket as unknown as Parameters[0], + roomManager, + roomType + ) + + beforeEach(() => { + vi.clearAllMocks() + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', + }) + }) + + it('rejects join when the socket is not authenticated', async () => { + const { socket, handlers } = createSocket({ userId: undefined, userName: undefined }) + setup(socket, createRoomManager()) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith(errorEvent, { + workspaceId: 'ws-1', + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + }) + + it('rejects join with a retryable error when realtime is unavailable', async () => { + const { socket, handlers } = createSocket() + setup(socket, createRoomManager({ isReady: vi.fn().mockReturnValue(false) })) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + errorEvent, + expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) + ) + }) + + it('rejects join when workspace access is denied', async () => { + mockAuthorizeRoom.mockResolvedValue({ + allowed: false, + status: 403, + workspaceId: 'ws-1', + workspacePermission: null, + }) + const { socket, handlers } = createSocket() + setup(socket, createRoomManager()) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + errorEvent, + expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) + ) + }) + + it('joins the room on success without any presence bookkeeping', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + setup(socket, roomManager) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.join).toHaveBeenCalledWith(roomOf('ws-1')) + expect(socket.emit).toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-1' }) + // The room is live-list-only: no room-manager presence is tracked or broadcast. + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + expect(roomManager.broadcastPresenceUpdate).not.toHaveBeenCalled() + }) + + it('leaves a previously-joined room when switching workspaces', async () => { + const { socket, handlers, rooms } = createSocket() + rooms.add(roomOf('ws-old')) + setup(socket, createRoomManager()) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.leave).toHaveBeenCalledWith(roomOf('ws-old')) + expect(socket.join).toHaveBeenCalledWith(roomOf('ws-1')) + }) + + it('leaves the scoped room on leave', () => { + const { socket, handlers, rooms } = createSocket() + rooms.add(roomOf('ws-1')) + setup(socket, createRoomManager()) + + handlers[leaveEvent]({ workspaceId: 'ws-1' }) + + expect(socket.leave).toHaveBeenCalledWith(roomOf('ws-1')) + }) + + it('cancels an in-flight join when the user leaves that workspace mid-authorize', async () => { + const { socket, handlers } = createSocket() + let resolveAuth: (value: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve + }) + ) + setup(socket, createRoomManager()) + + // Join ws-1 is awaiting authorization when the view unmounts and leaves ws-1. + const joinPromise = handlers[joinEvent]({ workspaceId: 'ws-1' }) + handlers[leaveEvent]({ workspaceId: 'ws-1' }) + resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + // The stale join must NOT join the room the client has since left (no stranded membership). + expect(socket.join).not.toHaveBeenCalled() + expect(socket.emit).not.toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-1' }) + }) + + it('does not cancel an in-flight join when a deferred leave targets a different workspace', async () => { + const { socket, handlers } = createSocket() + let resolveAuth: (value: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve + }) + ) + setup(socket, createRoomManager()) + + // The client has switched to ws-2 (join in-flight) when a stale leave for the prior ws-1 lands. + const joinPromise = handlers[joinEvent]({ workspaceId: 'ws-2' }) + handlers[leaveEvent]({ workspaceId: 'ws-1' }) + resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-2', workspacePermission: 'admin' }) + await joinPromise + + // The deferred leave for ws-1 must not abort the join the client actually wants (ws-2). + expect(socket.join).toHaveBeenCalledWith(roomOf('ws-2')) + expect(socket.emit).toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-2' }) + }) + } +) diff --git a/apps/realtime/src/handlers/workspace-files.ts b/apps/realtime/src/handlers/workspace-invalidation-room.ts similarity index 62% rename from apps/realtime/src/handlers/workspace-files.ts rename to apps/realtime/src/handlers/workspace-invalidation-room.ts index 1d5fa6ac5ce..aacb141e9ae 100644 --- a/apps/realtime/src/handlers/workspace-files.ts +++ b/apps/realtime/src/handlers/workspace-invalidation-room.ts @@ -1,40 +1,43 @@ import { createLogger } from '@sim/logger' -import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { type RoomRef, type RoomType, roomName } from '@sim/realtime-protocol/rooms' import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' -const logger = createLogger('WorkspaceFilesHandlers') - -/** The workspace-files room ref for a workspace id. */ -const filesRoom = (workspaceId: string): RoomRef => ({ - type: ROOM_TYPES.WORKSPACE_FILES, - id: workspaceId, -}) - -/** Socket.IO room-name prefix shared by every workspace-files room. */ -const FILES_ROOM_PREFIX = `${ROOM_TYPES.WORKSPACE_FILES}:` +const logger = createLogger('WorkspaceInvalidationRoom') interface JoinPayload { workspaceId: string } /** - * Keeps the workspace file browser live. The socket joins a workspace-scoped Socket.IO room - * so a `workspace-files-changed` event — fanned out by the HTTP mutation API — reaches every - * viewer, who then refetches. This room carries NO presence: "who's in a file" comes from - * the per-file doc room, and file mutations go over HTTP. Membership is tracked natively by - * Socket.IO (`socket.rooms`), so a workspace switch just leaves the prior files room — no - * room-manager presence bookkeeping to keep in sync. + * Wires a workspace-scoped, presence-free "invalidation room" onto a socket: the client joins a + * room named after its workspace, and a `${roomType}-changed` event — fanned out by the server-side + * mutation path over HTTP — reaches every viewer so they refetch. This is the shared core behind the + * workspace-files and workspace-tables browsers; they differ only in `roomType` (which also derives + * the event names, since each room type's wire token IS its event stem: `join-${roomType}`, + * `leave-${roomType}`, `join-${roomType}-success/-error`, and the `${roomType}-changed` broadcast). + * + * These rooms carry NO presence — "who's in a resource" comes from the per-resource room (file-doc / + * table), and mutations go over HTTP. Membership is tracked natively by Socket.IO (`socket.rooms`), + * so a workspace switch just leaves the prior room — no room-manager presence bookkeeping to sync. */ -export function setupWorkspaceFilesHandlers( +export function setupWorkspaceInvalidationRoom( socket: AuthenticatedSocket, - roomManager: IRoomManager + roomManager: IRoomManager, + roomType: RoomType ) { + const joinEvent = `join-${roomType}` + const leaveEvent = `leave-${roomType}` + const successEvent = `${joinEvent}-success` + const errorEvent = `${joinEvent}-error` + const roomPrefix = `${roomType}:` + const room = (workspaceId: string): RoomRef => ({ type: roomType, id: workspaceId }) + // Monotonic per-socket join counter: each join captures its number and, after the async // authorize, aborts if a newer intent has superseded it — a fast workspace switch A→B can // otherwise let A's late completion leave B and strand the socket in A, missing B's - // `workspace-files-changed` invalidations. + // `${roomType}-changed` invalidations. let joinGeneration = 0 // The workspace the socket currently intends to be in (set when a join starts). A leave that // targets this workspace — or an unscoped "leave all" — advances joinGeneration so an in-flight @@ -43,11 +46,11 @@ export function setupWorkspaceFilesHandlers( // switched to (the bug that bit the file-doc room in #5941). let currentWorkspace: string | null = null - socket.on('join-workspace-files', async ({ workspaceId }: JoinPayload) => { + socket.on(joinEvent, async ({ workspaceId }: JoinPayload) => { // Validate synchronously BEFORE claiming a generation, so a rejected/malformed join can't // advance joinGeneration and cancel a legitimate in-flight join for another workspace. if (!socket.userId || !socket.userName) { - socket.emit('join-workspace-files-error', { + socket.emit(errorEvent, { workspaceId, error: 'Authentication required', code: 'AUTHENTICATION_REQUIRED', @@ -57,7 +60,7 @@ export function setupWorkspaceFilesHandlers( } if (!roomManager.isReady()) { - socket.emit('join-workspace-files-error', { + socket.emit(errorEvent, { workspaceId, error: 'Realtime unavailable', code: 'ROOM_MANAGER_UNAVAILABLE', @@ -69,7 +72,7 @@ export function setupWorkspaceFilesHandlers( // Validate the client-supplied id before it reaches the DB query (join payloads are // otherwise raw client input) and before advancing the generation. if (typeof workspaceId !== 'string' || workspaceId.length === 0) { - socket.emit('join-workspace-files-error', { + socket.emit(errorEvent, { workspaceId: typeof workspaceId === 'string' ? workspaceId : '', error: 'Invalid workspace id', code: 'INVALID_PAYLOAD', @@ -81,21 +84,21 @@ export function setupWorkspaceFilesHandlers( const joinAttempt = (joinGeneration += 1) currentWorkspace = workspaceId try { - const room = filesRoom(workspaceId) + const ref = room(workspaceId) const authorized = await resolveRoomJoinAuth({ userId: socket.userId, - room, + room: ref, action: 'read', logger, - logLabel: `files room for ${socket.userId}`, + logLabel: `${roomType} room for ${socket.userId}`, messages: { verifyFailed: 'Failed to verify workspace access', notFound: 'Workspace not found', accessDenied: 'Access denied to workspace', }, emitError: ({ error, code, retryable }) => - socket.emit('join-workspace-files-error', { workspaceId, error, code, retryable }), + socket.emit(errorEvent, { workspaceId, error, code, retryable }), }) if (!authorized) return @@ -103,34 +106,34 @@ export function setupWorkspaceFilesHandlers( // stale join can't leave the room the client has since switched to. if (joinGeneration !== joinAttempt || socket.disconnected) return - // Leave any previously-joined files room (workspace switch), read straight from the + // Leave any previously-joined room of this type (workspace switch), read straight from the // socket's native room membership so there's no presence store to keep in sync. - const target = roomName(room) + const target = roomName(ref) for (const joined of socket.rooms) { - if (joined !== target && joined.startsWith(FILES_ROOM_PREFIX)) socket.leave(joined) + if (joined !== target && joined.startsWith(roomPrefix)) socket.leave(joined) } socket.join(target) - socket.emit('join-workspace-files-success', { workspaceId }) + socket.emit(successEvent, { workspaceId }) } catch (error) { - logger.error('Error joining workspace files room:', error) + logger.error(`Error joining ${roomType} room:`, error) try { - socket.leave(roomName(filesRoom(workspaceId))) + socket.leave(roomName(room(workspaceId))) } catch {} // Suppress the client-facing error when this join was already superseded: the client has // switched to a newer workspace, and a retryable error naming the abandoned one could make it // re-join and cancel the newer join. The leave above still runs. if (joinGeneration !== joinAttempt || socket.disconnected) return - socket.emit('join-workspace-files-error', { + socket.emit(errorEvent, { workspaceId, - error: 'Failed to join workspace files', + error: `Failed to join ${roomType}`, code: 'JOIN_FAILED', retryable: true, }) } }) - socket.on('leave-workspace-files', (payload?: { workspaceId?: string }) => { + socket.on(leaveEvent, (payload?: { workspaceId?: string }) => { // Cancel an in-flight join whose target the client is now leaving: a join awaiting // authorization when the view unmounts would otherwise complete afterwards and strand the // socket in a room it has left. Only when the leave targets the current join intent (or is @@ -141,10 +144,10 @@ export function setupWorkspaceFilesHandlers( currentWorkspace = null } // Scope the leave to a specific workspace when the client provides one: a deferred leave - // from a prior page must not evict a files room the socket has since switched into. - const target = payload?.workspaceId ? roomName(filesRoom(payload.workspaceId)) : null + // from a prior page must not evict a room the socket has since switched into. + const target = payload?.workspaceId ? roomName(room(payload.workspaceId)) : null for (const joined of socket.rooms) { - if (!joined.startsWith(FILES_ROOM_PREFIX)) continue + if (!joined.startsWith(roomPrefix)) continue if (target && joined !== target) continue socket.leave(joined) } diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 971162951f0..3b184eb9f15 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -183,6 +183,26 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } + // Fan out a table-list change to everyone viewing a workspace's tables, so their browser + // refetches. The list-level counterpart to workspace-files-changed; same lossy-signal contract. + if (req.method === 'POST' && req.url === '/api/workspace-tables-changed') { + try { + const body = await readRequestBody(req) + const { workspaceId } = JSON.parse(body) + if (!isNonEmptyString(workspaceId)) return sendError(res, 'Invalid workspaceId', 400) + roomManager.emitToRoom( + { type: ROOM_TYPES.WORKSPACE_TABLES, id: workspaceId }, + 'workspace-tables-changed', + { workspaceId, timestamp: Date.now() } + ) + sendSuccess(res) + } catch (error) { + logger.error('Error handling workspace tables changed notification:', error) + sendError(res, 'Failed to process tables change notification') + } + return + } + // Merge a copilot edit into a file's LIVE collaborative document so it streams into open editors // (Stage C). Returns `{ applied }`: when false, no seeded live room exists and the caller writes // the file directly instead. Any live user edits are preserved — the app builds a minimal CRDT diff. diff --git a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts index 34cda7dade0..3db8fed29f4 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room.ts @@ -1,101 +1,18 @@ 'use client' -import { useEffect } from 'react' -import { createLogger } from '@sim/logger' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { useQueryClient } from '@tanstack/react-query' -import { useSocket } from '@/app/workspace/providers/socket-provider' +import { useWorkspaceInvalidationRoom } from '@/app/workspace/[workspaceId]/hooks/use-workspace-invalidation-room' import { invalidateWorkspaceFileBrowsers } from '@/hooks/queries/workspace-file-folders' -const logger = createLogger('WorkspaceFilesRoom') - -/** Retry cap + base delay for a retryable join failure on an otherwise-live socket. */ -const MAX_JOIN_RETRIES = 3 -const JOIN_RETRY_BASE_MS = 1000 - -interface JoinErrorPayload { - workspaceId: string - error: string - code: string - retryable?: boolean -} - /** - * Keeps the file browser live: joins the workspace-files room over the shared socket so a - * `workspace-files-changed` broadcast (fanned out by the HTTP mutation API) invalidates the - * browser queries and every viewer refetches without waiting for staleness. File mutations - * stay on HTTP. - * - * This room carries no presence — "who's in a file" comes from the per-file doc room (see - * `FileDocRoomProvider`), not from who's browsing the Files section. + * Keeps the file browser live: joins the workspace-files room so a `workspace-files-changed` + * broadcast (fanned out by the file mutation API) invalidates the browser queries and every viewer + * refetches without waiting for staleness. Thin binding over {@link useWorkspaceInvalidationRoom}. */ export function useWorkspaceFilesRoom(workspaceId: string): void { - const { socket } = useSocket() const queryClient = useQueryClient() - - useEffect(() => { - if (!socket || !workspaceId) return - - let retries = 0 - let retryTimer: ReturnType | null = null - - const join = () => socket.emit('join-workspace-files', { workspaceId }) - - // A fresh (re)connect gets a fresh retry budget, so a prior full exhaustion doesn't leave the - // socket unable to retry a failed re-join until the next success. Cancel any retry still pending - // from before the reconnect so it can't fire a duplicate join after this immediate one. - const handleConnect = () => { - retries = 0 - if (retryTimer) { - clearTimeout(retryTimer) - retryTimer = null - } - join() - } - - const handleJoinSuccess = (data: { workspaceId: string }) => { - if (data.workspaceId !== workspaceId) return - retries = 0 - // Cancel any retry scheduled by a prior retryable error so it can't fire an extra - // join after we're already in. - if (retryTimer) { - clearTimeout(retryTimer) - retryTimer = null - } - } - const handleJoinError = (data: JoinErrorPayload) => { - if (data.workspaceId !== workspaceId) return - logger.warn('Failed to join workspace files room', { code: data.code, error: data.error }) - if (data.retryable && retries < MAX_JOIN_RETRIES) { - retries += 1 - // Clear any still-pending retry before scheduling a new one, so reconnect churn can't - // orphan a timer that fires an extra join(). - if (retryTimer) clearTimeout(retryTimer) - retryTimer = setTimeout(join, JOIN_RETRY_BASE_MS * retries) - } - } - const handleChanged = (data: { workspaceId: string }) => { - if (data.workspaceId === workspaceId) - invalidateWorkspaceFileBrowsers(queryClient, workspaceId) - } - - // Join now if the socket is already connected; `connect` covers (re)connects. - if (socket.connected) join() - socket.on('connect', handleConnect) - socket.on('join-workspace-files-success', handleJoinSuccess) - socket.on('join-workspace-files-error', handleJoinError) - socket.on('workspace-files-changed', handleChanged) - - return () => { - if (retryTimer) clearTimeout(retryTimer) - socket.off('connect', handleConnect) - socket.off('join-workspace-files-success', handleJoinSuccess) - socket.off('join-workspace-files-error', handleJoinError) - socket.off('workspace-files-changed', handleChanged) - - // Leave the room, scoped to THIS workspace: the server no-ops if the socket has - // already switched to another workspace's files room (so a workspace A→B switch, - // where B's join runs first and auto-leaves A, can't have A's leave evict B). - socket.emit('leave-workspace-files', { workspaceId }) - } - }, [socket, workspaceId, queryClient]) + useWorkspaceInvalidationRoom(workspaceId, ROOM_TYPES.WORKSPACE_FILES, () => + invalidateWorkspaceFileBrowsers(queryClient, workspaceId) + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/hooks/use-workspace-invalidation-room.ts b/apps/sim/app/workspace/[workspaceId]/hooks/use-workspace-invalidation-room.ts new file mode 100644 index 00000000000..0c652fcb8d6 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/hooks/use-workspace-invalidation-room.ts @@ -0,0 +1,111 @@ +'use client' + +import { useEffect, useRef } from 'react' +import { createLogger } from '@sim/logger' +import type { RoomType } from '@sim/realtime-protocol/rooms' +import { useSocket } from '@/app/workspace/providers/socket-provider' + +const logger = createLogger('WorkspaceInvalidationRoom') + +/** Retry cap + base delay for a retryable join failure on an otherwise-live socket. */ +const MAX_JOIN_RETRIES = 3 +const JOIN_RETRY_BASE_MS = 1000 + +interface JoinErrorPayload { + workspaceId: string + error: string + code: string + retryable?: boolean +} + +/** + * Joins a workspace-scoped, presence-free "invalidation room" over the shared socket and runs + * `onChanged` whenever the server broadcasts `${roomType}-changed` for this workspace, so the list + * refetches without waiting for staleness. Shared core behind {@link useWorkspaceFilesRoom} and + * {@link useWorkspaceTablesRoom}; event names derive from `roomType`. + * + * These rooms carry no presence — "who's in a resource" comes from the per-resource room, not from + * who's browsing the section. Mutations happen server-side (HTTP + copilot) and fan out this signal. + */ +export function useWorkspaceInvalidationRoom( + workspaceId: string, + roomType: RoomType, + onChanged: () => void +): void { + const { socket } = useSocket() + // Held by ref so a caller passing a fresh closure each render never re-subscribes the socket. + const onChangedRef = useRef(onChanged) + onChangedRef.current = onChanged + + useEffect(() => { + if (!socket || !workspaceId) return + + const joinEvent = `join-${roomType}` + const successEvent = `${joinEvent}-success` + const errorEvent = `${joinEvent}-error` + const leaveEvent = `leave-${roomType}` + const changedEvent = `${roomType}-changed` + + let retries = 0 + let retryTimer: ReturnType | null = null + + const join = () => socket.emit(joinEvent, { workspaceId }) + + // A fresh (re)connect gets a fresh retry budget, so a prior full exhaustion doesn't leave the + // socket unable to retry a failed re-join until the next success. Cancel any retry still pending + // from before the reconnect so it can't fire a duplicate join after this immediate one. + const handleConnect = () => { + retries = 0 + if (retryTimer) { + clearTimeout(retryTimer) + retryTimer = null + } + join() + } + + const handleJoinSuccess = (data: { workspaceId: string }) => { + if (data.workspaceId !== workspaceId) return + retries = 0 + // Cancel any retry scheduled by a prior retryable error so it can't fire an extra + // join after we're already in. + if (retryTimer) { + clearTimeout(retryTimer) + retryTimer = null + } + } + const handleJoinError = (data: JoinErrorPayload) => { + if (data.workspaceId !== workspaceId) return + logger.warn(`Failed to join ${roomType} room`, { code: data.code, error: data.error }) + if (data.retryable && retries < MAX_JOIN_RETRIES) { + retries += 1 + // Clear any still-pending retry before scheduling a new one, so reconnect churn can't + // orphan a timer that fires an extra join(). + if (retryTimer) clearTimeout(retryTimer) + retryTimer = setTimeout(join, JOIN_RETRY_BASE_MS * retries) + } + } + const handleChanged = (data: { workspaceId: string }) => { + if (data.workspaceId === workspaceId) onChangedRef.current() + } + + // Join now if the socket is already connected; `connect` covers (re)connects. + if (socket.connected) join() + socket.on('connect', handleConnect) + socket.on(successEvent, handleJoinSuccess) + socket.on(errorEvent, handleJoinError) + socket.on(changedEvent, handleChanged) + + return () => { + if (retryTimer) clearTimeout(retryTimer) + socket.off('connect', handleConnect) + socket.off(successEvent, handleJoinSuccess) + socket.off(errorEvent, handleJoinError) + socket.off(changedEvent, handleChanged) + + // Leave the room, scoped to THIS workspace: the server no-ops if the socket has already + // switched to another workspace's room (so a workspace A→B switch, where B's join runs first + // and auto-leaves A, can't have A's leave evict B). + socket.emit(leaveEvent, { workspaceId }) + } + }, [socket, workspaceId, roomType]) +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/hooks/use-workspace-tables-room.ts b/apps/sim/app/workspace/[workspaceId]/tables/hooks/use-workspace-tables-room.ts new file mode 100644 index 00000000000..3387ede2b5f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/hooks/use-workspace-tables-room.ts @@ -0,0 +1,23 @@ +'use client' + +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { useQueryClient } from '@tanstack/react-query' +import { useWorkspaceInvalidationRoom } from '@/app/workspace/[workspaceId]/hooks/use-workspace-invalidation-room' +import { folderKeys } from '@/hooks/queries/utils/folder-keys' +import { tableKeys } from '@/hooks/queries/utils/table-keys' + +/** + * Keeps the tables browser live: joins the workspace-tables room so a `workspace-tables-changed` + * broadcast (fanned out by the table + table-folder mutation services) invalidates the tables list + * AND the table folders so every viewer refetches without waiting for staleness. A created/renamed/ + * moved/deleted/restored table changes the list result (including folder placement); a folder + * create/rename/delete/restore changes the folder tree — the page renders both, so both are + * invalidated. Thin binding over {@link useWorkspaceInvalidationRoom}. + */ +export function useWorkspaceTablesRoom(workspaceId: string): void { + const queryClient = useQueryClient() + useWorkspaceInvalidationRoom(workspaceId, ROOM_TYPES.WORKSPACE_TABLES, () => { + queryClient.invalidateQueries({ queryKey: tableKeys.lists() }) + queryClient.invalidateQueries({ queryKey: folderKeys.resource('table') }) + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx b/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx index 94eecb940bd..81a44336bbd 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx @@ -48,6 +48,7 @@ import { TablesListContextMenu, } from '@/app/workspace/[workspaceId]/tables/components' import { TableContextMenu } from '@/app/workspace/[workspaceId]/tables/components/table-context-menu' +import { useWorkspaceTablesRoom } from '@/app/workspace/[workspaceId]/tables/hooks/use-workspace-tables-room' import { tablesParsers, tablesSortParams, @@ -112,6 +113,10 @@ export function Tables() { const userPermissions = useUserPermissionsContext() const canEdit = userPermissions.canEdit === true + // Joined for the live tables list: a `workspace-tables-changed` broadcast (fanned out by the table + // mutation service) invalidates the list so this view refetches without waiting for staleness. + useWorkspaceTablesRoom(workspaceId) + const { data: tables = EMPTY_TABLES, error } = useTablesList(workspaceId) const { data: members } = useWorkspaceMembersQuery(workspaceId) const pinnedTableIds = usePinnedIds(workspaceId, 'table') diff --git a/apps/sim/lib/folders/config.ts b/apps/sim/lib/folders/config.ts index 4c6b3d6333e..02e803f5749 100644 --- a/apps/sim/lib/folders/config.ts +++ b/apps/sim/lib/folders/config.ts @@ -306,6 +306,8 @@ async function archiveTableChildren(context: CascadeChildrenContext): Promise { + try { + const response = await fetch(`${getSocketServerUrl()}/api/workspace-tables-changed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ workspaceId }), + signal: AbortSignal.timeout(NOTIFY_TIMEOUT_MS), + }) + if (!response.ok) { + logger.warn('workspace-tables-changed notify failed', { + workspaceId, + status: response.status, + }) + } + } catch (error) { + logger.warn('workspace-tables-changed notify error', { + workspaceId, + error: getErrorMessage(error), + }) + } +} + +/** + * Folder resource types whose list is kept live by a workspace invalidation room: a folder mutation + * (create/rename/move/delete/restore) for one of these must fan out the same list-changed signal as a + * direct resource mutation, because a new/renamed/removed folder changes what that resource's browser + * shows. Extend this map as more resource lists adopt an invalidation room — `file` and + * `knowledge_base` currently refetch through their own paths, and `workflow` has no such list room. + */ +const FOLDER_RESOURCE_NOTIFIERS: Partial< + Record Promise> +> = { + table: notifyWorkspaceTablesChanged, +} + +/** + * Fan out the workspace live-list signal for a folder mutation, dispatched on the folder's resource + * type. A no-op for resource types without an invalidation room. Never throws (the underlying notify + * is best-effort). Callers `await` it so the dispatch is guaranteed before the mutation returns. + */ +export async function notifyFolderResourceChanged( + resourceType: FolderResourceType, + workspaceId: string +): Promise { + await FOLDER_RESOURCE_NOTIFIERS[resourceType]?.(workspaceId) +} + /** * Best-effort: ask the realtime relay to merge a copilot edit into a file's LIVE collaborative * document, so open editors see it stream in as a CRDT merge (Stage C) rather than the file changing diff --git a/apps/sim/lib/table/service.ts b/apps/sim/lib/table/service.ts index 112a41a8e72..2f5baf0f46c 100644 --- a/apps/sim/lib/table/service.ts +++ b/apps/sim/lib/table/service.ts @@ -17,6 +17,7 @@ import { and, count, eq, isNull, sql } from 'drizzle-orm' import { generateRestoreName } from '@/lib/core/utils/restore-name' import type { DbOrTx } from '@/lib/db/types' import { resolveRestoredFolderId } from '@/lib/folders/queries' +import { notifyWorkspaceTablesChanged } from '@/lib/realtime/notify' import { assertRowCapacity, notifyTableRowUsage } from '@/lib/table/billing' import { generateColumnId, getColumnId, withGeneratedColumnIds } from '@/lib/table/column-keys' import { COLUMN_TYPES, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants' @@ -424,6 +425,9 @@ export async function createTable( logger.info(`[${requestId}] Created table ${tableId} in workspace ${data.workspaceId}`) + // Live tables list: tell everyone viewing this workspace's tables to refetch. + await notifyWorkspaceTablesChanged(data.workspaceId) + return { id: newTable.id, name: newTable.name, @@ -612,6 +616,10 @@ export async function renameTable( } logger.info(`[${requestId}] Renamed table ${tableId} to "${newName}"`) + + // Live tables list: a rename changes the list result, so everyone viewing refetches. + if (workspaceId) await notifyWorkspaceTablesChanged(workspaceId) + return { id: tableId, name: newName } } catch (error: unknown) { if (getPostgresErrorCode(error) === '23505') { @@ -687,6 +695,9 @@ export async function moveTableToFolder( } logger.info(`[${requestId}] Moved table ${tableId} to folder ${folderId ?? 'root'}`) + + // Live tables list: a move changes each table's folder placement in the list result. + await notifyWorkspaceTablesChanged(workspaceId) } /** @@ -831,7 +842,7 @@ export async function deleteTable( tableId: string, requestId: string, actingUserId?: string, - options?: { archivedAt?: Date } + options?: { archivedAt?: Date; skipNotify?: boolean } ): Promise { const now = options?.archivedAt ?? new Date() // Archiving destroys access to every row, so it is gated on the delete lock. @@ -889,6 +900,12 @@ export async function deleteTable( } logger.info(`[${requestId}] Archived table ${tableId}`) + + // Live tables list: only on a genuine archive (a no-op/already-archived delete changes nothing). + // Skipped under a folder cascade — deleteFolder fires one folder-level notify for the whole subtree, + // so we don't spam one relay call per archived table. + if (deleted?.workspaceId && !options?.skipNotify) + await notifyWorkspaceTablesChanged(deleted.workspaceId) } /** @@ -903,7 +920,7 @@ export async function deleteTable( export async function restoreTable( tableId: string, requestId: string, - options?: { restoringFolderIds?: ReadonlySet } + options?: { restoringFolderIds?: ReadonlySet; skipNotify?: boolean } ): Promise { const table = await getTableById(tableId, { includeArchived: true }) if (!table) { @@ -988,4 +1005,9 @@ export async function restoreTable( } logger.info(`[${requestId}] Restored table ${tableId} as "${attemptedRestoreName}"`) + + // Live tables list: a restore re-adds the table to the active list. Skipped under a folder cascade — + // restoreFolder fires one folder-level notify for the whole subtree (see deleteTable). + if (table.workspaceId && !options?.skipNotify) + await notifyWorkspaceTablesChanged(table.workspaceId) } diff --git a/packages/platform-authz/src/rooms.ts b/packages/platform-authz/src/rooms.ts index 3688271d93a..6d76fdceb4f 100644 --- a/packages/platform-authz/src/rooms.ts +++ b/packages/platform-authz/src/rooms.ts @@ -85,6 +85,8 @@ async function resolveTableWorkspace(tableId: string): Promise> = { // A workspace-files room is addressed directly by its workspace id. [ROOM_TYPES.WORKSPACE_FILES]: resolveWorkspaceRoomWorkspace, + // A workspace-tables room is addressed directly by its workspace id. + [ROOM_TYPES.WORKSPACE_TABLES]: resolveWorkspaceRoomWorkspace, // A file-doc room is addressed by file id; resolve it to its workspace. [ROOM_TYPES.WORKSPACE_FILE_DOC]: resolveFileDocWorkspace, // A table room is addressed by table id; resolve it to its workspace. diff --git a/packages/realtime-protocol/src/rooms.ts b/packages/realtime-protocol/src/rooms.ts index f901f198f8d..afc3edde5a8 100644 --- a/packages/realtime-protocol/src/rooms.ts +++ b/packages/realtime-protocol/src/rooms.ts @@ -35,6 +35,14 @@ export const ROOM_TYPES = { * table id. */ TABLE: 'table', + /** + * The workspace tables browser (one room per workspace). The list-level + * counterpart to {@link ROOM_TYPES.TABLE}: it carries NO presence, only a + * lossy `workspace-tables-changed` invalidation signal so every viewer's + * tables list refetches when a table is created/renamed/moved/deleted. Its id + * space is the workspace id, mirroring {@link ROOM_TYPES.WORKSPACE_FILES}. + */ + WORKSPACE_TABLES: 'workspace-tables', } as const export type RoomType = (typeof ROOM_TYPES)[keyof typeof ROOM_TYPES] From 11747bdb92df8e25f7531772e0390bfce1fa98fd Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 29 Jul 2026 14:33:53 -0700 Subject: [PATCH 50/53] feat(collab-doc): Hocuspocus binary persistence + Next 16 seed/persist fixes (#6059) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(collab-doc): make server-side seed conversion work under Next 16 / Turbopack Opening a file left both collaborators read-only and stalled ~12s: the server-side seed (markdown -> Yjs, run through the headless editor engine) was failing, so the doc never seeded and the editor never left its readiness gate. Two root causes, both latent until a real build/runtime (typecheck + unit tests don't exercise either), surfaced by the Next 16 upgrade: 1. Build boundary: the server seed route imported the shared editor schema (`createMarkdownContentExtensions`), which pulled in the React node-view components (`useEffect`) -> 'client component in a Server Component'. Split each node's React-free schema into its own `*-schema.ts` (code-block, image, raw-markdown-snippet); the client editor still injects the React node views via the existing `nodeViews` param, unchanged. 2. Runtime DOM: the converter installs a jsdom `window` on `globalThis`, but Turbopack's server bundle gives bundled `@tiptap/core` a `window` that does NOT read `globalThis`, so `elementFromString` threw 'no window object available'. Externalize the `@tiptap/*` packages the converter uses (native Node require, so their `window` reads the real global) and fix the converter's DOM guard to gate on `window` (what TipTap checks) with no sticky flag. Verified: seed route returns 200 with the Yjs update; 514 collab-doc + editor tests pass; schema byte-identical after the split. * feat(collab-doc): persist the Yjs binary and load it on cold-start (Hocuspocus pattern) Adopt the industry-standard Hocuspocus store/load-document pattern so a cold room open loads the file's last-persisted Yjs binary directly instead of re-converting markdown -> Yjs on every open. Rebuilding the CRDT from markdown on each connect is the exact anti-pattern Tiptap/Yjs warn against (fresh client ids -> duplicated content); it also forced the fragile server-side headless-editor conversion on every open. Now conversion runs only on a genuine first open or an external markdown edit. - new table workspace_file_collab_state(file_id PK->workspace_files cascade, doc_state bytea, source_hash, updated_at): the Yjs binary + a hash of the markdown it was derived from (bounded <=~1MB by the 256KB round-trip gate). Mirrors Hocuspocus's extension-database (binary in a DB column). Migration 0275. - persist upserts the binary (tagged with the exact markdown just written) - cold-start seed returns the cached binary when its source_hash matches the file's current markdown; otherwise converts (and the next persist refreshes the cache) - also externalize yjs / y-protocols / lib0 alongside @tiptap: bundling loaded a second yjs copy, so @tiptap/y-tiptap's 'instanceof Y.XmlElement' failed on app-created nodes ('Unexpected case') during Yjs -> markdown Verified end-to-end: seed -> persist -> seed returns the exact persisted binary (a cache hit, no re-conversion). 18 collab-doc + 51 realtime file-doc tests pass. * fix(collab-doc): best-effort cache read + drop dead barrel - seed: a cache-read failure (transient DB error, not-yet-migrated cache table) no longer aborts a cold room open — the durable markdown is already in hand, so fall through to conversion. Symmetric with persist's best-effort cache write. Addresses the Cursor Bugbot finding on the read/write asymmetry. - remove the collab-doc index.ts barrel: nothing imported it (every consumer uses direct ./seed / ./merge / ./converter imports), so it was dead re-export surface. De-export COLLAB_DOC_FIELD accordingly — it is used only inside converter.ts. --- .../rich-markdown-editor/code-block-schema.ts | 34 + .../rich-markdown-editor/code-block.tsx | 27 +- .../rich-markdown-editor/extensions.ts | 11 +- .../rich-markdown-editor/image-schema.ts | 161 + .../rich-markdown-editor/image.tsx | 154 +- .../raw-markdown-snippet-schema.ts | 481 + .../raw-markdown-snippet.tsx | 475 +- apps/sim/lib/collab-doc/collab-state.ts | 61 + apps/sim/lib/collab-doc/converter.ts | 36 +- apps/sim/lib/collab-doc/index.ts | 8 - apps/sim/lib/collab-doc/persist.ts | 19 +- apps/sim/lib/collab-doc/seed.test.ts | 43 +- apps/sim/lib/collab-doc/seed.ts | 24 + apps/sim/next.config.ts | 22 + .../db/migrations/0275_cold_gorilla_man.sql | 8 + .../db/migrations/meta/0275_snapshot.json | 18284 ++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + packages/db/schema.ts | 30 + 18 files changed, 19201 insertions(+), 684 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block-schema.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet-schema.ts create mode 100644 apps/sim/lib/collab-doc/collab-state.ts delete mode 100644 apps/sim/lib/collab-doc/index.ts create mode 100644 packages/db/migrations/0275_cold_gorilla_man.sql create mode 100644 packages/db/migrations/meta/0275_snapshot.json diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block-schema.ts new file mode 100644 index 00000000000..5feaeaf85dc --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block-schema.ts @@ -0,0 +1,34 @@ +import type { JSONContent } from '@tiptap/core' +import { CodeBlock } from '@tiptap/extension-code-block' + +/** + * React-free schema half of the code-block node. Lives apart from {@link ./code-block} (its React + * node view) so the shared editor schema — `createMarkdownContentExtensions` in `./extensions` — can + * be imported by server code (the collab-doc seed converter) without pulling a client component + * (`useEffect`) into a Server Component module. The client editor injects the node-view variant + * ({@link CodeBlockWithLanguage}) via `nodeViews`. + */ + +function codeBlockText(node: JSONContent): string { + return (node.content ?? []).map((child) => child.text ?? '').join('') +} + +/** Fence sized to one backtick longer than the longest run inside the code (CommonMark rule). */ +function fenceFor(text: string): string { + const longestRun = Math.max(0, ...[...text.matchAll(/`+/g)].map((match) => match[0].length)) + return '`'.repeat(Math.max(3, longestRun + 1)) +} + +/** + * Code block whose markdown serializer sizes the fence to the interior backtick runs, so a code + * block that itself contains a ``` line round-trips instead of shattering. Shared by the test + * (plain) and live ({@link CodeBlockWithLanguage}) paths. + */ +export const MarkdownCodeBlock = CodeBlock.extend({ + renderMarkdown: (node: JSONContent) => { + const language = typeof node.attrs?.language === 'string' ? node.attrs.language : '' + const text = codeBlockText(node) + const fence = fenceFor(text) + return `${fence}${language}\n${text}\n${fence}` + }, +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block.tsx index 7db9b1481b9..2464ccaec44 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block.tsx @@ -8,12 +8,11 @@ import { DropdownMenuTrigger, useCopyToClipboard, } from '@sim/emcn' -import type { JSONContent } from '@tiptap/core' -import { CodeBlock } from '@tiptap/extension-code-block' import type { ReactNodeViewProps } from '@tiptap/react' import { NodeViewContent, NodeViewWrapper, ReactNodeViewRenderer } from '@tiptap/react' import { Check, ChevronDown, Code, Copy, Eye, WrapText } from 'lucide-react' import { looksLikeMermaid, MermaidDiagram } from '../mermaid-diagram' +import { MarkdownCodeBlock } from './code-block-schema' import { detectLanguage } from './detect-language' import { useEditorEditable } from './use-editor-editable' @@ -228,30 +227,6 @@ function CodeBlockView({ node, updateAttributes, editor, getPos }: ReactNodeView ) } -function codeBlockText(node: JSONContent): string { - return (node.content ?? []).map((child) => child.text ?? '').join('') -} - -/** Fence sized to one backtick longer than the longest run inside the code (CommonMark rule). */ -function fenceFor(text: string): string { - const longestRun = Math.max(0, ...[...text.matchAll(/`+/g)].map((match) => match[0].length)) - return '`'.repeat(Math.max(3, longestRun + 1)) -} - -/** - * Code block whose markdown serializer sizes the fence to the interior backtick runs, so a code - * block that itself contains a ``` line round-trips instead of shattering. Shared by the test - * (plain) and live ({@link CodeBlockWithLanguage}) paths. - */ -export const MarkdownCodeBlock = CodeBlock.extend({ - renderMarkdown: (node: JSONContent) => { - const language = typeof node.attrs?.language === 'string' ? node.attrs.language : '' - const text = codeBlockText(node) - const fence = fenceFor(text) - return `${fence}${language}\n${text}\n${fence}` - }, -}) - /** * Code block with hover-revealed controls (language picker, line-wrap toggle, copy). The * `language` attribute drives {@link CodeBlockHighlight}'s Prism highlighting and serializes to diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts index 0f83b3a32ef..8c7d42342f9 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts @@ -11,13 +11,18 @@ import { } from '@tiptap/extension-table' import { Markdown } from '@tiptap/markdown' import StarterKit from '@tiptap/starter-kit' -import { MarkdownCodeBlock } from './code-block' +import { MarkdownCodeBlock } from './code-block-schema' import { Highlight } from './highlight' -import { MarkdownImage } from './image' +import { MarkdownImage } from './image-schema' import { MarkdownLinkInputRule } from './link-input-rule' import { MarkdownMention } from './mention/mention-node' import { SIM_LINK_SCHEME } from './mention/sim-link' -import { FootnoteDef, FootnoteRef, RawHtmlBlock, RawInlineHtml } from './raw-markdown-snippet' +import { + FootnoteDef, + FootnoteRef, + RawHtmlBlock, + RawInlineHtml, +} from './raw-markdown-snippet-schema' /** * The `@`-mention link scheme, registered on the Link mark — without it the schema strips the diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts new file mode 100644 index 00000000000..dccc926d6b4 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts @@ -0,0 +1,161 @@ +import type { JSONContent } from '@tiptap/core' +import { Image } from '@tiptap/extension-image' + +/** + * React-free schema half of the image node. Lives apart from {@link ./image} (its React resize node + * view) so the shared editor schema — `createMarkdownContentExtensions` in `./extensions` — can be + * imported by server code (the collab-doc seed converter) without pulling a client component + * (`useEffect`) into a Server Component module. The client editor injects the node-view variant + * ({@link ResizableImage}) via `nodeViews`. + */ + +/** + * A markdown linked image `[![alt](src "t")](href "t2")` — an image wrapped in a link, the canonical + * form of a README badge. `@tiptap/markdown` parses this as a link mark over an image node, but an + * image node can't carry inline marks, so the wrapping link is silently dropped. We instead tokenize + * the whole construct ourselves and hang the link target on the image node's `href` attribute, so it + * round-trips losslessly (and the file stays editable rather than opening read-only). + */ +const LINKED_IMAGE_RE = + /^\[!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)/ + +/** Escape a value for safe interpolation into a double-quoted HTML attribute. */ +function escapeAttr(value: string): string { + return value + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(//g, '>') +} + +/** + * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when + * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to + * preserve its dimensions. Unsized images stay clean `![alt](src)`. An image with an `href` is + * wrapped in a markdown link so a linked badge round-trips as `[![alt](src)](href)`. + * + * A *sized **and** linked* image is the one case markdown can't represent: the linked-image tokenizer + * only recognizes `[![alt](src)](href)`, so emitting `[](href)` would silently drop the link on + * reparse (and the round-trip-safety probe wouldn't catch it). We keep the link and fall back to the + * unsized `[![alt](src)](href)` form — the link matters more than the exact dimensions for a badge. + */ +function imageMarkdown(node: JSONContent): string { + const attrs = node.attrs ?? {} + const src = typeof attrs.src === 'string' ? attrs.src : '' + const alt = typeof attrs.alt === 'string' ? attrs.alt : '' + const title = typeof attrs.title === 'string' ? attrs.title : '' + const href = typeof attrs.href === 'string' ? attrs.href : '' + const hrefTitle = typeof attrs.hrefTitle === 'string' ? attrs.hrefTitle : '' + const width = attrs.width + const height = attrs.height + let image: string + if ((width || height) && !href) { + const parts = [`src="${escapeAttr(src)}"`] + if (alt) parts.push(`alt="${escapeAttr(alt)}"`) + if (title) parts.push(`title="${escapeAttr(title)}"`) + if (width) parts.push(`width="${escapeAttr(String(width))}"`) + if (height) parts.push(`height="${escapeAttr(String(height))}"`) + image = `` + } else { + // Escape so an alt with `]`/`[` or a title with `"` can't break out of the `![…](… "…")` syntax + // and corrupt the round-trip; a src with spaces/parens goes in angle brackets (CommonMark). + const titlePart = title ? ` "${title.replace(/["\\]/g, '\\$&')}"` : '' + const safeSrc = /[\s()]/.test(src) ? `<${src}>` : src + image = `![${alt.replace(/[\\[\]]/g, '\\$&')}](${safeSrc}${titlePart})` + } + if (!href) return image + // Escape `"`/`\` so an href title can't break out of the `[…](href "title")` syntax (mirrors the + // image title escaping above). + const hrefTitlePart = hrefTitle ? ` "${hrefTitle.replace(/["\\]/g, '\\$&')}"` : '' + return `[${image}](${href}${hrefTitlePart})` +} + +interface MarkdownImageToken { + /** Set only by our linked-image tokenizer; absent on the built-in `![](src)` token. */ + src?: string + alt?: string + title?: string | null + /** Built-in image token holds the source URL here; our linked token holds the link target. */ + href?: string + hrefTitle?: string | null + /** Built-in image token holds the alt text here. */ + text?: string +} + +/** Map both the built-in image token and our linked-image token onto the image node's attributes. */ +function parseImageToken(token: MarkdownImageToken): JSONContent { + const isLinked = typeof token.src === 'string' + return { + type: 'image', + attrs: isLinked + ? { + src: token.src, + alt: token.alt ?? '', + title: token.title ?? null, + href: token.href ?? null, + hrefTitle: token.hrefTitle ?? null, + } + : { + src: token.href ?? '', + alt: token.text ?? '', + title: token.title ?? null, + href: null, + hrefTitle: null, + }, + } +} + +const widthAttr = { + default: null, + parseHTML: (element: HTMLElement) => element.getAttribute('width'), + renderHTML: (attributes: Record) => + attributes.width ? { width: String(attributes.width) } : {}, +} + +const heightAttr = { + default: null, + parseHTML: (element: HTMLElement) => element.getAttribute('height'), + renderHTML: (attributes: Record) => + attributes.height ? { height: String(attributes.height) } : {}, +} + +/** Link target of a linked image — markdown-only state, never emitted as an HTML `` attribute. */ +const hrefAttr = { default: null, rendered: false } +const hrefTitleAttr = { default: null, rendered: false } + +/** + * Image node that carries optional `width`/`height` (serialized as an HTML `` tag) and an + * optional `href`/`hrefTitle` (a wrapping markdown link, for badges). Shared by the headless + * round-trip path (no node view) and the live {@link ResizableImage}. + */ +export const MarkdownImage = Image.extend({ + addAttributes() { + return { + ...this.parent?.(), + width: widthAttr, + height: heightAttr, + href: hrefAttr, + hrefTitle: hrefTitleAttr, + } + }, + markdownTokenizer: { + name: 'image', + level: 'inline', + start: (src: string) => src.indexOf('[!['), + tokenize: (src: string): (MarkdownImageToken & { type: string; raw: string }) | undefined => { + const match = LINKED_IMAGE_RE.exec(src) + if (!match) return undefined + return { + type: 'image', + raw: match[0], + alt: match[1] ?? '', + src: match[2], + title: match[3] ?? null, + href: match[4], + hrefTitle: match[5] ?? null, + } + }, + }, + parseMarkdown: parseImageToken, + renderMarkdown: imageMarkdown, +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx index fa198e8658e..fc9ce30fc1b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx @@ -1,167 +1,15 @@ import { useEffect, useRef, useState } from 'react' import { cn } from '@sim/emcn' -import type { JSONContent } from '@tiptap/core' -import { Image } from '@tiptap/extension-image' import { NodeSelection, Plugin } from '@tiptap/pm/state' import type { ReactNodeViewProps } from '@tiptap/react' import { NodeViewWrapper, ReactNodeViewRenderer } from '@tiptap/react' import { useFileContentSource } from '@/hooks/use-file-content-source' +import { MarkdownImage } from './image-schema' import { normalizeLinkHref } from './markdown-fidelity' import { useEditorEditable } from './use-editor-editable' const MIN_WIDTH = 64 -/** - * A markdown linked image `[![alt](src "t")](href "t2")` — an image wrapped in a link, the canonical - * form of a README badge. `@tiptap/markdown` parses this as a link mark over an image node, but an - * image node can't carry inline marks, so the wrapping link is silently dropped. We instead tokenize - * the whole construct ourselves and hang the link target on the image node's `href` attribute, so it - * round-trips losslessly (and the file stays editable rather than opening read-only). - */ -const LINKED_IMAGE_RE = - /^\[!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)/ - -/** Escape a value for safe interpolation into a double-quoted HTML attribute. */ -function escapeAttr(value: string): string { - return value - .replace(/&/g, '&') - .replace(/"/g, '"') - .replace(//g, '>') -} - -/** - * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when - * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to - * preserve its dimensions. Unsized images stay clean `![alt](src)`. An image with an `href` is - * wrapped in a markdown link so a linked badge round-trips as `[![alt](src)](href)`. - * - * A *sized **and** linked* image is the one case markdown can't represent: the linked-image tokenizer - * only recognizes `[![alt](src)](href)`, so emitting `[](href)` would silently drop the link on - * reparse (and the round-trip-safety probe wouldn't catch it). We keep the link and fall back to the - * unsized `[![alt](src)](href)` form — the link matters more than the exact dimensions for a badge. - */ -function imageMarkdown(node: JSONContent): string { - const attrs = node.attrs ?? {} - const src = typeof attrs.src === 'string' ? attrs.src : '' - const alt = typeof attrs.alt === 'string' ? attrs.alt : '' - const title = typeof attrs.title === 'string' ? attrs.title : '' - const href = typeof attrs.href === 'string' ? attrs.href : '' - const hrefTitle = typeof attrs.hrefTitle === 'string' ? attrs.hrefTitle : '' - const width = attrs.width - const height = attrs.height - let image: string - if ((width || height) && !href) { - const parts = [`src="${escapeAttr(src)}"`] - if (alt) parts.push(`alt="${escapeAttr(alt)}"`) - if (title) parts.push(`title="${escapeAttr(title)}"`) - if (width) parts.push(`width="${escapeAttr(String(width))}"`) - if (height) parts.push(`height="${escapeAttr(String(height))}"`) - image = `` - } else { - // Escape so an alt with `]`/`[` or a title with `"` can't break out of the `![…](… "…")` syntax - // and corrupt the round-trip; a src with spaces/parens goes in angle brackets (CommonMark). - const titlePart = title ? ` "${title.replace(/["\\]/g, '\\$&')}"` : '' - const safeSrc = /[\s()]/.test(src) ? `<${src}>` : src - image = `![${alt.replace(/[\\[\]]/g, '\\$&')}](${safeSrc}${titlePart})` - } - if (!href) return image - // Escape `"`/`\` so an href title can't break out of the `[…](href "title")` syntax (mirrors the - // image title escaping above). - const hrefTitlePart = hrefTitle ? ` "${hrefTitle.replace(/["\\]/g, '\\$&')}"` : '' - return `[${image}](${href}${hrefTitlePart})` -} - -interface MarkdownImageToken { - /** Set only by our linked-image tokenizer; absent on the built-in `![](src)` token. */ - src?: string - alt?: string - title?: string | null - /** Built-in image token holds the source URL here; our linked token holds the link target. */ - href?: string - hrefTitle?: string | null - /** Built-in image token holds the alt text here. */ - text?: string -} - -/** Map both the built-in image token and our linked-image token onto the image node's attributes. */ -function parseImageToken(token: MarkdownImageToken): JSONContent { - const isLinked = typeof token.src === 'string' - return { - type: 'image', - attrs: isLinked - ? { - src: token.src, - alt: token.alt ?? '', - title: token.title ?? null, - href: token.href ?? null, - hrefTitle: token.hrefTitle ?? null, - } - : { - src: token.href ?? '', - alt: token.text ?? '', - title: token.title ?? null, - href: null, - hrefTitle: null, - }, - } -} - -const widthAttr = { - default: null, - parseHTML: (element: HTMLElement) => element.getAttribute('width'), - renderHTML: (attributes: Record) => - attributes.width ? { width: String(attributes.width) } : {}, -} - -const heightAttr = { - default: null, - parseHTML: (element: HTMLElement) => element.getAttribute('height'), - renderHTML: (attributes: Record) => - attributes.height ? { height: String(attributes.height) } : {}, -} - -/** Link target of a linked image — markdown-only state, never emitted as an HTML `` attribute. */ -const hrefAttr = { default: null, rendered: false } -const hrefTitleAttr = { default: null, rendered: false } - -/** - * Image node that carries optional `width`/`height` (serialized as an HTML `` tag) and an - * optional `href`/`hrefTitle` (a wrapping markdown link, for badges). Shared by the headless - * round-trip path (no node view) and the live {@link ResizableImage}. - */ -export const MarkdownImage = Image.extend({ - addAttributes() { - return { - ...this.parent?.(), - width: widthAttr, - height: heightAttr, - href: hrefAttr, - hrefTitle: hrefTitleAttr, - } - }, - markdownTokenizer: { - name: 'image', - level: 'inline', - start: (src: string) => src.indexOf('[!['), - tokenize: (src: string): (MarkdownImageToken & { type: string; raw: string }) | undefined => { - const match = LINKED_IMAGE_RE.exec(src) - if (!match) return undefined - return { - type: 'image', - raw: match[0], - alt: match[1] ?? '', - src: match[2], - title: match[3] ?? null, - href: match[4], - hrefTitle: match[5] ?? null, - } - }, - }, - parseMarkdown: parseImageToken, - renderMarkdown: imageMarkdown, -}) - /** * Drag-to-resize image node view (handle at the bottom-right, revealed on selection). Dragging * commits the new pixel width to the `width` attribute, which serializes to ``. diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet-schema.ts new file mode 100644 index 00000000000..abe01ce42e2 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet-schema.ts @@ -0,0 +1,481 @@ +/** + * React-free schema half of the raw-HTML/footnote nodes. Lives apart from {@link ./raw-markdown-snippet} + * (their React node views) so the shared editor schema — `createMarkdownContentExtensions` in + * `./extensions` — can be imported by server code (the collab-doc seed converter) without pulling a + * client component (`useEffect`) into a Server Component module. The client editor injects the + * node-view variants ({@link RawHtmlBlockWithView}, {@link FootnoteDefWithView}) via `nodeViews`. + */ +import type { JSONContent, MarkdownToken } from '@tiptap/core' +import { mergeAttributes, Node } from '@tiptap/core' + +/** + * Constructs the schema has no node/mark for: raw HTML blocks (`
`, `
`, …), HTML + * comments, and footnotes. Before this file, all four made the *entire* document open read-only + * (see {@link isRoundTripSafe in ./round-trip-safety}) because the stock pipeline silently drops + * or mangles them. Each node below instead holds the exact source text as its content and + * re-emits it byte-for-byte on serialize — the same "hold raw source, re-render specially" shape + * `MarkdownCodeBlock` uses for Mermaid (see `./code-block.tsx`), just without the diagram render. + * + * Inline tags already covered by a real mark/node — `em`/`i`, `strong`/`b`, `s`/`del`/`strike`, + * `code`, `a`, `br`, `img` — are deliberately excluded from {@link RawInlineHtml} so they keep + * parsing into their proper mark (e.g. `x` → italic) instead of freezing as raw source. + */ +const HANDLED_INLINE_TAGS = new Set([ + 'br', + 'img', + 'em', + 'i', + 'strong', + 'b', + 's', + 'del', + 'strike', + 'code', + 'a', +]) + +const VOID_TAGS = new Set([ + 'area', + 'base', + 'br', + 'col', + 'embed', + 'hr', + 'img', + 'input', + 'link', + 'meta', + 'param', + 'source', + 'track', + 'wbr', +]) + +function verbatimText(node: JSONContent): string { + return (node.content ?? []).map((child) => child.text ?? '').join('') +} + +const RAW_HTML_COMMENT_RE = /^/ + +/** + * One HTML attribute: `name` or `name="value"`/`name='value'`/`name=bare`. The quoted-value + * alternatives are what matter — `[^"]*`/`[^']*` consume a literal `>` inside the quotes as part of + * the value, so an attribute like `data-example="a > b"` is treated as one unit instead of ending + * the tag match at the internal `>`. + */ +const ATTRS_RE_SOURCE = String.raw`(?:\s+[^\s"'=<>\`]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>\`]+))?)*` + +/** Matches one opening HTML tag, attributes included (see {@link ATTRS_RE_SOURCE}). Group 1 is the + * tag name, group 2 is the self-closing `/` if present — shared by inline and block tokenizing. */ +const OPEN_TAG_RE = new RegExp(`^<([a-z][\\w-]*)\\b${ATTRS_RE_SOURCE}\\s*(/)?>`, 'i') + +/** A fenced block's opening/closing marker may sit inside a blockquote (each line prefixed with + * up to 3 spaces then one or more `>` markers, each optionally followed by a space) and/or be + * independently indented up to 3 spaces with no blockquote at all (CommonMark's own fence-indent + * tolerance — matches `FENCE_OPEN`/`FENCE_CLOSE` in `./markdown-parse.ts`) — matched on both the + * open and close fence line so `> \`\`\`` and ` \`\`\`` both mask correctly. */ +const FENCE_PREFIX_SOURCE = '(?:[ ]{0,3}>[ ]?)*[ ]{0,3}' + +/** Same as {@link RAW_HTML_COMMENT_RE} but not anchored to the start of the string — used by + * {@link maskCodeRegions} to find a comment anywhere in the scanned text, not just at position 0. */ +const HTML_COMMENT_ANYWHERE_RE = //g + +/** + * Mask fenced code blocks, inline code spans, and HTML comments with same-length filler (newlines + * kept, everything else replaced with a space) so a tag-like mention *inside one of these* — + * `` `
` ``, a fenced example showing HTML syntax, or a comment documenting the tag + * (``) — is never mistaken for a real balancing tag while scanning. Mirrors + * the fenced/inline patterns `stripCode` in `./round-trip-safety.ts` matches (extended to also + * tolerate an indented and/or blockquoted fence marker via {@link FENCE_PREFIX_SOURCE}, since a raw + * HTML block can itself be indented or quoted), but preserves length/position (masks in place) + * instead of deleting, so match indices still map onto the original, unmodified `src` the caller + * slices from. + */ +function maskCodeRegions(src: string): string { + const fenceRe = new RegExp( + `^${FENCE_PREFIX_SOURCE}([\`~]{3,})[^\\n]*\\n[\\s\\S]*?^${FENCE_PREFIX_SOURCE}\\1[\`~]*[ \\t]*$`, + 'gm' + ) + return src + .replace(fenceRe, (m) => m.replace(/[^\n]/g, ' ')) + .replace(/`+[^`\n]*`+/g, (m) => ' '.repeat(m.length)) + .replace(HTML_COMMENT_ANYWHERE_RE, (m) => m.replace(/[^\n]/g, ' ')) +} + +/** + * Find the end of the close tag that balances the open tag of `tag` ending at `src[0, fromIndex)`, + * tracking nesting depth from `fromIndex` onward so `outer inner` consumes + * both levels instead of stopping at the first (inner) ``. Returns -1 if unterminated. A + * nested self-closing same-name tag (``) is skipped — it neither opens nor closes a level. + * Shared by the inline tokenizer (single line) and the block tokenizer (spans blank lines). + * + * Scans a {@link maskCodeRegions}-masked copy of `src` so a tag name mentioned inside code doesn't + * count as real markup — this narrows, but can't eliminate, the inherent ambiguity of regex-based + * (non-DOM) tag matching: a *bare, unescaped* mention of the same tag name in plain prose (not in + * code) is indistinguishable from a real closing tag here, exactly as it would be to a real HTML + * parser given the same ambiguous input (there is no valid way to "escape" a literal `` inside + * real HTML content other than an entity or code region). Verified this can't lose data even in that + * case — the result still reaches a stable fixpoint on save, just restructured — matching this file's + * "reject on doubt, but never require doubt-free input" gate (`isRoundTripSafe`). + */ +function findBalancedCloseEnd(src: string, tag: string, fromIndex: number): number { + const masked = maskCodeRegions(src) + const tagRe = new RegExp(`<(/?)${tag}\\b${ATTRS_RE_SOURCE}\\s*(/)?>`, 'gi') + tagRe.lastIndex = fromIndex + let depth = 1 + for (let match = tagRe.exec(masked); match; match = tagRe.exec(masked)) { + const isClose = match[1] === '/' + const isSelfClosing = Boolean(match[2]) + if (isSelfClosing) continue + if (isClose) { + depth -= 1 + if (depth === 0) return match.index + match[0].length + } else { + depth += 1 + } + } + return -1 +} + +/** + * Marked's own block tokenizer greedily consumes the blank-line run *after* an HTML block/comment + * or a def line as part of that token's own `raw` (the same behavior `PipeSafeTable` in + * `./extensions.ts` works around for tables) — storing it verbatim would double it up with the + * block joiner's own separator, growing by two newlines on every save. Block-level callers trim it; + * inline callers never carry one (inline tokens can't span a blank line), so trimming is a no-op there. + */ +function verbatimParse(raw: string): JSONContent[] { + const trimmed = raw.replace(/\n+$/, '') + return trimmed ? [{ type: 'text', text: trimmed }] : [] +} + +interface VerbatimNodeOptions { + name: string + /** Whether this node sits among block content (own line) or inline content (mid-paragraph). */ + inline: boolean + badgeLabel: string +} + +/** + * Shared shape for a node that holds a markdown construct's exact source text and re-emits it + * unchanged — parsing and rendering never inspect or transform the text, so there is nothing for + * these constructs to lose. `markdownTokenName`/`parseMarkdown`/`renderMarkdown` are read directly + * off the returned config by `@tiptap/markdown`'s `MarkdownManager` (see + * `node_modules/@tiptap/markdown/src/MarkdownManager.ts`), independent of the node's `name`. + */ +function verbatimNodeConfig({ name, inline, badgeLabel }: VerbatimNodeOptions) { + return { + name, + inline, + group: inline ? 'inline' : 'block', + content: 'text*', + marks: '', + code: true, + defining: !inline, + // Block verbatim nodes hold exact source text; `isolating` stops a boundary Backspace/Delete from + // joining across their edge, which would otherwise merge their raw markdown into an adjacent + // paragraph as HTML-escaped prose and destroy the node (silent data loss on save). + isolating: !inline, + selectable: true, + atom: false, + parseHTML() { + return [ + { + tag: `${inline ? 'span' : 'div'}[data-raw-markdown="${name}"]`, + preserveWhitespace: 'full' as const, + }, + ] + }, + renderHTML({ HTMLAttributes }: { HTMLAttributes: Record }) { + return [ + inline ? 'span' : 'div', + mergeAttributes(HTMLAttributes, { + 'data-raw-markdown': name, + 'data-raw-markdown-label': badgeLabel, + class: inline ? 'raw-markdown-inline' : 'raw-markdown-block', + }), + 0, + ] as const + }, + renderMarkdown(node: JSONContent) { + return verbatimText(node) + }, + } +} + +/** + * Tag names CommonMark/GFM treat as "block-starting" HTML (marked's own type-6 list — see + * `_tag` in `node_modules/marked/src/rules.ts`, verified against the CommonMark spec): a block + * opening with one of these ends at its *matching closing tag*, not at the first blank line. Tags + * NOT in this list (`em`, `a`, `span`, `code`, `kbd`, …) can legitimately start an ordinary + * paragraph (`hi there`), so they're deliberately left to marked's own stricter, single-line + * block-HTML detection below — claiming them here would risk swallowing a paragraph that merely + * starts with inline HTML. + */ +const BLOCK_HTML_TAG_NAMES = new Set([ + 'address', + 'article', + 'aside', + 'base', + 'basefont', + 'blockquote', + 'body', + 'caption', + 'center', + 'col', + 'colgroup', + 'dd', + 'details', + 'dialog', + 'dir', + 'div', + 'dl', + 'dt', + 'fieldset', + 'figcaption', + 'figure', + 'footer', + 'form', + 'frame', + 'frameset', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'head', + 'header', + 'hr', + 'html', + 'iframe', + 'legend', + 'li', + 'link', + 'main', + 'menu', + 'menuitem', + 'meta', + 'nav', + 'noframes', + 'ol', + 'optgroup', + 'option', + 'p', + 'param', + 'search', + 'section', + 'summary', + 'table', + 'tbody', + 'td', + 'tfoot', + 'th', + 'thead', + 'title', + 'tr', + 'track', + 'ul', +]) + +/** + * Marked's built-in block-HTML rule ends a `
`/`
`/… block at the *first blank line* + * (CommonMark's HTML-block-type-6 rule) — correct for normal rendering, but wrong for verbatim + * preservation: any real-world `
` with a paragraph inside would fragment into a raw chip, + * an ordinary (rendered) paragraph, and a second raw chip, stranding genuine content in between. + * This tokenizer instead scans to the tag's *matching* close via {@link findBalancedCloseEnd}, blank + * lines included, for tags in {@link BLOCK_HTML_TAG_NAMES}; anything else returns `undefined` and + * falls through to the existing `markdownTokenName: 'html'` handling below (marked's own block + * tokenizer, unchanged). Comments are matched the same way as the inline case — marked's own comment + * rule already spans blank lines correctly, but routing through one path keeps the two tokenizers + * symmetric and independently testable. CommonMark allows up to 3 leading spaces before a block-HTML + * opening line, so the leading indent is split off, matched against separately, and stitched back + * onto `raw` — everything after that first line (including the tag's own body) can be indented + * however the author wrote it, since the balanced scan doesn't care about column position there. + */ +function tokenizeRawHtmlBlockTag(src: string): MarkdownToken | undefined { + const indent = /^ {0,3}/.exec(src)?.[0] ?? '' + const rest = src.slice(indent.length) + + const comment = RAW_HTML_COMMENT_RE.exec(rest) + if (comment) { + const raw = indent + comment[0] + return { type: 'html', raw, text: raw, block: true } + } + + const open = OPEN_TAG_RE.exec(rest) + if (!open) return undefined + const tag = open[1].toLowerCase() + if (!BLOCK_HTML_TAG_NAMES.has(tag)) return undefined + // A handful of BLOCK_HTML_TAG_NAMES entries (link, meta, base, col, …) are void elements with no + // closing tag at all — treat them as complete right after the open tag (like an explicit `/>`), + // same as `tokenizeRawInlineHtml` already does via VOID_TAGS. Without this, scanning for a + // ``/`` that will never legitimately appear risks grabbing unrelated later content + // (or a stray same-name mention) as if it belonged to this block. + if (open[2] || VOID_TAGS.has(tag)) { + const raw = indent + open[0] + return { type: 'html', raw, text: raw, block: true } + } + + const end = findBalancedCloseEnd(rest, tag, open[0].length) + if (end < 0) return undefined + const raw = indent + rest.slice(0, end) + return { type: 'html', raw, text: raw, block: true } +} + +const SKIP_BLOCK_HTML_TAGS = /^<(img|br)\b[^>]*\/?>\s*$/i + +export const RawHtmlBlock = Node.create({ + ...verbatimNodeConfig({ name: 'rawHtmlBlock', inline: false, badgeLabel: 'Raw HTML' }), + markdownTokenName: 'html', + markdownTokenizer: { + name: 'rawHtmlBlockTag', + level: 'block' as const, + // Always -1 (never claims an early interrupt point): when `start` is omitted, `@tiptap/markdown` + // auto-generates one that calls `this.createLexer()` on every paragraph-continuation check, which + // corrupts the in-progress lexer's shared state (verified directly — every other construct on the + // page silently loses its content once a tokenizer without an explicit `start` is registered). + // The other custom tokenizers below all reference this comment rather than repeating it. + // + // The tokenizer above emits `type: 'html'` explicitly, so its tokens flow into the same + // `markdownTokenName: 'html'` parse registration as marked's own block-HTML tokens below — the + // distinct `name` here only avoids colliding with marked's own built-in `html` extension. + start: () => -1, + tokenize: tokenizeRawHtmlBlockTag, + }, + parseMarkdown(token: MarkdownToken) { + if (!token.block) return [] + const raw = token.raw ?? token.text ?? '' + if (!raw.trim()) return [] + // A lone ``/`
` tag block — leave it to the stock path (Image node / hard break), + // matching the same exclusion `round-trip-safety.ts` used to carve out for these two tags. + if (SKIP_BLOCK_HTML_TAGS.test(raw.trim())) return [] + return { type: 'rawHtmlBlock', content: verbatimParse(raw) } + }, +}) + +const FOOTNOTE_DEF_HEAD_RE = /^ {0,3}\[\^[^\]]+\]:/ +const FOOTNOTE_CONTINUATION_RE = /^ {4,}\S/ + +/** + * Consume a footnote definition's opening line plus any continuation lines GFM allows — indented by + * ≥4 spaces, optionally with blank lines between them (a multi-paragraph definition). Stops at the + * first line that is neither indented nor blank, and never consumes a blank line that isn't followed + * by further continuation (that blank line belongs to whatever block comes next). + */ +function tokenizeFootnoteDef(src: string): MarkdownToken | undefined { + const lines = src.split('\n') + if (!FOOTNOTE_DEF_HEAD_RE.test(lines[0])) return undefined + let lineCount = 1 + while (lineCount < lines.length) { + const line = lines[lineCount] + if (FOOTNOTE_CONTINUATION_RE.test(line)) { + lineCount += 1 + continue + } + if (line === '' && FOOTNOTE_CONTINUATION_RE.test(lines[lineCount + 1] ?? '')) { + lineCount += 2 + continue + } + break + } + const raw = lines.slice(0, lineCount).join('\n') + return { type: 'footnoteDef', raw, text: raw } +} + +/** Footnote definition (`[^id]: the note`, with optional ≥4-space-indented continuation lines) — + * marked has no footnote syntax at all, so without this tokenizer the definition is swallowed as a + * plain paragraph and the reference/definition link is lost. */ +export const FootnoteDef = Node.create({ + ...verbatimNodeConfig({ name: 'footnoteDef', inline: false, badgeLabel: 'Footnote' }), + markdownTokenName: 'footnoteDef', + markdownTokenizer: { + name: 'footnoteDef', + level: 'block' as const, + // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. The cost + // here is narrow and safe: a footnote def sharing a line-run with the preceding paragraph (no + // blank line between them) is picked up on the next block boundary instead of interrupting early. + start: () => -1, + tokenize: tokenizeFootnoteDef, + }, + parseMarkdown(token: MarkdownToken) { + const raw = token.raw ?? token.text ?? '' + if (!raw) return [] + return { type: 'footnoteDef', content: verbatimParse(raw) } + }, +}) + +const FOOTNOTE_REF_RE = /^\[\^[^\]]+\]/ + +/** Footnote reference (`text[^id]`) — verbatim passthrough, same rationale as {@link FootnoteDef}. */ +export const FootnoteRef = Node.create({ + ...verbatimNodeConfig({ name: 'footnoteRef', inline: true, badgeLabel: 'Footnote ref' }), + markdownTokenName: 'footnoteRef', + markdownTokenizer: { + name: 'footnoteRef', + level: 'inline' as const, + // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. + start: () => -1, + tokenize(src: string) { + const match = FOOTNOTE_REF_RE.exec(src) + if (!match) return undefined + return { type: 'footnoteRef', raw: match[0], text: match[0] } + }, + }, + parseMarkdown(token: MarkdownToken) { + const raw = token.raw ?? token.text ?? '' + if (!raw) return [] + return { type: 'footnoteRef', content: verbatimParse(raw) } + }, +}) + +/** + * Attempt to consume an inline HTML comment or a tag (with its matching close tag, or as a single + * void/self-closing element) starting at `src[0]`. Returns `undefined` for a tag this schema + * already has a real mark/node for ({@link HANDLED_INLINE_TAGS}) so it keeps parsing normally, and + * for an unterminated open tag (rare/malformed input — falls back to the stock, lossy behavior + * rather than risk mis-consuming the rest of the document). + */ +function tokenizeRawInlineHtml(src: string): MarkdownToken | undefined { + const comment = RAW_HTML_COMMENT_RE.exec(src) + if (comment) return { type: 'rawInlineHtml', raw: comment[0], text: comment[0] } + + const open = OPEN_TAG_RE.exec(src) + if (!open) return undefined + const tag = open[1].toLowerCase() + if (HANDLED_INLINE_TAGS.has(tag)) return undefined + if (open[2] || VOID_TAGS.has(tag)) { + return { type: 'rawInlineHtml', raw: open[0], text: open[0] } + } + + const end = findBalancedCloseEnd(src, tag, open[0].length) + if (end < 0) return undefined + const raw = src.slice(0, end) + return { type: 'rawInlineHtml', raw, text: raw } +} + +/** Inline raw HTML — ``, ``, ``, ``, `` (no Underline mark is registered), + * and any other tag this schema has no mark/node for, plus an inline-position HTML comment. Marked + * classifies inline HTML as its own `'html'` token type, and `@tiptap/markdown`'s inline parser + * hardcodes handling for that type *before* checking its extension registry (unlike block tokens) — + * so claiming it here needs a custom tokenizer, registered under a different token name + * (`rawInlineHtml`) so it's never confused with the stock `'html'` inline path. marked.js runs + * custom extension tokenizers before its own built-ins at both block and inline level (see + * `blockTokens`/`inlineTokens` in `node_modules/marked/lib/marked.esm.js`), so this reliably wins + * the race against marked's default inline HTML/tag tokenizer. */ +export const RawInlineHtml = Node.create({ + ...verbatimNodeConfig({ name: 'rawInlineHtml', inline: true, badgeLabel: 'Raw HTML' }), + markdownTokenName: 'rawInlineHtml', + markdownTokenizer: { + name: 'rawInlineHtml', + level: 'inline' as const, + // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. + start: () => -1, + tokenize: tokenizeRawInlineHtml, + }, + parseMarkdown(token: MarkdownToken) { + const raw = token.raw ?? token.text ?? '' + if (!raw) return [] + return { type: 'rawInlineHtml', content: verbatimParse(raw) } + }, +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet.tsx index 1aad444de29..ea4058895e7 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet.tsx @@ -1,479 +1,6 @@ -import type { JSONContent, MarkdownToken } from '@tiptap/core' -import { mergeAttributes, Node } from '@tiptap/core' import type { ReactNodeViewProps } from '@tiptap/react' import { NodeViewContent, NodeViewWrapper, ReactNodeViewRenderer } from '@tiptap/react' - -/** - * Constructs the schema has no node/mark for: raw HTML blocks (`
`, `
`, …), HTML - * comments, and footnotes. Before this file, all four made the *entire* document open read-only - * (see {@link isRoundTripSafe in ./round-trip-safety}) because the stock pipeline silently drops - * or mangles them. Each node below instead holds the exact source text as its content and - * re-emits it byte-for-byte on serialize — the same "hold raw source, re-render specially" shape - * `MarkdownCodeBlock` uses for Mermaid (see `./code-block.tsx`), just without the diagram render. - * - * Inline tags already covered by a real mark/node — `em`/`i`, `strong`/`b`, `s`/`del`/`strike`, - * `code`, `a`, `br`, `img` — are deliberately excluded from {@link RawInlineHtml} so they keep - * parsing into their proper mark (e.g. `x` → italic) instead of freezing as raw source. - */ -const HANDLED_INLINE_TAGS = new Set([ - 'br', - 'img', - 'em', - 'i', - 'strong', - 'b', - 's', - 'del', - 'strike', - 'code', - 'a', -]) - -const VOID_TAGS = new Set([ - 'area', - 'base', - 'br', - 'col', - 'embed', - 'hr', - 'img', - 'input', - 'link', - 'meta', - 'param', - 'source', - 'track', - 'wbr', -]) - -function verbatimText(node: JSONContent): string { - return (node.content ?? []).map((child) => child.text ?? '').join('') -} - -const RAW_HTML_COMMENT_RE = /^/ - -/** - * One HTML attribute: `name` or `name="value"`/`name='value'`/`name=bare`. The quoted-value - * alternatives are what matter — `[^"]*`/`[^']*` consume a literal `>` inside the quotes as part of - * the value, so an attribute like `data-example="a > b"` is treated as one unit instead of ending - * the tag match at the internal `>`. - */ -const ATTRS_RE_SOURCE = String.raw`(?:\s+[^\s"'=<>\`]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>\`]+))?)*` - -/** Matches one opening HTML tag, attributes included (see {@link ATTRS_RE_SOURCE}). Group 1 is the - * tag name, group 2 is the self-closing `/` if present — shared by inline and block tokenizing. */ -const OPEN_TAG_RE = new RegExp(`^<([a-z][\\w-]*)\\b${ATTRS_RE_SOURCE}\\s*(/)?>`, 'i') - -/** A fenced block's opening/closing marker may sit inside a blockquote (each line prefixed with - * up to 3 spaces then one or more `>` markers, each optionally followed by a space) and/or be - * independently indented up to 3 spaces with no blockquote at all (CommonMark's own fence-indent - * tolerance — matches `FENCE_OPEN`/`FENCE_CLOSE` in `./markdown-parse.ts`) — matched on both the - * open and close fence line so `> \`\`\`` and ` \`\`\`` both mask correctly. */ -const FENCE_PREFIX_SOURCE = '(?:[ ]{0,3}>[ ]?)*[ ]{0,3}' - -/** Same as {@link RAW_HTML_COMMENT_RE} but not anchored to the start of the string — used by - * {@link maskCodeRegions} to find a comment anywhere in the scanned text, not just at position 0. */ -const HTML_COMMENT_ANYWHERE_RE = //g - -/** - * Mask fenced code blocks, inline code spans, and HTML comments with same-length filler (newlines - * kept, everything else replaced with a space) so a tag-like mention *inside one of these* — - * `` `
` ``, a fenced example showing HTML syntax, or a comment documenting the tag - * (``) — is never mistaken for a real balancing tag while scanning. Mirrors - * the fenced/inline patterns `stripCode` in `./round-trip-safety.ts` matches (extended to also - * tolerate an indented and/or blockquoted fence marker via {@link FENCE_PREFIX_SOURCE}, since a raw - * HTML block can itself be indented or quoted), but preserves length/position (masks in place) - * instead of deleting, so match indices still map onto the original, unmodified `src` the caller - * slices from. - */ -function maskCodeRegions(src: string): string { - const fenceRe = new RegExp( - `^${FENCE_PREFIX_SOURCE}([\`~]{3,})[^\\n]*\\n[\\s\\S]*?^${FENCE_PREFIX_SOURCE}\\1[\`~]*[ \\t]*$`, - 'gm' - ) - return src - .replace(fenceRe, (m) => m.replace(/[^\n]/g, ' ')) - .replace(/`+[^`\n]*`+/g, (m) => ' '.repeat(m.length)) - .replace(HTML_COMMENT_ANYWHERE_RE, (m) => m.replace(/[^\n]/g, ' ')) -} - -/** - * Find the end of the close tag that balances the open tag of `tag` ending at `src[0, fromIndex)`, - * tracking nesting depth from `fromIndex` onward so `outer inner` consumes - * both levels instead of stopping at the first (inner) ``. Returns -1 if unterminated. A - * nested self-closing same-name tag (``) is skipped — it neither opens nor closes a level. - * Shared by the inline tokenizer (single line) and the block tokenizer (spans blank lines). - * - * Scans a {@link maskCodeRegions}-masked copy of `src` so a tag name mentioned inside code doesn't - * count as real markup — this narrows, but can't eliminate, the inherent ambiguity of regex-based - * (non-DOM) tag matching: a *bare, unescaped* mention of the same tag name in plain prose (not in - * code) is indistinguishable from a real closing tag here, exactly as it would be to a real HTML - * parser given the same ambiguous input (there is no valid way to "escape" a literal `` inside - * real HTML content other than an entity or code region). Verified this can't lose data even in that - * case — the result still reaches a stable fixpoint on save, just restructured — matching this file's - * "reject on doubt, but never require doubt-free input" gate (`isRoundTripSafe`). - */ -function findBalancedCloseEnd(src: string, tag: string, fromIndex: number): number { - const masked = maskCodeRegions(src) - const tagRe = new RegExp(`<(/?)${tag}\\b${ATTRS_RE_SOURCE}\\s*(/)?>`, 'gi') - tagRe.lastIndex = fromIndex - let depth = 1 - for (let match = tagRe.exec(masked); match; match = tagRe.exec(masked)) { - const isClose = match[1] === '/' - const isSelfClosing = Boolean(match[2]) - if (isSelfClosing) continue - if (isClose) { - depth -= 1 - if (depth === 0) return match.index + match[0].length - } else { - depth += 1 - } - } - return -1 -} - -/** - * Marked's own block tokenizer greedily consumes the blank-line run *after* an HTML block/comment - * or a def line as part of that token's own `raw` (the same behavior `PipeSafeTable` in - * `./extensions.ts` works around for tables) — storing it verbatim would double it up with the - * block joiner's own separator, growing by two newlines on every save. Block-level callers trim it; - * inline callers never carry one (inline tokens can't span a blank line), so trimming is a no-op there. - */ -function verbatimParse(raw: string): JSONContent[] { - const trimmed = raw.replace(/\n+$/, '') - return trimmed ? [{ type: 'text', text: trimmed }] : [] -} - -interface VerbatimNodeOptions { - name: string - /** Whether this node sits among block content (own line) or inline content (mid-paragraph). */ - inline: boolean - badgeLabel: string -} - -/** - * Shared shape for a node that holds a markdown construct's exact source text and re-emits it - * unchanged — parsing and rendering never inspect or transform the text, so there is nothing for - * these constructs to lose. `markdownTokenName`/`parseMarkdown`/`renderMarkdown` are read directly - * off the returned config by `@tiptap/markdown`'s `MarkdownManager` (see - * `node_modules/@tiptap/markdown/src/MarkdownManager.ts`), independent of the node's `name`. - */ -function verbatimNodeConfig({ name, inline, badgeLabel }: VerbatimNodeOptions) { - return { - name, - inline, - group: inline ? 'inline' : 'block', - content: 'text*', - marks: '', - code: true, - defining: !inline, - // Block verbatim nodes hold exact source text; `isolating` stops a boundary Backspace/Delete from - // joining across their edge, which would otherwise merge their raw markdown into an adjacent - // paragraph as HTML-escaped prose and destroy the node (silent data loss on save). - isolating: !inline, - selectable: true, - atom: false, - parseHTML() { - return [ - { - tag: `${inline ? 'span' : 'div'}[data-raw-markdown="${name}"]`, - preserveWhitespace: 'full' as const, - }, - ] - }, - renderHTML({ HTMLAttributes }: { HTMLAttributes: Record }) { - return [ - inline ? 'span' : 'div', - mergeAttributes(HTMLAttributes, { - 'data-raw-markdown': name, - 'data-raw-markdown-label': badgeLabel, - class: inline ? 'raw-markdown-inline' : 'raw-markdown-block', - }), - 0, - ] as const - }, - renderMarkdown(node: JSONContent) { - return verbatimText(node) - }, - } -} - -/** - * Tag names CommonMark/GFM treat as "block-starting" HTML (marked's own type-6 list — see - * `_tag` in `node_modules/marked/src/rules.ts`, verified against the CommonMark spec): a block - * opening with one of these ends at its *matching closing tag*, not at the first blank line. Tags - * NOT in this list (`em`, `a`, `span`, `code`, `kbd`, …) can legitimately start an ordinary - * paragraph (`hi there`), so they're deliberately left to marked's own stricter, single-line - * block-HTML detection below — claiming them here would risk swallowing a paragraph that merely - * starts with inline HTML. - */ -const BLOCK_HTML_TAG_NAMES = new Set([ - 'address', - 'article', - 'aside', - 'base', - 'basefont', - 'blockquote', - 'body', - 'caption', - 'center', - 'col', - 'colgroup', - 'dd', - 'details', - 'dialog', - 'dir', - 'div', - 'dl', - 'dt', - 'fieldset', - 'figcaption', - 'figure', - 'footer', - 'form', - 'frame', - 'frameset', - 'h1', - 'h2', - 'h3', - 'h4', - 'h5', - 'h6', - 'head', - 'header', - 'hr', - 'html', - 'iframe', - 'legend', - 'li', - 'link', - 'main', - 'menu', - 'menuitem', - 'meta', - 'nav', - 'noframes', - 'ol', - 'optgroup', - 'option', - 'p', - 'param', - 'search', - 'section', - 'summary', - 'table', - 'tbody', - 'td', - 'tfoot', - 'th', - 'thead', - 'title', - 'tr', - 'track', - 'ul', -]) - -/** - * Marked's built-in block-HTML rule ends a `
`/`
`/… block at the *first blank line* - * (CommonMark's HTML-block-type-6 rule) — correct for normal rendering, but wrong for verbatim - * preservation: any real-world `
` with a paragraph inside would fragment into a raw chip, - * an ordinary (rendered) paragraph, and a second raw chip, stranding genuine content in between. - * This tokenizer instead scans to the tag's *matching* close via {@link findBalancedCloseEnd}, blank - * lines included, for tags in {@link BLOCK_HTML_TAG_NAMES}; anything else returns `undefined` and - * falls through to the existing `markdownTokenName: 'html'` handling below (marked's own block - * tokenizer, unchanged). Comments are matched the same way as the inline case — marked's own comment - * rule already spans blank lines correctly, but routing through one path keeps the two tokenizers - * symmetric and independently testable. CommonMark allows up to 3 leading spaces before a block-HTML - * opening line, so the leading indent is split off, matched against separately, and stitched back - * onto `raw` — everything after that first line (including the tag's own body) can be indented - * however the author wrote it, since the balanced scan doesn't care about column position there. - */ -function tokenizeRawHtmlBlockTag(src: string): MarkdownToken | undefined { - const indent = /^ {0,3}/.exec(src)?.[0] ?? '' - const rest = src.slice(indent.length) - - const comment = RAW_HTML_COMMENT_RE.exec(rest) - if (comment) { - const raw = indent + comment[0] - return { type: 'html', raw, text: raw, block: true } - } - - const open = OPEN_TAG_RE.exec(rest) - if (!open) return undefined - const tag = open[1].toLowerCase() - if (!BLOCK_HTML_TAG_NAMES.has(tag)) return undefined - // A handful of BLOCK_HTML_TAG_NAMES entries (link, meta, base, col, …) are void elements with no - // closing tag at all — treat them as complete right after the open tag (like an explicit `/>`), - // same as `tokenizeRawInlineHtml` already does via VOID_TAGS. Without this, scanning for a - // ``/`` that will never legitimately appear risks grabbing unrelated later content - // (or a stray same-name mention) as if it belonged to this block. - if (open[2] || VOID_TAGS.has(tag)) { - const raw = indent + open[0] - return { type: 'html', raw, text: raw, block: true } - } - - const end = findBalancedCloseEnd(rest, tag, open[0].length) - if (end < 0) return undefined - const raw = indent + rest.slice(0, end) - return { type: 'html', raw, text: raw, block: true } -} - -const SKIP_BLOCK_HTML_TAGS = /^<(img|br)\b[^>]*\/?>\s*$/i - -export const RawHtmlBlock = Node.create({ - ...verbatimNodeConfig({ name: 'rawHtmlBlock', inline: false, badgeLabel: 'Raw HTML' }), - markdownTokenName: 'html', - markdownTokenizer: { - name: 'rawHtmlBlockTag', - level: 'block' as const, - // Always -1 (never claims an early interrupt point): when `start` is omitted, `@tiptap/markdown` - // auto-generates one that calls `this.createLexer()` on every paragraph-continuation check, which - // corrupts the in-progress lexer's shared state (verified directly — every other construct on the - // page silently loses its content once a tokenizer without an explicit `start` is registered). - // The other custom tokenizers below all reference this comment rather than repeating it. - // - // The tokenizer above emits `type: 'html'` explicitly, so its tokens flow into the same - // `markdownTokenName: 'html'` parse registration as marked's own block-HTML tokens below — the - // distinct `name` here only avoids colliding with marked's own built-in `html` extension. - start: () => -1, - tokenize: tokenizeRawHtmlBlockTag, - }, - parseMarkdown(token: MarkdownToken) { - if (!token.block) return [] - const raw = token.raw ?? token.text ?? '' - if (!raw.trim()) return [] - // A lone ``/`
` tag block — leave it to the stock path (Image node / hard break), - // matching the same exclusion `round-trip-safety.ts` used to carve out for these two tags. - if (SKIP_BLOCK_HTML_TAGS.test(raw.trim())) return [] - return { type: 'rawHtmlBlock', content: verbatimParse(raw) } - }, -}) - -const FOOTNOTE_DEF_HEAD_RE = /^ {0,3}\[\^[^\]]+\]:/ -const FOOTNOTE_CONTINUATION_RE = /^ {4,}\S/ - -/** - * Consume a footnote definition's opening line plus any continuation lines GFM allows — indented by - * ≥4 spaces, optionally with blank lines between them (a multi-paragraph definition). Stops at the - * first line that is neither indented nor blank, and never consumes a blank line that isn't followed - * by further continuation (that blank line belongs to whatever block comes next). - */ -function tokenizeFootnoteDef(src: string): MarkdownToken | undefined { - const lines = src.split('\n') - if (!FOOTNOTE_DEF_HEAD_RE.test(lines[0])) return undefined - let lineCount = 1 - while (lineCount < lines.length) { - const line = lines[lineCount] - if (FOOTNOTE_CONTINUATION_RE.test(line)) { - lineCount += 1 - continue - } - if (line === '' && FOOTNOTE_CONTINUATION_RE.test(lines[lineCount + 1] ?? '')) { - lineCount += 2 - continue - } - break - } - const raw = lines.slice(0, lineCount).join('\n') - return { type: 'footnoteDef', raw, text: raw } -} - -/** Footnote definition (`[^id]: the note`, with optional ≥4-space-indented continuation lines) — - * marked has no footnote syntax at all, so without this tokenizer the definition is swallowed as a - * plain paragraph and the reference/definition link is lost. */ -export const FootnoteDef = Node.create({ - ...verbatimNodeConfig({ name: 'footnoteDef', inline: false, badgeLabel: 'Footnote' }), - markdownTokenName: 'footnoteDef', - markdownTokenizer: { - name: 'footnoteDef', - level: 'block' as const, - // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. The cost - // here is narrow and safe: a footnote def sharing a line-run with the preceding paragraph (no - // blank line between them) is picked up on the next block boundary instead of interrupting early. - start: () => -1, - tokenize: tokenizeFootnoteDef, - }, - parseMarkdown(token: MarkdownToken) { - const raw = token.raw ?? token.text ?? '' - if (!raw) return [] - return { type: 'footnoteDef', content: verbatimParse(raw) } - }, -}) - -const FOOTNOTE_REF_RE = /^\[\^[^\]]+\]/ - -/** Footnote reference (`text[^id]`) — verbatim passthrough, same rationale as {@link FootnoteDef}. */ -export const FootnoteRef = Node.create({ - ...verbatimNodeConfig({ name: 'footnoteRef', inline: true, badgeLabel: 'Footnote ref' }), - markdownTokenName: 'footnoteRef', - markdownTokenizer: { - name: 'footnoteRef', - level: 'inline' as const, - // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. - start: () => -1, - tokenize(src: string) { - const match = FOOTNOTE_REF_RE.exec(src) - if (!match) return undefined - return { type: 'footnoteRef', raw: match[0], text: match[0] } - }, - }, - parseMarkdown(token: MarkdownToken) { - const raw = token.raw ?? token.text ?? '' - if (!raw) return [] - return { type: 'footnoteRef', content: verbatimParse(raw) } - }, -}) - -/** - * Attempt to consume an inline HTML comment or a tag (with its matching close tag, or as a single - * void/self-closing element) starting at `src[0]`. Returns `undefined` for a tag this schema - * already has a real mark/node for ({@link HANDLED_INLINE_TAGS}) so it keeps parsing normally, and - * for an unterminated open tag (rare/malformed input — falls back to the stock, lossy behavior - * rather than risk mis-consuming the rest of the document). - */ -function tokenizeRawInlineHtml(src: string): MarkdownToken | undefined { - const comment = RAW_HTML_COMMENT_RE.exec(src) - if (comment) return { type: 'rawInlineHtml', raw: comment[0], text: comment[0] } - - const open = OPEN_TAG_RE.exec(src) - if (!open) return undefined - const tag = open[1].toLowerCase() - if (HANDLED_INLINE_TAGS.has(tag)) return undefined - if (open[2] || VOID_TAGS.has(tag)) { - return { type: 'rawInlineHtml', raw: open[0], text: open[0] } - } - - const end = findBalancedCloseEnd(src, tag, open[0].length) - if (end < 0) return undefined - const raw = src.slice(0, end) - return { type: 'rawInlineHtml', raw, text: raw } -} - -/** Inline raw HTML — ``, ``, ``, ``, `` (no Underline mark is registered), - * and any other tag this schema has no mark/node for, plus an inline-position HTML comment. Marked - * classifies inline HTML as its own `'html'` token type, and `@tiptap/markdown`'s inline parser - * hardcodes handling for that type *before* checking its extension registry (unlike block tokens) — - * so claiming it here needs a custom tokenizer, registered under a different token name - * (`rawInlineHtml`) so it's never confused with the stock `'html'` inline path. marked.js runs - * custom extension tokenizers before its own built-ins at both block and inline level (see - * `blockTokens`/`inlineTokens` in `node_modules/marked/lib/marked.esm.js`), so this reliably wins - * the race against marked's default inline HTML/tag tokenizer. */ -export const RawInlineHtml = Node.create({ - ...verbatimNodeConfig({ name: 'rawInlineHtml', inline: true, badgeLabel: 'Raw HTML' }), - markdownTokenName: 'rawInlineHtml', - markdownTokenizer: { - name: 'rawInlineHtml', - level: 'inline' as const, - // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. - start: () => -1, - tokenize: tokenizeRawInlineHtml, - }, - parseMarkdown(token: MarkdownToken) { - const raw = token.raw ?? token.text ?? '' - if (!raw) return [] - return { type: 'rawInlineHtml', content: verbatimParse(raw) } - }, -}) +import { FootnoteDef, RawHtmlBlock } from './raw-markdown-snippet-schema' const BLOCK_CONTROL_CLASS = 'pointer-events-none absolute top-1.5 right-2 select-none rounded-md bg-[var(--surface-4)] px-1.5 py-0.5 text-[10px] text-[var(--text-muted)] uppercase tracking-wide opacity-0 transition-opacity group-focus-within:opacity-100 group-hover:opacity-100' diff --git a/apps/sim/lib/collab-doc/collab-state.ts b/apps/sim/lib/collab-doc/collab-state.ts new file mode 100644 index 00000000000..97828bece51 --- /dev/null +++ b/apps/sim/lib/collab-doc/collab-state.ts @@ -0,0 +1,61 @@ +import { createHash } from 'crypto' +import { db } from '@sim/db' +import { workspaceFileCollabState } from '@sim/db/schema' +import { eq } from 'drizzle-orm' + +/** + * The cached-collab-state cache (`workspace_file_collab_state`) lets a cold room open load the file's + * last-persisted Yjs binary directly instead of re-converting markdown → Yjs on every open — the + * Hocuspocus load-document pattern. See {@link workspaceFileCollabState} for the full rationale. + */ + +/** sha256 (hex) of a markdown buffer — the freshness tag matching a cached doc state to the live file. */ +export function hashMarkdown(markdown: Buffer): string { + return createHash('sha256').update(markdown).digest('hex') +} + +/** + * Load a file's cached Yjs binary IF it is still fresh — i.e. was derived from markdown whose hash + * matches `sourceHash` (the current file's markdown). Returns the binary to apply directly, or `null` + * when there is no cache or it is stale (the markdown changed externally since it was saved), in which + * case the caller re-converts from markdown. Applying the stored binary — rather than rebuilding a Y.Doc + * from markdown — is what preserves the CRDT's client ids and prevents duplicated content on reconnect. + */ +export async function loadFreshCollabDocState( + fileId: string, + sourceHash: string +): Promise { + const [row] = await db + .select({ + docState: workspaceFileCollabState.docState, + sourceHash: workspaceFileCollabState.sourceHash, + }) + .from(workspaceFileCollabState) + .where(eq(workspaceFileCollabState.fileId, fileId)) + .limit(1) + + if (!row || row.sourceHash !== sourceHash) return null + return new Uint8Array(row.docState) +} + +/** + * Persist a collaborative doc's Yjs binary as the file's cold-start state, tagged with the hash of the + * markdown it was derived from. Upsert — one row per file. Called from the server-side persist right + * after the markdown is written, so the cached binary and its `sourceHash` are always consistent with + * the file that was just saved. + */ +export async function saveCollabDocState( + fileId: string, + docState: Uint8Array, + sourceHash: string +): Promise { + const state = Buffer.from(docState) + const updatedAt = new Date() + await db + .insert(workspaceFileCollabState) + .values({ fileId, docState: state, sourceHash, updatedAt }) + .onConflictDoUpdate({ + target: workspaceFileCollabState.fileId, + set: { docState: state, sourceHash, updatedAt }, + }) +} diff --git a/apps/sim/lib/collab-doc/converter.ts b/apps/sim/lib/collab-doc/converter.ts index c3c5b9cc69c..990e0f906c6 100644 --- a/apps/sim/lib/collab-doc/converter.ts +++ b/apps/sim/lib/collab-doc/converter.ts @@ -41,7 +41,7 @@ import { * `Collaboration.configure({ document })` with no explicit `field`, so it uses TipTap's default, * `'default'`. The server MUST target the same fragment or the client would sync an empty document. */ -export const COLLAB_DOC_FIELD = 'default' +const COLLAB_DOC_FIELD = 'default' let cachedSchema: Schema | null = null @@ -51,27 +51,29 @@ function markdownSchema(): Schema { return cachedSchema } -let domReady = false - /** - * Ensure a DOM exists for the TipTap editor the markdown engine constructs. In a `jsdom` test - * environment `document` already exists and this is a no-op; in a plain Node server it installs a - * single shared jsdom window's globals once. Kept minimal and idempotent — TipTap only needs - * `document`/`window`/`navigator` to build its (never-mounted) editor for parse/serialize. + * Ensure a DOM exists for the TipTap editor the markdown engine constructs. In a `jsdom`/browser + * environment `window` + `document` already exist and this is a no-op; in a plain Node server it + * installs a single shared jsdom window's globals. Cheap and idempotent — TipTap only needs + * `window`/`document`/`navigator` to build its (never-mounted) editor for parse/serialize. + * + * Gate on `window` (what TipTap's `elementFromString` actually checks), not just `document`, and hold + * NO cached "ready" flag: the Next server runtime exposes a partial `document` with NO `window`, and a + * `document`-only guard (plus a sticky flag) skipped this setup — leaving TipTap to throw "there is no + * window object available". Re-checking the globals every call means a partial stub can never wedge it. + * When `window` is missing we install a coherent jsdom window+document pair, overwriting any such stub. */ function ensureDomForTipTap(): void { - if (domReady || typeof document !== 'undefined') { - domReady = true - return - } - // Lazy require so the client bundle never pulls jsdom in. + if (typeof window !== 'undefined' && typeof document !== 'undefined') return + // Lazy require so the client bundle never pulls jsdom in. Bind to `jsdomWindow`, NOT `window` — a + // local `const window` would shadow the global and put the `typeof window` guard above in its + // temporal dead zone ("Cannot access 'window' before initialization"). const { JSDOM } = require('jsdom') as typeof import('jsdom') - const { window } = new JSDOM('') + const { window: jsdomWindow } = new JSDOM('') const g = globalThis as unknown as Record - g.window ??= window - g.document ??= window.document - g.navigator ??= window.navigator - domReady = true + g.window = jsdomWindow + g.document = jsdomWindow.document + g.navigator ??= jsdomWindow.navigator } /** Convert a file's markdown to a fresh collaborative {@link Y.Doc} (cold-start seed). */ diff --git a/apps/sim/lib/collab-doc/index.ts b/apps/sim/lib/collab-doc/index.ts deleted file mode 100644 index 1309372b0bf..00000000000 --- a/apps/sim/lib/collab-doc/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export { - applyMarkdownToYDoc, - COLLAB_DOC_FIELD, - markdownToYDoc, - yDocToMarkdown, -} from './converter' -export { buildFileDocMergeUpdate } from './merge' -export { buildFileDocSeed, type FileDocSeed } from './seed' diff --git a/apps/sim/lib/collab-doc/persist.ts b/apps/sim/lib/collab-doc/persist.ts index 2a0a2393292..2c69d875d46 100644 --- a/apps/sim/lib/collab-doc/persist.ts +++ b/apps/sim/lib/collab-doc/persist.ts @@ -1,6 +1,8 @@ import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import * as Y from 'yjs' import { getWorkspaceFile, updateWorkspaceFileContent } from '@/lib/uploads/contexts/workspace' +import { hashMarkdown, saveCollabDocState } from './collab-state' import { yDocToFileMarkdown } from './converter' const logger = createLogger('FileDocPersist') @@ -29,14 +31,27 @@ export async function persistFileDoc( if (!record) return false const ydoc = new Y.Doc() + let markdownBuffer: Buffer try { Y.applyUpdate(ydoc, docState) - const markdown = yDocToFileMarkdown(ydoc) - await updateWorkspaceFileContent(workspaceId, fileId, userId, Buffer.from(markdown, 'utf-8')) + markdownBuffer = Buffer.from(yDocToFileMarkdown(ydoc), 'utf-8') } finally { ydoc.destroy() } + await updateWorkspaceFileContent(workspaceId, fileId, userId, markdownBuffer) + + // Cache the Yjs binary (tagged with the exact markdown just written) so a later cold room open loads + // it directly instead of re-converting markdown → Yjs. Best-effort: the markdown IS the durable file, + // so a cache failure only means the next cold open re-converts — never lost data. + try { + await saveCollabDocState(fileId, docState, hashMarkdown(markdownBuffer)) + } catch (error) { + logger.warn(`Failed to cache collab doc state for file ${fileId}`, { + error: getErrorMessage(error), + }) + } + logger.info(`Persisted live collaborative document to file ${fileId} (workspace ${workspaceId})`) return true } diff --git a/apps/sim/lib/collab-doc/seed.test.ts b/apps/sim/lib/collab-doc/seed.test.ts index 8dcd660bb90..a3785a1c5ec 100644 --- a/apps/sim/lib/collab-doc/seed.test.ts +++ b/apps/sim/lib/collab-doc/seed.test.ts @@ -5,9 +5,10 @@ import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' import { beforeEach, describe, expect, it, vi } from 'vitest' import * as Y from 'yjs' -const { mockGetWorkspaceFile, mockFetchBuffer } = vi.hoisted(() => ({ +const { mockGetWorkspaceFile, mockFetchBuffer, mockLoadFresh } = vi.hoisted(() => ({ mockGetWorkspaceFile: vi.fn(), mockFetchBuffer: vi.fn(), + mockLoadFresh: vi.fn(), })) vi.mock('@/lib/uploads/contexts/workspace', () => ({ @@ -15,6 +16,13 @@ vi.mock('@/lib/uploads/contexts/workspace', () => ({ fetchWorkspaceFileBuffer: mockFetchBuffer, })) +// The DB-backed cold-start cache is exercised in its own suite; here we default it to a MISS so these +// tests cover the markdown → Yjs conversion path (the cache-hit fast path has its own test below). +vi.mock('./collab-state', () => ({ + hashMarkdown: () => 'test-source-hash', + loadFreshCollabDocState: mockLoadFresh, +})) + import { serializeMarkdownBody } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' import { yDocToMarkdown } from './converter' import { buildFileDocSeed } from './seed' @@ -28,6 +36,8 @@ describe('buildFileDocSeed', () => { key: 'k', context: 'workspace', }) + // Default: no cached binary → the conversion path runs (the case these tests cover). + mockLoadFresh.mockResolvedValue(null) }) it('builds a seed whose applied update reproduces the file body (through the client engine)', async () => { @@ -41,6 +51,37 @@ describe('buildFileDocSeed', () => { expect(yDocToMarkdown(doc)).toBe(serializeMarkdownBody('# Title\n\nHello **world**.')) }) + it('cold-start fast path: returns the cached binary directly without re-converting when it is fresh', async () => { + // A cached binary derived from the current markdown → seed returns it verbatim (no conversion), the + // Hocuspocus load-document path that preserves the CRDT's client ids across reopens. + const cachedDoc = new Y.Doc() + cachedDoc.getText('marker').insert(0, 'cached') + const cached = Y.encodeStateAsUpdate(cachedDoc) + mockFetchBuffer.mockResolvedValue(Buffer.from('# Anything', 'utf-8')) + mockLoadFresh.mockResolvedValue(cached) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + + expect(seed?.update).toBe(cached) + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(doc.getText('marker').toString()).toBe('cached') + }) + + it('falls through to conversion when the cache read fails (never blocks a cold open)', async () => { + // The cache is a best-effort optimization over the durable markdown we already hold; a transient DB + // error or a not-yet-migrated cache table must convert, not abort the seed. + mockFetchBuffer.mockResolvedValue(Buffer.from('# Title\n\ntext.', 'utf-8')) + mockLoadFresh.mockRejectedValue(new Error('cache table missing')) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + expect(seed).not.toBeNull() + + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(yDocToMarkdown(doc)).toBe(serializeMarkdownBody('# Title\n\ntext.')) + }) + it('strips frontmatter — only the body seeds the collaborative doc', async () => { mockFetchBuffer.mockResolvedValue(Buffer.from('---\ntitle: X\n---\n\n# Body\n\ntext.', 'utf-8')) diff --git a/apps/sim/lib/collab-doc/seed.ts b/apps/sim/lib/collab-doc/seed.ts index d83653cc0da..4fbfaa544c3 100644 --- a/apps/sim/lib/collab-doc/seed.ts +++ b/apps/sim/lib/collab-doc/seed.ts @@ -1,9 +1,14 @@ +import { createLogger } from '@sim/logger' import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import { getErrorMessage } from '@sim/utils/errors' import * as Y from 'yjs' import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { hashMarkdown, loadFreshCollabDocState } from './collab-state' import { markdownToYDoc } from './converter' +const logger = createLogger('FileDocSeed') + /** * The largest file we will build a collaborative seed for. Beyond this the editor uses its * non-collaborative path anyway; converting a huge document server-side would be wasted work. @@ -37,6 +42,25 @@ export async function buildFileDocSeed( if (!record) return null const buffer = await fetchWorkspaceFileBuffer(record, { maxBytes: MAX_SEED_BYTES }) + + // Cold-start fast path: if we hold a cached Yjs binary derived from THIS exact markdown, apply it + // directly (the Hocuspocus load-document pattern) instead of re-converting. This preserves the CRDT's + // client ids across reopens — no duplicated content, no split-brain — and skips the server-side + // headless conversion. A stale/absent cache (markdown edited externally, or first ever open) falls + // through to the conversion below, and the next persist refreshes the cache. + // + // Best-effort read: the cache is an optimization over the durable markdown we already hold, so a + // transient DB error (or a not-yet-migrated cache table) must fall through to conversion rather than + // block the cold open — symmetric with persist's best-effort cache write. + try { + const cached = await loadFreshCollabDocState(fileId, hashMarkdown(buffer)) + if (cached) return { update: cached } + } catch (error) { + logger.warn(`Failed to read cached collab doc state for file ${fileId}`, { + error: getErrorMessage(error), + }) + } + const markdown = buffer.toString('utf-8') const { frontmatter, body } = splitFrontmatter(markdown) diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index 5e16f1a9797..1e5ed4245e6 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -135,6 +135,28 @@ const nextConfig: NextConfig = { // The collab-doc seed converter lazily `require`s jsdom for a headless TipTap editor. Keep it // external so webpack doesn't try to bundle jsdom's dynamic internal requires. 'jsdom', + // The collab-doc converter runs TipTap + Yjs headlessly server-side. Two reasons these must be + // external (native Node require), not bundled: (1) the server bundler gives bundled TipTap a + // `window` that does NOT read `globalThis`, so `elementFromString` throws "no window object" even + // after the converter installs a jsdom window; (2) bundling would load a SECOND copy of `yjs`, so + // `@tiptap/y-tiptap`'s `item instanceof Y.XmlElement` checks — against the external `yjs` — would + // fail on nodes the app created with the bundled `yjs` ("Unexpected case"). One external copy fixes + // both. Server-only — the client editor bundles its own copies for the browser. + 'yjs', + 'y-protocols', + 'lib0', + '@tiptap/core', + '@tiptap/pm', + '@tiptap/markdown', + '@tiptap/y-tiptap', + '@tiptap/starter-kit', + '@tiptap/extension-code', + '@tiptap/extension-code-block', + '@tiptap/extension-image', + '@tiptap/extension-list', + '@tiptap/extension-paragraph', + '@tiptap/extension-table', + '@tiptap/extension-highlight', ], outputFileTracingIncludes: { '/api/tools/stagehand/*': ['./node_modules/ws/**/*'], diff --git a/packages/db/migrations/0275_cold_gorilla_man.sql b/packages/db/migrations/0275_cold_gorilla_man.sql new file mode 100644 index 00000000000..a40748242cc --- /dev/null +++ b/packages/db/migrations/0275_cold_gorilla_man.sql @@ -0,0 +1,8 @@ +CREATE TABLE "workspace_file_collab_state" ( + "file_id" text PRIMARY KEY NOT NULL, + "doc_state" "bytea" NOT NULL, + "source_hash" text NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "workspace_file_collab_state" ADD CONSTRAINT "workspace_file_collab_state_file_id_workspace_files_id_fk" FOREIGN KEY ("file_id") REFERENCES "public"."workspace_files"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/db/migrations/meta/0275_snapshot.json b/packages/db/migrations/meta/0275_snapshot.json new file mode 100644 index 00000000000..5c3e38b2aa5 --- /dev/null +++ b/packages/db/migrations/meta/0275_snapshot.json @@ -0,0 +1,18284 @@ +{ + "id": "8861bd0e-952d-49ba-855f-dc859d6e84fb", + "prevId": "c39b8ee3-5b5f-4790-baf5-e224c95faba7", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.academy_certificate": { + "name": "academy_certificate", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "course_id": { + "name": "course_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "academy_cert_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "certificate_number": { + "name": "certificate_number", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "academy_certificate_user_id_idx": { + "name": "academy_certificate_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_course_id_idx": { + "name": "academy_certificate_course_id_idx", + "columns": [ + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_user_course_unique": { + "name": "academy_certificate_user_course_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_number_idx": { + "name": "academy_certificate_number_idx", + "columns": [ + { + "expression": "certificate_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_status_idx": { + "name": "academy_certificate_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "academy_certificate_user_id_user_id_fk": { + "name": "academy_certificate_user_id_user_id_fk", + "tableFrom": "academy_certificate", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "academy_certificate_certificate_number_unique": { + "name": "academy_certificate_certificate_number_unique", + "nullsNotDistinct": false, + "columns": ["certificate_number"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_account_on_account_id_provider_id": { + "name": "idx_account_on_account_id_provider_id", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_key": { + "name": "api_key", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'personal'" + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_key_workspace_type_idx": { + "name": "api_key_workspace_type_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_user_type_idx": { + "name": "api_key_user_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_key_hash_idx": { + "name": "api_key_key_hash_idx", + "columns": [ + { + "expression": "key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_key_user_id_user_id_fk": { + "name": "api_key_user_id_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_workspace_id_workspace_id_fk": { + "name": "api_key_workspace_id_workspace_id_fk", + "tableFrom": "api_key", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_created_by_user_id_fk": { + "name": "api_key_created_by_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_key_key_unique": { + "name": "api_key_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": { + "workspace_type_check": { + "name": "workspace_type_check", + "value": "(type = 'workspace' AND workspace_id IS NOT NULL) OR (type = 'personal' AND workspace_id IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.async_jobs": { + "name": "async_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "run_at": { + "name": "run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output": { + "name": "output", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "async_jobs_status_started_at_idx": { + "name": "async_jobs_status_started_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_status_completed_at_idx": { + "name": "async_jobs_status_completed_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "completed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_pending_run_at_idx": { + "name": "async_jobs_schedule_pending_run_at_idx", + "columns": [ + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_processing_started_at_idx": { + "name": "async_jobs_schedule_processing_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'processing'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_name": { + "name": "actor_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_email": { + "name": "actor_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource_name": { + "name": "resource_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_log_workspace_created_idx": { + "name": "audit_log_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_workspace_created_at_id_idx": { + "name": "audit_log_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_actor_created_idx": { + "name": "audit_log_actor_created_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_resource_idx": { + "name": "audit_log_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_action_idx": { + "name": "audit_log_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "audit_log_workspace_id_workspace_id_fk": { + "name": "audit_log_workspace_id_workspace_id_fk", + "tableFrom": "audit_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "audit_log_actor_id_user_id_fk": { + "name": "audit_log_actor_id_user_id_fk", + "tableFrom": "audit_log", + "tableTo": "user", + "columnsFrom": ["actor_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.background_work_status": { + "name": "background_work_status", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "background_work_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "background_work_status_value", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "background_work_status_workspace_status_idx": { + "name": "background_work_status_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_workflow_status_idx": { + "name": "background_work_status_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_child_ws_idx": { + "name": "background_work_status_meta_child_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'childWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_other_ws_idx": { + "name": "background_work_status_meta_other_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'otherWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "background_work_status_workspace_id_workspace_id_fk": { + "name": "background_work_status_workspace_id_workspace_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "background_work_status_workflow_id_workflow_id_fk": { + "name": "background_work_status_workflow_id_workflow_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat": { + "name": "chat", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "customizations": { + "name": "customizations", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "output_configs": { + "name": "output_configs", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "include_thinking": { + "name": "include_thinking", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "include_tool_calls": { + "name": "include_tool_calls", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "identifier_idx": { + "name": "identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"chat\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_archived_at_partial_idx": { + "name": "chat_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"chat\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_chat_on_workflow_id_archived_at": { + "name": "idx_chat_on_workflow_id_archived_at", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_workflow_id_workflow_id_fk": { + "name": "chat_workflow_id_workflow_id_fk", + "tableFrom": "chat", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_user_id_user_id_fk": { + "name": "chat_user_id_user_id_fk", + "tableFrom": "chat", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_async_tool_calls": { + "name": "copilot_async_tool_calls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "checkpoint_id": { + "name": "checkpoint_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "args": { + "name": "args", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "status": { + "name": "status", + "type": "copilot_async_tool_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "permission_decision": { + "name": "permission_decision", + "type": "copilot_tool_permission_decision", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "permission_decided_at": { + "name": "permission_decided_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_async_tool_calls_run_id_idx": { + "name": "copilot_async_tool_calls_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_checkpoint_id_idx": { + "name": "copilot_async_tool_calls_checkpoint_id_idx", + "columns": [ + { + "expression": "checkpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_tool_call_id_idx": { + "name": "copilot_async_tool_calls_tool_call_id_idx", + "columns": [ + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_status_idx": { + "name": "copilot_async_tool_calls_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_run_status_idx": { + "name": "copilot_async_tool_calls_run_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_tool_call_id_unique": { + "name": "copilot_async_tool_calls_tool_call_id_unique", + "columns": [ + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_async_tool_calls_run_id_copilot_runs_id_fk": { + "name": "copilot_async_tool_calls_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk": { + "name": "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_run_checkpoints", + "columnsFrom": ["checkpoint_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_chats": { + "name": "copilot_chats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "chat_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'copilot'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'claude-3-7-sonnet-latest'" + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preview_yaml": { + "name": "preview_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan_artifact": { + "name": "plan_artifact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "auto_allowed_tools": { + "name": "auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "pinned": { + "name": "pinned", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_chats_user_id_idx": { + "name": "copilot_chats_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workflow_id_idx": { + "name": "copilot_chats_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workflow_idx": { + "name": "copilot_chats_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_idx": { + "name": "copilot_chats_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_created_at_idx": { + "name": "copilot_chats_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_updated_at_idx": { + "name": "copilot_chats_updated_at_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workspace_created_at_id_idx": { + "name": "copilot_chats_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_deleted_partial_idx": { + "name": "copilot_chats_user_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_chats\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_chats_user_id_user_id_fk": { + "name": "copilot_chats_user_id_user_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workflow_id_workflow_id_fk": { + "name": "copilot_chats_workflow_id_workflow_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workspace_id_workspace_id_fk": { + "name": "copilot_chats_workspace_id_workspace_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_feedback": { + "name": "copilot_feedback", + "schema": "", + "columns": { + "feedback_id": { + "name": "feedback_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_query": { + "name": "user_query", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_response": { + "name": "agent_response", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_positive": { + "name": "is_positive", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "feedback": { + "name": "feedback", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_yaml": { + "name": "workflow_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_feedback_user_id_idx": { + "name": "copilot_feedback_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_chat_id_idx": { + "name": "copilot_feedback_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_user_chat_idx": { + "name": "copilot_feedback_user_chat_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_is_positive_idx": { + "name": "copilot_feedback_is_positive_idx", + "columns": [ + { + "expression": "is_positive", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_created_at_idx": { + "name": "copilot_feedback_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_feedback_user_id_user_id_fk": { + "name": "copilot_feedback_user_id_user_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_feedback_chat_id_copilot_chats_id_fk": { + "name": "copilot_feedback_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_messages": { + "name": "copilot_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parent_message_id": { + "name": "parent_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens_in": { + "name": "tokens_in", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "tokens_out": { + "name": "tokens_out", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_messages_chat_message_unique": { + "name": "copilot_messages_chat_message_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_created_at_idx": { + "name": "copilot_messages_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_seq_idx": { + "name": "copilot_messages_chat_seq_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_stream_idx": { + "name": "copilot_messages_chat_stream_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"stream_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_user_created_at_idx": { + "name": "copilot_messages_user_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"role\" = 'user' AND \"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_messages_chat_id_copilot_chats_id_fk": { + "name": "copilot_messages_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_messages", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_run_checkpoints": { + "name": "copilot_run_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "pending_tool_call_id": { + "name": "pending_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conversation_snapshot": { + "name": "conversation_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "agent_state": { + "name": "agent_state", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "provider_request": { + "name": "provider_request", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_run_checkpoints_run_id_idx": { + "name": "copilot_run_checkpoints_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_pending_tool_call_id_idx": { + "name": "copilot_run_checkpoints_pending_tool_call_id_idx", + "columns": [ + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_run_pending_tool_unique": { + "name": "copilot_run_checkpoints_run_pending_tool_unique", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_run_checkpoints_run_id_copilot_runs_id_fk": { + "name": "copilot_run_checkpoints_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_run_checkpoints", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_runs": { + "name": "copilot_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_run_id": { + "name": "parent_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent": { + "name": "agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "copilot_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "request_context": { + "name": "request_context", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "copilot_runs_execution_id_idx": { + "name": "copilot_runs_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_parent_run_id_idx": { + "name": "copilot_runs_parent_run_id_idx", + "columns": [ + { + "expression": "parent_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_id_idx": { + "name": "copilot_runs_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_user_id_idx": { + "name": "copilot_runs_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workflow_id_idx": { + "name": "copilot_runs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_id_idx": { + "name": "copilot_runs_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_status_idx": { + "name": "copilot_runs_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_execution_idx": { + "name": "copilot_runs_chat_execution_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_execution_started_at_idx": { + "name": "copilot_runs_execution_started_at_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_completed_at_id_idx": { + "name": "copilot_runs_workspace_completed_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"completed_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_stream_id_unique": { + "name": "copilot_runs_stream_id_unique", + "columns": [ + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_runs_chat_id_copilot_chats_id_fk": { + "name": "copilot_runs_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_user_id_user_id_fk": { + "name": "copilot_runs_user_id_user_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workflow_id_workflow_id_fk": { + "name": "copilot_runs_workflow_id_workflow_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workspace_id_workspace_id_fk": { + "name": "copilot_runs_workspace_id_workspace_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_workflow_read_hashes": { + "name": "copilot_workflow_read_hashes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_workflow_read_hashes_chat_id_idx": { + "name": "copilot_workflow_read_hashes_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_workflow_id_idx": { + "name": "copilot_workflow_read_hashes_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_chat_workflow_unique": { + "name": "copilot_workflow_read_hashes_chat_workflow_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk": { + "name": "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_workflow_read_hashes_workflow_id_workflow_id_fk": { + "name": "copilot_workflow_read_hashes_workflow_id_workflow_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credential": { + "name": "credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "credential_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_key": { + "name": "env_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_owner_user_id": { + "name": "env_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_service_account_key": { + "name": "encrypted_service_account_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_workspace_id_idx": { + "name": "credential_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_type_idx": { + "name": "credential_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_provider_id_idx": { + "name": "credential_provider_id_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_account_id_idx": { + "name": "credential_account_id_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_env_owner_user_id_idx": { + "name": "credential_env_owner_user_id_idx", + "columns": [ + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_account_unique": { + "name": "credential_workspace_account_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "account_id IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_env_unique": { + "name": "credential_workspace_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_workspace'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_personal_env_unique": { + "name": "credential_workspace_personal_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_personal'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_workspace_id_workspace_id_fk": { + "name": "credential_workspace_id_workspace_id_fk", + "tableFrom": "credential", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_account_id_account_id_fk": { + "name": "credential_account_id_account_id_fk", + "tableFrom": "credential", + "tableTo": "account", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_env_owner_user_id_user_id_fk": { + "name": "credential_env_owner_user_id_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["env_owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_created_by_user_id_fk": { + "name": "credential_created_by_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "credential_oauth_source_check": { + "name": "credential_oauth_source_check", + "value": "(type <> 'oauth') OR (account_id IS NOT NULL AND provider_id IS NOT NULL)" + }, + "credential_workspace_env_source_check": { + "name": "credential_workspace_env_source_check", + "value": "(type <> 'env_workspace') OR (env_key IS NOT NULL AND env_owner_user_id IS NULL)" + }, + "credential_personal_env_source_check": { + "name": "credential_personal_env_source_check", + "value": "(type <> 'env_personal') OR (env_key IS NOT NULL AND env_owner_user_id IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.credential_member": { + "name": "credential_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "credential_member_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "status": { + "name": "status", + "type": "credential_member_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "joined_at": { + "name": "joined_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_member_user_id_idx": { + "name": "credential_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_role_idx": { + "name": "credential_member_role_idx", + "columns": [ + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_status_idx": { + "name": "credential_member_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_unique": { + "name": "credential_member_unique", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_member_credential_id_credential_id_fk": { + "name": "credential_member_credential_id_credential_id_fk", + "tableFrom": "credential_member", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_user_id_user_id_fk": { + "name": "credential_member_user_id_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_invited_by_user_id_fk": { + "name": "credential_member_invited_by_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_block": { + "name": "custom_block", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inputs": { + "name": "inputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "outputs": { + "name": "outputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_block_organization_id_idx": { + "name": "custom_block_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_workflow_id_idx": { + "name": "custom_block_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_organization_type_unique": { + "name": "custom_block_organization_type_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_block_organization_id_organization_id_fk": { + "name": "custom_block_organization_id_organization_id_fk", + "tableFrom": "custom_block", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_workflow_id_workflow_id_fk": { + "name": "custom_block_workflow_id_workflow_id_fk", + "tableFrom": "custom_block", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_created_by_user_id_fk": { + "name": "custom_block_created_by_user_id_fk", + "tableFrom": "custom_block", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_tools": { + "name": "custom_tools", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_tools_workspace_id_idx": { + "name": "custom_tools_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_tools_workspace_title_unique": { + "name": "custom_tools_workspace_title_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "title", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_tools_workspace_id_workspace_id_fk": { + "name": "custom_tools_workspace_id_workspace_id_fk", + "tableFrom": "custom_tools", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_tools_user_id_user_id_fk": { + "name": "custom_tools_user_id_user_id_fk", + "tableFrom": "custom_tools", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drain_runs": { + "name": "data_drain_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "drain_id": { + "name": "drain_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "data_drain_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "trigger": { + "name": "trigger", + "type": "data_drain_run_trigger", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rows_exported": { + "name": "rows_exported", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bytes_written": { + "name": "bytes_written", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "cursor_before": { + "name": "cursor_before", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cursor_after": { + "name": "cursor_after", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locators": { + "name": "locators", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + } + }, + "indexes": { + "data_drain_runs_drain_started_idx": { + "name": "data_drain_runs_drain_started_idx", + "columns": [ + { + "expression": "drain_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drain_runs_drain_id_data_drains_id_fk": { + "name": "data_drain_runs_drain_id_data_drains_id_fk", + "tableFrom": "data_drain_runs", + "tableTo": "data_drains", + "columnsFrom": ["drain_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drains": { + "name": "data_drains", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "data_drain_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_type": { + "name": "destination_type", + "type": "data_drain_destination", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_config": { + "name": "destination_config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "destination_credentials": { + "name": "destination_credentials", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schedule_cadence": { + "name": "schedule_cadence", + "type": "data_drain_cadence", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_drains_org_idx": { + "name": "data_drains_org_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_due_idx": { + "name": "data_drains_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_org_name_unique": { + "name": "data_drains_org_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drains_organization_id_organization_id_fk": { + "name": "data_drains_organization_id_organization_id_fk", + "tableFrom": "data_drains", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_drains_created_by_user_id_fk": { + "name": "data_drains_created_by_user_id_fk", + "tableFrom": "data_drains", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.docs_embeddings": { + "name": "docs_embeddings", + "schema": "", + "columns": { + "chunk_id": { + "name": "chunk_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chunk_text": { + "name": "chunk_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_document": { + "name": "source_document", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_link": { + "name": "source_link", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_text": { + "name": "header_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_level": { + "name": "header_level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": true + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "chunk_text_tsv": { + "name": "chunk_text_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"docs_embeddings\".\"chunk_text\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "docs_emb_source_document_idx": { + "name": "docs_emb_source_document_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_header_level_idx": { + "name": "docs_emb_header_level_idx", + "columns": [ + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_source_header_idx": { + "name": "docs_emb_source_header_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_model_idx": { + "name": "docs_emb_model_idx", + "columns": [ + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_created_at_idx": { + "name": "docs_emb_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_embedding_vector_hnsw_idx": { + "name": "docs_embedding_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "docs_emb_metadata_gin_idx": { + "name": "docs_emb_metadata_gin_idx", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "docs_emb_chunk_text_fts_idx": { + "name": "docs_emb_chunk_text_fts_idx", + "columns": [ + { + "expression": "chunk_text_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "docs_embedding_not_null_check": { + "name": "docs_embedding_not_null_check", + "value": "\"embedding\" IS NOT NULL" + }, + "docs_header_level_check": { + "name": "docs_header_level_check", + "value": "\"header_level\" >= 1 AND \"header_level\" <= 6" + } + }, + "isRLSEnabled": false + }, + "public.document": { + "name": "document", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_url": { + "name": "file_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "character_count": { + "name": "character_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "processing_status": { + "name": "processing_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_completed_at": { + "name": "processing_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "user_excluded": { + "name": "user_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "doc_kb_id_idx": { + "name": "doc_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_filename_idx": { + "name": "doc_filename_idx", + "columns": [ + { + "expression": "filename", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_processing_status_idx": { + "name": "doc_processing_status_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "processing_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_external_id_idx": { + "name": "doc_connector_external_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_id_idx": { + "name": "doc_connector_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_storage_key_idx": { + "name": "doc_storage_key_idx", + "columns": [ + { + "expression": "storage_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"storage_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_archived_at_partial_idx": { + "name": "doc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_deleted_at_partial_idx": { + "name": "doc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag1_idx": { + "name": "doc_tag1_idx", + "columns": [ + { + "expression": "tag1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag2_idx": { + "name": "doc_tag2_idx", + "columns": [ + { + "expression": "tag2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag3_idx": { + "name": "doc_tag3_idx", + "columns": [ + { + "expression": "tag3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag4_idx": { + "name": "doc_tag4_idx", + "columns": [ + { + "expression": "tag4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag5_idx": { + "name": "doc_tag5_idx", + "columns": [ + { + "expression": "tag5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag6_idx": { + "name": "doc_tag6_idx", + "columns": [ + { + "expression": "tag6", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag7_idx": { + "name": "doc_tag7_idx", + "columns": [ + { + "expression": "tag7", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number1_idx": { + "name": "doc_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number2_idx": { + "name": "doc_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number3_idx": { + "name": "doc_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number4_idx": { + "name": "doc_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number5_idx": { + "name": "doc_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date1_idx": { + "name": "doc_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date2_idx": { + "name": "doc_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean1_idx": { + "name": "doc_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean2_idx": { + "name": "doc_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean3_idx": { + "name": "doc_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_knowledge_base_id_knowledge_base_id_fk": { + "name": "document_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "document_connector_id_knowledge_connector_id_fk": { + "name": "document_connector_id_knowledge_connector_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "document_uploaded_by_user_id_fk": { + "name": "document_uploaded_by_user_id_fk", + "tableFrom": "document", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.embedding": { + "name": "embedding", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "chunk_hash": { + "name": "chunk_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_length": { + "name": "content_length", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "content_tsv": { + "name": "content_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"embedding\".\"content\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "emb_kb_id_idx": { + "name": "emb_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_id_idx": { + "name": "emb_doc_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_chunk_idx": { + "name": "emb_doc_chunk_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chunk_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_model_idx": { + "name": "emb_kb_model_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_enabled_idx": { + "name": "emb_kb_enabled_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_enabled_idx": { + "name": "emb_doc_enabled_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "embedding_vector_hnsw_idx": { + "name": "embedding_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "emb_tag1_idx": { + "name": "emb_tag1_idx", + "columns": [ + { + "expression": "tag1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag2_idx": { + "name": "emb_tag2_idx", + "columns": [ + { + "expression": "tag2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag3_idx": { + "name": "emb_tag3_idx", + "columns": [ + { + "expression": "tag3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag4_idx": { + "name": "emb_tag4_idx", + "columns": [ + { + "expression": "tag4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag5_idx": { + "name": "emb_tag5_idx", + "columns": [ + { + "expression": "tag5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag6_idx": { + "name": "emb_tag6_idx", + "columns": [ + { + "expression": "tag6", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag7_idx": { + "name": "emb_tag7_idx", + "columns": [ + { + "expression": "tag7", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number1_idx": { + "name": "emb_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number2_idx": { + "name": "emb_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number3_idx": { + "name": "emb_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number4_idx": { + "name": "emb_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number5_idx": { + "name": "emb_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date1_idx": { + "name": "emb_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date2_idx": { + "name": "emb_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean1_idx": { + "name": "emb_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean2_idx": { + "name": "emb_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean3_idx": { + "name": "emb_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_content_fts_idx": { + "name": "emb_content_fts_idx", + "columns": [ + { + "expression": "content_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "embedding_knowledge_base_id_knowledge_base_id_fk": { + "name": "embedding_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "embedding", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embedding_document_id_document_id_fk": { + "name": "embedding_document_id_document_id_fk", + "tableFrom": "embedding", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "embedding_not_null_check": { + "name": "embedding_not_null_check", + "value": "\"embedding\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.environment": { + "name": "environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "environment_user_id_user_id_fk": { + "name": "environment_user_id_user_id_fk", + "tableFrom": "environment", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "environment_user_id_unique": { + "name": "environment_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_dependencies": { + "name": "execution_large_value_dependencies", + "schema": "", + "columns": { + "parent_key": { + "name": "parent_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_key": { + "name": "child_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_dependencies_workspace_parent_key_idx": { + "name": "execution_large_value_dependencies_workspace_parent_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_value_dependencies_workspace_child_key_idx": { + "name": "execution_large_value_dependencies_workspace_child_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_dependencies_workspace_id_workspace_id_fk": { + "name": "execution_large_value_dependencies_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_dependencies", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_dependencies_parent_key_child_key_pk": { + "name": "execution_large_value_dependencies_parent_key_child_key_pk", + "columns": ["parent_key", "child_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_references": { + "name": "execution_large_value_references", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "execution_large_value_reference_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_references_workspace_execution_source_idx": { + "name": "execution_large_value_references_workspace_execution_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_references_workspace_id_workspace_id_fk": { + "name": "execution_large_value_references_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_value_references_workflow_id_workflow_id_fk": { + "name": "execution_large_value_references_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_references_key_execution_id_source_pk": { + "name": "execution_large_value_references_key_execution_id_source_pk", + "columns": ["key", "execution_id", "source"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_values": { + "name": "execution_large_values", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_execution_id": { + "name": "owner_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "execution_large_values_owner_execution_id_idx": { + "name": "execution_large_values_owner_execution_id_idx", + "columns": [ + { + "expression": "owner_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_cleanup_idx": { + "name": "execution_large_values_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_tombstone_cleanup_idx": { + "name": "execution_large_values_tombstone_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_values_workspace_id_workspace_id_fk": { + "name": "execution_large_values_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_values_workflow_id_workflow_id_fk": { + "name": "execution_large_values_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.folder": { + "name": "folder", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "folder_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "folder_user_idx": { + "name": "folder_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_idx": { + "name": "folder_workspace_resource_parent_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_parent_sort_idx": { + "name": "folder_parent_sort_idx", + "columns": [ + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_deleted_at_idx": { + "name": "folder_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_deleted_partial_idx": { + "name": "folder_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"folder\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_name_active_unique": { + "name": "folder_workspace_resource_parent_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"parent_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"folder\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "folder_user_id_user_id_fk": { + "name": "folder_user_id_user_id_fk", + "tableFrom": "folder", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_workspace_id_workspace_id_fk": { + "name": "folder_workspace_id_workspace_id_fk", + "tableFrom": "folder", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_parent_id_folder_id_fk": { + "name": "folder_parent_id_folder_id_fk", + "tableFrom": "folder", + "tableTo": "folder", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.idempotency_key": { + "name": "idempotency_key", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idempotency_key_created_at_idx": { + "name": "idempotency_key_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "invitation_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'organization'" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "membership_intent": { + "name": "membership_intent", + "type": "invitation_membership_intent", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'internal'" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "invitation_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_status_idx": { + "name": "invitation_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_pending_email_org_unique": { + "name": "invitation_pending_email_org_unique", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"invitation\".\"status\" = 'pending' AND \"invitation\".\"organization_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "invitation_token_unique": { + "name": "invitation_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation_workspace_grant": { + "name": "invitation_workspace_grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "invitation_id": { + "name": "invitation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission": { + "name": "permission", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_workspace_grant_unique": { + "name": "invitation_workspace_grant_unique", + "columns": [ + { + "expression": "invitation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_workspace_grant_workspace_id_idx": { + "name": "invitation_workspace_grant_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_workspace_grant_invitation_id_invitation_id_fk": { + "name": "invitation_workspace_grant_invitation_id_invitation_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "invitation", + "columnsFrom": ["invitation_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_workspace_grant_workspace_id_workspace_id_fk": { + "name": "invitation_workspace_grant_workspace_id_workspace_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.job_execution_logs": { + "name": "job_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schedule_id": { + "name": "schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost": { + "name": "cost", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "job_execution_logs_schedule_id_idx": { + "name": "job_execution_logs_schedule_id_idx", + "columns": [ + { + "expression": "schedule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_started_at_idx": { + "name": "job_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_ended_at_id_idx": { + "name": "job_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_execution_id_unique": { + "name": "job_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_trigger_idx": { + "name": "job_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "job_execution_logs_schedule_id_workflow_schedule_id_fk": { + "name": "job_execution_logs_schedule_id_workflow_schedule_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workflow_schedule", + "columnsFrom": ["schedule_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "job_execution_logs_workspace_id_workspace_id_fk": { + "name": "job_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_base": { + "name": "knowledge_base", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "embedding_dimension": { + "name": "embedding_dimension", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1536 + }, + "chunking_config": { + "name": "chunking_config", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{\"maxSize\": 1024, \"minSize\": 1, \"overlap\": 200}'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_user_id_idx": { + "name": "kb_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_id_idx": { + "name": "kb_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_user_workspace_idx": { + "name": "kb_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_folder_id_idx": { + "name": "kb_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_deleted_at_idx": { + "name": "kb_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_deleted_partial_idx": { + "name": "kb_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_base\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_name_active_unique": { + "name": "kb_workspace_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_user_id_user_id_fk": { + "name": "knowledge_base_user_id_user_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_workspace_id_workspace_id_fk": { + "name": "knowledge_base_workspace_id_workspace_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_folder_id_folder_id_fk": { + "name": "knowledge_base_folder_id_folder_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_base_tag_definitions": { + "name": "knowledge_base_tag_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_slot": { + "name": "tag_slot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "field_type": { + "name": "field_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_tag_definitions_kb_slot_idx": { + "name": "kb_tag_definitions_kb_slot_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_slot", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_display_name_idx": { + "name": "kb_tag_definitions_kb_display_name_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_id_idx": { + "name": "kb_tag_definitions_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_base_tag_definitions", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_connector": { + "name": "knowledge_connector", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connector_type": { + "name": "connector_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_config": { + "name": "source_config", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "sync_mode": { + "name": "sync_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'full'" + }, + "sync_interval_minutes": { + "name": "sync_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1440 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_sync_doc_count": { + "name": "last_sync_doc_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "next_sync_at": { + "name": "next_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kc_knowledge_base_id_idx": { + "name": "kc_knowledge_base_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_status_next_sync_idx": { + "name": "kc_status_next_sync_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_sync_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_archived_at_partial_idx": { + "name": "kc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_deleted_at_partial_idx": { + "name": "kc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_connector_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_connector", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_connector_sync_log": { + "name": "knowledge_connector_sync_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "docs_added": { + "name": "docs_added", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_updated": { + "name": "docs_updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_deleted": { + "name": "docs_deleted", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_unchanged": { + "name": "docs_unchanged", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_failed": { + "name": "docs_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kcsl_connector_id_idx": { + "name": "kcsl_connector_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_sync_log", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_server_oauth": { + "name": "mcp_server_oauth", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_server_id": { + "name": "mcp_server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_information": { + "name": "client_information", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_created_at": { + "name": "state_created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_refreshed_at": { + "name": "last_refreshed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_server_oauth_server_unique": { + "name": "mcp_server_oauth_server_unique", + "columns": [ + { + "expression": "mcp_server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_server_oauth_state_idx": { + "name": "mcp_server_oauth_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk": { + "name": "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "mcp_servers", + "columnsFrom": ["mcp_server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_server_oauth_user_id_user_id_fk": { + "name": "mcp_server_oauth_user_id_user_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "mcp_server_oauth_workspace_id_workspace_id_fk": { + "name": "mcp_server_oauth_workspace_id_workspace_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "transport": { + "name": "transport", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'headers'" + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_secret": { + "name": "oauth_client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "headers": { + "name": "headers", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "timeout": { + "name": "timeout", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30000 + }, + "retries": { + "name": "retries", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 3 + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_connected": { + "name": "last_connected", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "connection_status": { + "name": "connection_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'disconnected'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status_config": { + "name": "status_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "tool_count": { + "name": "tool_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_tools_refresh": { + "name": "last_tools_refresh", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_servers_workspace_enabled_idx": { + "name": "mcp_servers_workspace_enabled_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_workspace_deleted_partial_idx": { + "name": "mcp_servers_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"mcp_servers\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_servers_workspace_id_workspace_id_fk": { + "name": "mcp_servers_workspace_id_workspace_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_servers_created_by_user_id_fk": { + "name": "mcp_servers_created_by_user_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "member_user_id_unique": { + "name": "member_user_id_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory": { + "name": "memory", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_key_idx": { + "name": "memory_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_idx": { + "name": "memory_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_key_idx": { + "name": "memory_workspace_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_deleted_partial_idx": { + "name": "memory_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"memory\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_workspace_id_workspace_id_fk": { + "name": "memory_workspace_id_workspace_id_fk", + "tableFrom": "memory", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_allowed_sender": { + "name": "mothership_inbox_allowed_sender", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inbox_sender_ws_email_idx": { + "name": "inbox_sender_ws_email_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_allowed_sender_added_by_user_id_fk": { + "name": "mothership_inbox_allowed_sender_added_by_user_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "user", + "columnsFrom": ["added_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_task": { + "name": "mothership_inbox_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_email": { + "name": "from_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_name": { + "name": "from_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body_preview": { + "name": "body_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_text": { + "name": "body_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_html": { + "name": "body_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_message_id": { + "name": "email_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "in_reply_to": { + "name": "in_reply_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "response_message_id": { + "name": "response_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agentmail_message_id": { + "name": "agentmail_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "trigger_job_id": { + "name": "trigger_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result_summary": { + "name": "result_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rejection_reason": { + "name": "rejection_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "has_attachments": { + "name": "has_attachments", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cc_recipients": { + "name": "cc_recipients", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "inbox_task_ws_created_at_idx": { + "name": "inbox_task_ws_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_ws_status_idx": { + "name": "inbox_task_ws_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_response_msg_id_idx": { + "name": "inbox_task_response_msg_id_idx", + "columns": [ + { + "expression": "response_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_email_msg_id_idx": { + "name": "inbox_task_email_msg_id_idx", + "columns": [ + { + "expression": "email_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_task_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_task_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_task_chat_id_copilot_chats_id_fk": { + "name": "mothership_inbox_task_chat_id_copilot_chats_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_webhook": { + "name": "mothership_inbox_webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mothership_inbox_webhook_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_webhook_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_webhook", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mothership_inbox_webhook_workspace_id_unique": { + "name": "mothership_inbox_webhook_workspace_id_unique", + "nullsNotDistinct": false, + "columns": ["workspace_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_settings": { + "name": "mothership_settings", + "schema": "", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_tool_refs": { + "name": "mcp_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "custom_tool_refs": { + "name": "custom_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "skill_refs": { + "name": "skill_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mothership_settings_workspace_id_idx": { + "name": "mothership_settings_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_settings_workspace_id_workspace_id_fk": { + "name": "mothership_settings_workspace_id_workspace_id_fk", + "tableFrom": "mothership_settings", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "session_policy_settings": { + "name": "session_policy_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "security_policy_version": { + "name": "security_policy_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "whitelabel_settings": { + "name": "whitelabel_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "data_retention_settings": { + "name": "data_retention_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "org_usage_limit": { + "name": "org_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "departed_member_usage": { + "name": "departed_member_usage", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_member_usage_limit": { + "name": "organization_member_usage_limit", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "usage_limit": { + "name": "usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "set_by": { + "name": "set_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_member_usage_limit_org_user_unique": { + "name": "org_member_usage_limit_org_user_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_member_usage_limit_organization_id_idx": { + "name": "org_member_usage_limit_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_member_usage_limit_organization_id_organization_id_fk": { + "name": "organization_member_usage_limit_organization_id_organization_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_user_id_user_id_fk": { + "name": "organization_member_usage_limit_user_id_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_set_by_user_id_fk": { + "name": "organization_member_usage_limit_set_by_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["set_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.outbox_event": { + "name": "outbox_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10 + }, + "available_at": { + "name": "available_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "locked_at": { + "name": "locked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "outbox_event_status_available_idx": { + "name": "outbox_event_status_available_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "available_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_locked_at_idx": { + "name": "outbox_event_locked_at_idx", + "columns": [ + { + "expression": "locked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_type_created_idx": { + "name": "outbox_event_type_created_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.paused_executions": { + "name": "paused_executions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_snapshot": { + "name": "execution_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "pause_points": { + "name": "pause_points", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "total_pause_count": { + "name": "total_pause_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "resumed_count": { + "name": "resumed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "automatic_resume_retry_count": { + "name": "automatic_resume_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paused'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "paused_at": { + "name": "paused_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "next_resume_at": { + "name": "next_resume_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "paused_executions_workflow_id_idx": { + "name": "paused_executions_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_status_idx": { + "name": "paused_executions_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_execution_id_unique": { + "name": "paused_executions_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_next_resume_at_idx": { + "name": "paused_executions_next_resume_at_idx", + "columns": [ + { + "expression": "next_resume_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'paused' AND next_resume_at IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "paused_executions_workflow_id_workflow_id_fk": { + "name": "paused_executions_workflow_id_workflow_id_fk", + "tableFrom": "paused_executions", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pending_credential_draft": { + "name": "pending_credential_draft", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pending_draft_user_provider_ws": { + "name": "pending_draft_user_provider_ws", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pending_credential_draft_user_id_user_id_fk": { + "name": "pending_credential_draft_user_id_user_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_workspace_id_workspace_id_fk": { + "name": "pending_credential_draft_workspace_id_workspace_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_credential_id_credential_id_fk": { + "name": "pending_credential_draft_credential_id_credential_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group": { + "name": "permission_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "permission_group_created_by_idx": { + "name": "permission_group_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_name_unique": { + "name": "permission_group_organization_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_default_unique": { + "name": "permission_group_organization_default_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_organization_id_organization_id_fk": { + "name": "permission_group_organization_id_organization_id_fk", + "tableFrom": "permission_group", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_created_by_user_id_fk": { + "name": "permission_group_created_by_user_id_fk", + "tableFrom": "permission_group", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_member": { + "name": "permission_group_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "assigned_by": { + "name": "assigned_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assigned_at": { + "name": "assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_member_group_id_idx": { + "name": "permission_group_member_group_id_idx", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_group_user_unique": { + "name": "permission_group_member_group_user_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_organization_user_idx": { + "name": "permission_group_member_organization_user_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_member_permission_group_id_permission_group_id_fk": { + "name": "permission_group_member_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_organization_id_organization_id_fk": { + "name": "permission_group_member_organization_id_organization_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_user_id_user_id_fk": { + "name": "permission_group_member_user_id_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_assigned_by_user_id_fk": { + "name": "permission_group_member_assigned_by_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["assigned_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_workspace": { + "name": "permission_group_workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_workspace_workspace_id_idx": { + "name": "permission_group_workspace_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_workspace_group_workspace_unique": { + "name": "permission_group_workspace_group_workspace_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_workspace_permission_group_id_permission_group_id_fk": { + "name": "permission_group_workspace_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_workspace_id_workspace_id_fk": { + "name": "permission_group_workspace_workspace_id_workspace_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_organization_id_organization_id_fk": { + "name": "permission_group_workspace_organization_id_organization_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permissions": { + "name": "permissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission_type": { + "name": "permission_type", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permissions_user_id_idx": { + "name": "permissions_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_entity_idx": { + "name": "permissions_entity_idx", + "columns": [ + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_type_idx": { + "name": "permissions_user_entity_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_permission_idx": { + "name": "permissions_user_entity_permission_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "permission_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_idx": { + "name": "permissions_user_entity_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_unique_constraint": { + "name": "permissions_unique_constraint", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permissions_user_id_user_id_fk": { + "name": "permissions_user_id_user_id_fk", + "tableFrom": "permissions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pinned_item": { + "name": "pinned_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pinned_item_user_workspace_idx": { + "name": "pinned_item_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_resource_idx": { + "name": "pinned_item_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_user_resource_unique": { + "name": "pinned_item_user_resource_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pinned_item_user_id_user_id_fk": { + "name": "pinned_item_user_id_user_id_fk", + "tableFrom": "pinned_item", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pinned_item_workspace_id_workspace_id_fk": { + "name": "pinned_item_workspace_id_workspace_id_fk", + "tableFrom": "pinned_item", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.public_share": { + "name": "public_share", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "public_share_token_unique": { + "name": "public_share_token_unique", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_unique": { + "name": "public_share_resource_unique", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_id_idx": { + "name": "public_share_resource_id_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_workspace_id_idx": { + "name": "public_share_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "public_share_workspace_id_workspace_id_fk": { + "name": "public_share_workspace_id_workspace_id_fk", + "tableFrom": "public_share", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "public_share_created_by_user_id_fk": { + "name": "public_share_created_by_user_id_fk", + "tableFrom": "public_share", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit_bucket": { + "name": "rate_limit_bucket", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tokens": { + "name": "tokens", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.resume_queue": { + "name": "resume_queue", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "paused_execution_id": { + "name": "paused_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_execution_id": { + "name": "parent_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "new_execution_id": { + "name": "new_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context_id": { + "name": "context_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resume_input": { + "name": "resume_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "resume_queue_parent_status_idx": { + "name": "resume_queue_parent_status_idx", + "columns": [ + { + "expression": "parent_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resume_queue_new_execution_idx": { + "name": "resume_queue_new_execution_idx", + "columns": [ + { + "expression": "new_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "resume_queue_paused_execution_id_paused_executions_id_fk": { + "name": "resume_queue_paused_execution_id_paused_executions_id_fk", + "tableFrom": "resume_queue", + "tableTo": "paused_executions", + "columnsFrom": ["paused_execution_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "session_token_idx": { + "name": "session_token_idx", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_active_organization_id_organization_id_fk": { + "name": "session_active_organization_id_organization_id_fk", + "tableFrom": "session", + "tableTo": "organization", + "columnsFrom": ["active_organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.settings": { + "name": "settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "theme": { + "name": "theme", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'system'" + }, + "auto_connect": { + "name": "auto_connect", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "telemetry_enabled": { + "name": "telemetry_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "email_preferences": { + "name": "email_preferences", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "billing_usage_notifications_enabled": { + "name": "billing_usage_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "show_training_controls": { + "name": "show_training_controls", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "super_user_mode_enabled": { + "name": "super_user_mode_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "mothership_environment": { + "name": "mothership_environment", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "error_notifications_enabled": { + "name": "error_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "snap_to_grid_size": { + "name": "snap_to_grid_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "show_action_bar": { + "name": "show_action_bar", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "copilot_enabled_models": { + "name": "copilot_enabled_models", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "copilot_auto_allowed_tools": { + "name": "copilot_auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_active_workspace_id": { + "name": "last_active_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "settings_user_id_user_id_fk": { + "name": "settings_user_id_user_id_fk", + "tableFrom": "settings", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "settings_user_id_unique": { + "name": "settings_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sim_trigger_state": { + "name": "sim_trigger_state", + "schema": "", + "columns": { + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope_key": { + "name": "scope_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "last_fired_at": { + "name": "last_fired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sim_trigger_state_workflow_id_workflow_id_fk": { + "name": "sim_trigger_state_workflow_id_workflow_id_fk", + "tableFrom": "sim_trigger_state", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "sim_trigger_state_workflow_id_block_id_scope_key_pk": { + "name": "sim_trigger_state_workflow_id_block_id_scope_key_pk", + "columns": ["workflow_id", "block_id", "scope_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill": { + "name": "skill", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_workspace_name_unique": { + "name": "skill_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_workspace_id_workspace_id_fk": { + "name": "skill_workspace_id_workspace_id_fk", + "tableFrom": "skill", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_user_id_user_id_fk": { + "name": "skill_user_id_user_id_fk", + "tableFrom": "skill", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_member": { + "name": "skill_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_member_user_id_idx": { + "name": "skill_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skill_member_unique": { + "name": "skill_member_unique", + "columns": [ + { + "expression": "skill_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_member_skill_id_skill_id_fk": { + "name": "skill_member_skill_id_skill_id_fk", + "tableFrom": "skill_member", + "tableTo": "skill", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_user_id_user_id_fk": { + "name": "skill_member_user_id_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_invited_by_user_id_fk": { + "name": "skill_member_invited_by_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_domain": { + "name": "sso_domain", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "verification_token": { + "name": "verification_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sso_domain_organization_id_idx": { + "name": "sso_domain_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_domain_idx": { + "name": "sso_domain_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_org_domain_unique": { + "name": "sso_domain_org_domain_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_verified_unique": { + "name": "sso_domain_verified_unique", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "status = 'verified'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_domain_organization_id_organization_id_fk": { + "name": "sso_domain_organization_id_organization_id_fk", + "tableFrom": "sso_domain", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_domain_created_by_user_id_fk": { + "name": "sso_domain_created_by_user_id_fk", + "tableFrom": "sso_domain", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_provider": { + "name": "sso_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sso_provider_provider_id_idx": { + "name": "sso_provider_provider_id_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_domain_idx": { + "name": "sso_provider_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_user_id_idx": { + "name": "sso_provider_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_organization_id_idx": { + "name": "sso_provider_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_provider_user_id_user_id_fk": { + "name": "sso_provider_user_id_user_id_fk", + "tableFrom": "sso_provider", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_provider_organization_id_organization_id_fk": { + "name": "sso_provider_organization_id_organization_id_fk", + "tableFrom": "sso_provider", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subscription": { + "name": "subscription", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_subscription_id": { + "name": "stripe_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "period_start": { + "name": "period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "period_end": { + "name": "period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancel_at_period_end": { + "name": "cancel_at_period_end", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cancel_at": { + "name": "cancel_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "canceled_at": { + "name": "canceled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "seats": { + "name": "seats", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "trial_start": { + "name": "trial_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trial_end": { + "name": "trial_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_interval": { + "name": "billing_interval", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_schedule_id": { + "name": "stripe_schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "subscription_reference_status_idx": { + "name": "subscription_reference_status_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "check_enterprise_metadata": { + "name": "check_enterprise_metadata", + "value": "plan != 'enterprise' OR metadata IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.table_jobs": { + "name": "table_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rows_processed": { + "name": "rows_processed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_jobs_one_active_per_table": { + "name": "table_jobs_one_active_per_table", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"table_jobs\".\"status\" = 'running' AND \"table_jobs\".\"type\" <> 'export'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_watchdog_idx": { + "name": "table_jobs_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_table_started_idx": { + "name": "table_jobs_table_started_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_jobs_table_id_user_table_definitions_id_fk": { + "name": "table_jobs_table_id_user_table_definitions_id_fk", + "tableFrom": "table_jobs", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_jobs_workspace_id_workspace_id_fk": { + "name": "table_jobs_workspace_id_workspace_id_fk", + "tableFrom": "table_jobs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_row_executions": { + "name": "table_row_executions", + "schema": "", + "columns": { + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "running_block_ids": { + "name": "running_block_ids", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "block_errors": { + "name": "block_errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "enrichment_details": { + "name": "enrichment_details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "table_row_executions_table_status_idx": { + "name": "table_row_executions_table_status_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"status\" IN ('queued', 'running', 'pending')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_execution_id_idx": { + "name": "table_row_executions_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"execution_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_table_group_idx": { + "name": "table_row_executions_table_group_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_row_executions_table_id_user_table_definitions_id_fk": { + "name": "table_row_executions_table_id_user_table_definitions_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_row_executions_row_id_user_table_rows_id_fk": { + "name": "table_row_executions_row_id_user_table_rows_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_rows", + "columnsFrom": ["row_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "table_row_executions_row_id_group_id_pk": { + "name": "table_row_executions_row_id_group_id_pk", + "columns": ["row_id", "group_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_run_dispatches": { + "name": "table_run_dispatches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "cursor": { + "name": "cursor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit": { + "name": "limit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "processed_count": { + "name": "processed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_manual_run": { + "name": "is_manual_run", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "triggered_by_user_id": { + "name": "triggered_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_run_dispatches_active_idx": { + "name": "table_run_dispatches_active_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_run_dispatches_watchdog_idx": { + "name": "table_run_dispatches_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_run_dispatches_table_id_user_table_definitions_id_fk": { + "name": "table_run_dispatches_table_id_user_table_definitions_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_workspace_id_workspace_id_fk": { + "name": "table_run_dispatches_workspace_id_workspace_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_triggered_by_user_id_user_id_fk": { + "name": "table_run_dispatches_triggered_by_user_id_user_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user", + "columnsFrom": ["triggered_by_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_log": { + "name": "usage_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "usage_log_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "usage_log_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "cost": { + "name": "cost", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_entity_type": { + "name": "billing_entity_type", + "type": "billing_entity_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "billing_entity_id": { + "name": "billing_entity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_period_start": { + "name": "billing_period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_period_end": { + "name": "billing_period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "usage_log_user_created_at_idx": { + "name": "usage_log_user_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_source_idx": { + "name": "usage_log_source_idx", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_id_idx": { + "name": "usage_log_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workflow_id_idx": { + "name": "usage_log_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_event_key_unique": { + "name": "usage_log_event_key_unique", + "columns": [ + { + "expression": "event_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"usage_log\".\"event_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_entity_period_idx": { + "name": "usage_log_billing_entity_period_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_period_cost_idx": { + "name": "usage_log_billing_period_cost_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_created_at_idx": { + "name": "usage_log_workspace_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_execution_id_idx": { + "name": "usage_log_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_log_user_id_user_id_fk": { + "name": "usage_log_user_id_user_id_fk", + "tableFrom": "usage_log", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "usage_log_workspace_id_workspace_id_fk": { + "name": "usage_log_workspace_id_workspace_id_fk", + "tableFrom": "usage_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "usage_log_workflow_id_workflow_id_fk": { + "name": "usage_log_workflow_id_workflow_id_fk", + "tableFrom": "usage_log", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_log_billing_scope_all_or_none": { + "name": "usage_log_billing_scope_all_or_none", + "value": "(\n (\"usage_log\".\"billing_entity_type\" IS NULL AND \"usage_log\".\"billing_entity_id\" IS NULL AND \"usage_log\".\"billing_period_start\" IS NULL AND \"usage_log\".\"billing_period_end\" IS NULL)\n OR\n (\"usage_log\".\"billing_entity_type\" IS NOT NULL AND \"usage_log\".\"billing_entity_id\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" IS NOT NULL AND \"usage_log\".\"billing_period_end\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" < \"usage_log\".\"billing_period_end\")\n )" + } + }, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": ["normalized_email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_stats": { + "name": "user_stats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "total_manual_executions": { + "name": "total_manual_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_api_calls": { + "name": "total_api_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_webhook_triggers": { + "name": "total_webhook_triggers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_scheduled_executions": { + "name": "total_scheduled_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_chat_executions": { + "name": "total_chat_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_executions": { + "name": "total_mcp_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_tokens_used": { + "name": "total_tokens_used", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_cost": { + "name": "total_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_usage_limit": { + "name": "current_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'5'" + }, + "usage_limit_updated_at": { + "name": "usage_limit_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "current_period_cost": { + "name": "current_period_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_period_cost": { + "name": "last_period_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "billed_overage_this_period": { + "name": "billed_overage_this_period", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "pro_period_cost_snapshot": { + "name": "pro_period_cost_snapshot", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "pro_period_cost_snapshot_at": { + "name": "pro_period_cost_snapshot_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "total_copilot_cost": { + "name": "total_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_period_copilot_cost": { + "name": "current_period_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_period_copilot_cost": { + "name": "last_period_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "total_copilot_tokens": { + "name": "total_copilot_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_copilot_calls": { + "name": "total_copilot_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_copilot_calls": { + "name": "total_mcp_copilot_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_copilot_cost": { + "name": "total_mcp_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_period_mcp_copilot_cost": { + "name": "current_period_mcp_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_active": { + "name": "last_active", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "billing_blocked": { + "name": "billing_blocked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "billing_blocked_reason": { + "name": "billing_blocked_reason", + "type": "billing_blocked_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + } + }, + "indexes": {}, + "foreignKeys": { + "user_stats_user_id_user_id_fk": { + "name": "user_stats_user_id_user_id_fk", + "tableFrom": "user_stats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_stats_user_id_unique": { + "name": "user_stats_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_definitions": { + "name": "user_table_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "max_rows": { + "name": "max_rows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "row_count": { + "name": "row_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "rows_version": { + "name": "rows_version", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "schema_locked": { + "name": "schema_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "insert_locked": { + "name": "insert_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "update_locked": { + "name": "update_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "delete_locked": { + "name": "delete_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "user_table_def_workspace_id_idx": { + "name": "user_table_def_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_folder_id_idx": { + "name": "user_table_def_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_name_unique": { + "name": "user_table_def_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"user_table_definitions\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_archived_at_idx": { + "name": "user_table_def_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_archived_partial_idx": { + "name": "user_table_def_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"user_table_definitions\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_definitions_workspace_id_workspace_id_fk": { + "name": "user_table_definitions_workspace_id_workspace_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_definitions_folder_id_folder_id_fk": { + "name": "user_table_definitions_folder_id_folder_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "user_table_definitions_created_by_user_id_fk": { + "name": "user_table_definitions_created_by_user_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_rows": { + "name": "user_table_rows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "user_table_rows_tenant_data_gin_idx": { + "name": "user_table_rows_tenant_data_gin_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"data\" jsonb_path_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "user_table_rows_workspace_table_idx": { + "name": "user_table_rows_workspace_table_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_position_idx": { + "name": "user_table_rows_table_position_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "position", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_order_key_idx": { + "name": "user_table_rows_table_order_key_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "order_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_id_id_idx": { + "name": "user_table_rows_table_id_id_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_rows_table_id_user_table_definitions_id_fk": { + "name": "user_table_rows_table_id_user_table_definitions_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_workspace_id_workspace_id_fk": { + "name": "user_table_rows_workspace_id_workspace_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_created_by_user_id_fk": { + "name": "user_table_rows_created_by_user_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "verification_expires_at_idx": { + "name": "verification_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.waitlist": { + "name": "waitlist", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "waitlist_email_unique": { + "name": "waitlist_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook": { + "name": "webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_status": { + "name": "registration_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_generation": { + "name": "registration_generation", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "config_fingerprint": { + "name": "config_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prepared_at": { + "name": "prepared_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "routing_key": { + "name": "routing_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_config": { + "name": "provider_config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "path_deployment_unique": { + "name": "path_deployment_unique", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_workflow_deployment_idx": { + "name": "webhook_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_routing_key_active_idx": { + "name": "webhook_routing_key_active_idx", + "columns": [ + { + "expression": "routing_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NULL AND \"webhook\".\"routing_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_archived_at_partial_idx": { + "name": "webhook_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468": { + "name": "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_tiktok_credential_id_idx": { + "name": "webhook_tiktok_credential_id_idx", + "columns": [ + { + "expression": "((\"provider_config\")::jsonb ->> 'credentialId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"provider\" = 'tiktok' AND \"webhook\".\"is_active\" = true AND \"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_workflow_id_block_id_updated_at_desc": { + "name": "idx_webhook_on_workflow_id_block_id_updated_at_desc", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_active_registration_unique": { + "name": "webhook_active_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'active' AND \"webhook\".\"block_id\" IS NOT NULL AND \"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_candidate_registration_unique": { + "name": "webhook_candidate_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'candidate' AND \"webhook\".\"block_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_registration_status_generation_idx": { + "name": "webhook_registration_status_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_workflow_id_workflow_id_fk": { + "name": "webhook_workflow_id_workflow_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "webhook_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_registration_status_check": { + "name": "webhook_registration_status_check", + "value": "\"webhook\".\"registration_status\" IS NULL OR \"webhook\".\"registration_status\" IN ('active', 'candidate', 'retired', 'orphaned')" + }, + "webhook_registration_generation_check": { + "name": "webhook_registration_generation_check", + "value": "\"webhook\".\"registration_generation\" IS NULL OR \"webhook\".\"registration_generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.webhook_path_claim": { + "name": "webhook_path_claim", + "schema": "", + "columns": { + "path": { + "name": "path", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "webhook_path_claim_workflow_idx": { + "name": "webhook_path_claim_workflow_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_path_claim_workflow_id_workflow_id_fk": { + "name": "webhook_path_claim_workflow_id_workflow_id_fk", + "tableFrom": "webhook_path_claim", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_path_claim_generation_check": { + "name": "webhook_path_claim_generation_check", + "value": "\"webhook_path_claim\".\"generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow": { + "name": "workflow", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_synced": { + "name": "last_synced", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "is_deployed": { + "name": "is_deployed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deployed_at": { + "name": "deployed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "is_public_api": { + "name": "is_public_api", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "fork_sync_excluded": { + "name": "fork_sync_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_user_id_idx": { + "name": "workflow_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_id_idx": { + "name": "workflow_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_user_workspace_idx": { + "name": "workflow_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_folder_name_active_unique": { + "name": "workflow_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_folder_sort_idx": { + "name": "workflow_folder_sort_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_archived_at_idx": { + "name": "workflow_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_archived_partial_idx": { + "name": "workflow_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_user_id_user_id_fk": { + "name": "workflow_user_id_user_id_fk", + "tableFrom": "workflow", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_workspace_id_workspace_id_fk": { + "name": "workflow_workspace_id_workspace_id_fk", + "tableFrom": "workflow", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_blocks": { + "name": "workflow_blocks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position_x": { + "name": "position_x", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "position_y": { + "name": "position_y", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "horizontal_handles": { + "name": "horizontal_handles", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_wide": { + "name": "is_wide", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "advanced_mode": { + "name": "advanced_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "trigger_mode": { + "name": "trigger_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "height": { + "name": "height", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "sub_blocks": { + "name": "sub_blocks", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "outputs": { + "name": "outputs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_blocks_workflow_id_idx": { + "name": "workflow_blocks_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_blocks_type_idx": { + "name": "workflow_blocks_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_blocks_workflow_id_workflow_id_fk": { + "name": "workflow_blocks_workflow_id_workflow_id_fk", + "tableFrom": "workflow_blocks", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_checkpoints": { + "name": "workflow_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_state": { + "name": "workflow_state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_checkpoints_user_id_idx": { + "name": "workflow_checkpoints_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_id_idx": { + "name": "workflow_checkpoints_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_id_idx": { + "name": "workflow_checkpoints_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_message_id_idx": { + "name": "workflow_checkpoints_message_id_idx", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_user_workflow_idx": { + "name": "workflow_checkpoints_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_chat_idx": { + "name": "workflow_checkpoints_workflow_chat_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_created_at_idx": { + "name": "workflow_checkpoints_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_created_at_idx": { + "name": "workflow_checkpoints_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_checkpoints_user_id_user_id_fk": { + "name": "workflow_checkpoints_user_id_user_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_workflow_id_workflow_id_fk": { + "name": "workflow_checkpoints_workflow_id_workflow_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_chat_id_copilot_chats_id_fk": { + "name": "workflow_checkpoints_chat_id_copilot_chats_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_deployment_operation": { + "name": "workflow_deployment_operation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "previous_active_version_id": { + "name": "previous_active_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "protocol_version": { + "name": "protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'preparing'" + }, + "component_readiness": { + "name": "component_readiness", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_hash": { + "name": "request_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_deployment_operation_workflow_generation_unique": { + "name": "workflow_deployment_operation_workflow_generation_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_idempotency_unique": { + "name": "workflow_deployment_operation_workflow_idempotency_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"idempotency_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_in_flight_unique": { + "name": "workflow_deployment_operation_workflow_in_flight_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_status_idx": { + "name": "workflow_deployment_operation_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_deployment_version_idx": { + "name": "workflow_deployment_operation_deployment_version_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_version_generation_idx": { + "name": "workflow_deployment_operation_workflow_version_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_operation_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_operation_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["previous_active_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_deployment_operation_action_check": { + "name": "workflow_deployment_operation_action_check", + "value": "\"workflow_deployment_operation\".\"action\" IN ('deploy', 'activate')" + }, + "workflow_deployment_operation_status_check": { + "name": "workflow_deployment_operation_status_check", + "value": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating', 'active', 'failed', 'superseded')" + }, + "workflow_deployment_operation_generation_check": { + "name": "workflow_deployment_operation_generation_check", + "value": "\"workflow_deployment_operation\".\"generation\" > 0" + }, + "workflow_deployment_operation_protocol_version_check": { + "name": "workflow_deployment_operation_protocol_version_check", + "value": "\"workflow_deployment_operation\".\"protocol_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow_deployment_version": { + "name": "workflow_deployment_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_deployment_version_workflow_version_unique": { + "name": "workflow_deployment_version_workflow_version_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_workflow_active_idx": { + "name": "workflow_deployment_version_workflow_active_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_created_at_idx": { + "name": "workflow_deployment_version_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_version_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_version_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_version", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_edges": { + "name": "workflow_edges", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_block_id": { + "name": "source_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_handle": { + "name": "source_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_handle": { + "name": "target_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_edges_workflow_id_idx": { + "name": "workflow_edges_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_source_idx": { + "name": "workflow_edges_workflow_source_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_target_idx": { + "name": "workflow_edges_workflow_target_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_edges_workflow_id_workflow_id_fk": { + "name": "workflow_edges_workflow_id_workflow_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_source_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_source_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["source_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_target_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_target_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["target_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_logs": { + "name": "workflow_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_snapshot_id": { + "name": "state_snapshot_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost": { + "name": "cost", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "cost_total": { + "name": "cost_total", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "models_used": { + "name": "models_used", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "files": { + "name": "files", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_execution_logs_workflow_id_idx": { + "name": "workflow_execution_logs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_state_snapshot_id_idx": { + "name": "workflow_execution_logs_state_snapshot_id_idx", + "columns": [ + { + "expression": "state_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_deployment_version_id_idx": { + "name": "workflow_execution_logs_deployment_version_id_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_trigger_idx": { + "name": "workflow_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_level_idx": { + "name": "workflow_execution_logs_level_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_started_at_idx": { + "name": "workflow_execution_logs_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_execution_id_unique": { + "name": "workflow_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workflow_started_at_idx": { + "name": "workflow_execution_logs_workflow_started_at_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_idx": { + "name": "workflow_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_id_desc_idx": { + "name": "workflow_execution_logs_workspace_started_at_id_desc_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"started_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "\"id\" DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_cost_total_idx": { + "name": "workflow_execution_logs_workspace_cost_total_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_total", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_models_used_idx": { + "name": "workflow_execution_logs_models_used_idx", + "columns": [ + { + "expression": "models_used", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "workflow_execution_logs_workspace_ended_at_id_idx": { + "name": "workflow_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_running_started_at_idx": { + "name": "workflow_execution_logs_running_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_completed_ended_at_idx": { + "name": "workflow_execution_logs_completed_ended_at_idx", + "columns": [ + { + "expression": "ended_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_execution_logs\".\"status\" = 'completed' AND \"workflow_execution_logs\".\"level\" = 'info' AND \"workflow_execution_logs\".\"ended_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_logs_workflow_id_workflow_id_fk": { + "name": "workflow_execution_logs_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_execution_logs_workspace_id_workspace_id_fk": { + "name": "workflow_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk": { + "name": "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_execution_snapshots", + "columnsFrom": ["state_snapshot_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_snapshots": { + "name": "workflow_execution_snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_hash": { + "name": "state_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_data": { + "name": "state_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_snapshots_workflow_id_idx": { + "name": "workflow_snapshots_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_hash_idx": { + "name": "workflow_snapshots_hash_idx", + "columns": [ + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_workflow_hash_idx": { + "name": "workflow_snapshots_workflow_hash_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_created_at_idx": { + "name": "workflow_snapshots_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_snapshots_workflow_id_workflow_id_fk": { + "name": "workflow_execution_snapshots_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_snapshots", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_folder": { + "name": "workflow_folder", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'#6B7280'" + }, + "is_expanded": { + "name": "is_expanded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_folder_user_idx": { + "name": "workflow_folder_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_folder_workspace_parent_idx": { + "name": "workflow_folder_workspace_parent_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_folder_parent_sort_idx": { + "name": "workflow_folder_parent_sort_idx", + "columns": [ + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_folder_archived_at_idx": { + "name": "workflow_folder_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_folder_workspace_archived_partial_idx": { + "name": "workflow_folder_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_folder\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_folder_user_id_user_id_fk": { + "name": "workflow_folder_user_id_user_id_fk", + "tableFrom": "workflow_folder", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_folder_workspace_id_workspace_id_fk": { + "name": "workflow_folder_workspace_id_workspace_id_fk", + "tableFrom": "workflow_folder", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_server": { + "name": "workflow_mcp_server", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_server_workspace_id_idx": { + "name": "workflow_mcp_server_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_created_by_idx": { + "name": "workflow_mcp_server_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_deleted_at_idx": { + "name": "workflow_mcp_server_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_workspace_deleted_partial_idx": { + "name": "workflow_mcp_server_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_server\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_server_workspace_id_workspace_id_fk": { + "name": "workflow_mcp_server_workspace_id_workspace_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_server_created_by_user_id_fk": { + "name": "workflow_mcp_server_created_by_user_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_tool": { + "name": "workflow_mcp_tool", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_description": { + "name": "tool_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parameter_schema": { + "name": "parameter_schema", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "parameter_description_overrides": { + "name": "parameter_description_overrides", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'::json" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_tool_server_id_idx": { + "name": "workflow_mcp_tool_server_id_idx", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_workflow_id_idx": { + "name": "workflow_mcp_tool_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_server_workflow_unique": { + "name": "workflow_mcp_tool_server_workflow_unique", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_archived_at_partial_idx": { + "name": "workflow_mcp_tool_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk": { + "name": "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow_mcp_server", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_tool_workflow_id_workflow_id_fk": { + "name": "workflow_mcp_tool_workflow_id_workflow_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_schedule": { + "name": "workflow_schedule", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_operation_id": { + "name": "deployment_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cron_expression": { + "name": "cron_expression", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_ran_at": { + "name": "last_ran_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_queued_at": { + "name": "last_queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trigger_type": { + "name": "trigger_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "infra_retry_count": { + "name": "infra_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'workflow'" + }, + "job_title": { + "name": "job_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lifecycle": { + "name": "lifecycle", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'persistent'" + }, + "success_condition": { + "name": "success_condition", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "max_runs": { + "name": "max_runs", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "source_chat_id": { + "name": "source_chat_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_task_name": { + "name": "source_task_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_user_id": { + "name": "source_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "job_history": { + "name": "job_history", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "contexts": { + "name": "contexts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "excluded_dates": { + "name": "excluded_dates", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ends_at": { + "name": "ends_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_schedule_workflow_block_deployment_unique": { + "name": "workflow_schedule_workflow_block_deployment_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_workflow_deployment_idx": { + "name": "workflow_schedule_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_archived_at_partial_idx": { + "name": "workflow_schedule_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6": { + "name": "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6", + "columns": [ + { + "expression": "source_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_workflow_idx": { + "name": "workflow_schedule_due_workflow_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND (\"workflow_schedule\".\"source_type\" = 'workflow' OR \"workflow_schedule\".\"source_type\" IS NULL)", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_job_idx": { + "name": "workflow_schedule_due_job_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND \"workflow_schedule\".\"source_type\" = 'job'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_schedule_workflow_id_workflow_id_fk": { + "name": "workflow_schedule_workflow_id_workflow_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk": { + "name": "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_operation", + "columnsFrom": ["deployment_operation_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_schedule_source_user_id_user_id_fk": { + "name": "workflow_schedule_source_user_id_user_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "user", + "columnsFrom": ["source_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_source_workspace_id_workspace_id_fk": { + "name": "workflow_schedule_source_workspace_id_workspace_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workspace", + "columnsFrom": ["source_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_subflows": { + "name": "workflow_subflows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_subflows_workflow_id_idx": { + "name": "workflow_subflows_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_subflows_workflow_type_idx": { + "name": "workflow_subflows_workflow_type_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_subflows_workflow_id_workflow_id_fk": { + "name": "workflow_subflows_workflow_id_workflow_id_fk", + "tableFrom": "workflow_subflows", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace": { + "name": "workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'#33C482'" + }, + "logo_url": { + "name": "logo_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_mode": { + "name": "workspace_mode", + "type": "workspace_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'grandfathered_shared'" + }, + "billed_account_user_id": { + "name": "billed_account_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "allow_personal_api_keys": { + "name": "allow_personal_api_keys", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "inbox_enabled": { + "name": "inbox_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "inbox_address": { + "name": "inbox_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inbox_provider_id": { + "name": "inbox_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "organization_assigned_at": { + "name": "organization_assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "forked_from_workspace_id": { + "name": "forked_from_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_owner_id_idx": { + "name": "workspace_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_organization_id_idx": { + "name": "workspace_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_mode_idx": { + "name": "workspace_mode_idx", + "columns": [ + { + "expression": "workspace_mode", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_forked_from_workspace_id_idx": { + "name": "workspace_forked_from_workspace_id_idx", + "columns": [ + { + "expression": "forked_from_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_owner_id_user_id_fk": { + "name": "workspace_owner_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_organization_id_organization_id_fk": { + "name": "workspace_organization_id_organization_id_fk", + "tableFrom": "workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workspace_billed_account_user_id_user_id_fk": { + "name": "workspace_billed_account_user_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["billed_account_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workspace_forked_from_workspace_id_workspace_id_fk": { + "name": "workspace_forked_from_workspace_id_workspace_id_fk", + "tableFrom": "workspace", + "tableTo": "workspace", + "columnsFrom": ["forked_from_workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_storage_used_bytes_non_negative": { + "name": "workspace_storage_used_bytes_non_negative", + "value": "\"workspace\".\"storage_used_bytes\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workspace_byok_keys": { + "name": "workspace_byok_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_byok_workspace_provider_idx": { + "name": "workspace_byok_workspace_provider_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_byok_keys_workspace_id_workspace_id_fk": { + "name": "workspace_byok_keys_workspace_id_workspace_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_byok_keys_created_by_user_id_fk": { + "name": "workspace_byok_keys_created_by_user_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_environment": { + "name": "workspace_environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_environment_workspace_unique": { + "name": "workspace_environment_workspace_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_environment_workspace_id_workspace_id_fk": { + "name": "workspace_environment_workspace_id_workspace_id_fk", + "tableFrom": "workspace_environment", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file": { + "name": "workspace_file", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_workspace_id_idx": { + "name": "workspace_file_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_key_idx": { + "name": "workspace_file_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_deleted_at_idx": { + "name": "workspace_file_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_workspace_deleted_partial_idx": { + "name": "workspace_file_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_workspace_id_workspace_id_fk": { + "name": "workspace_file_workspace_id_workspace_id_fk", + "tableFrom": "workspace_file", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_uploaded_by_user_id_fk": { + "name": "workspace_file_uploaded_by_user_id_fk", + "tableFrom": "workspace_file", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_file_key_unique": { + "name": "workspace_file_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_collab_state": { + "name": "workspace_file_collab_state", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "doc_state": { + "name": "doc_state", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "source_hash": { + "name": "source_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_file_collab_state_file_id_workspace_files_id_fk": { + "name": "workspace_file_collab_state_file_id_workspace_files_id_fk", + "tableFrom": "workspace_file_collab_state", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_folders": { + "name": "workspace_file_folders", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_folders_workspace_parent_idx": { + "name": "workspace_file_folders_workspace_parent_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_folders_parent_sort_idx": { + "name": "workspace_file_folders_parent_sort_idx", + "columns": [ + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_folders_deleted_at_idx": { + "name": "workspace_file_folders_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_folders_workspace_deleted_partial_idx": { + "name": "workspace_file_folders_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file_folders\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_folders_workspace_parent_name_active_unique": { + "name": "workspace_file_folders_workspace_parent_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"parent_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_file_folders\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_folders_user_id_user_id_fk": { + "name": "workspace_file_folders_user_id_user_id_fk", + "tableFrom": "workspace_file_folders", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_folders_workspace_id_workspace_id_fk": { + "name": "workspace_file_folders_workspace_id_workspace_id_fk", + "tableFrom": "workspace_file_folders", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_folders_parent_id_workspace_file_folders_id_fk": { + "name": "workspace_file_folders_parent_id_workspace_file_folders_id_fk", + "tableFrom": "workspace_file_folders", + "tableTo": "workspace_file_folders", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_files": { + "name": "workspace_files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context": { + "name": "context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_files_key_active_unique": { + "name": "workspace_files_key_active_unique", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_folder_name_active_unique": { + "name": "workspace_files_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "original_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL AND \"workspace_files\".\"context\" = 'workspace' AND \"workspace_files\".\"workspace_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_chat_display_name_unique": { + "name": "workspace_files_chat_display_name_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"context\" = 'mothership' AND \"workspace_files\".\"chat_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_key_idx": { + "name": "workspace_files_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_user_id_idx": { + "name": "workspace_files_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_id_idx": { + "name": "workspace_files_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_folder_id_idx": { + "name": "workspace_files_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_context_idx": { + "name": "workspace_files_context_idx", + "columns": [ + { + "expression": "context", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_chat_id_idx": { + "name": "workspace_files_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_deleted_at_idx": { + "name": "workspace_files_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_deleted_partial_idx": { + "name": "workspace_files_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_files\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_files_user_id_user_id_fk": { + "name": "workspace_files_user_id_user_id_fk", + "tableFrom": "workspace_files", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_workspace_id_workspace_id_fk": { + "name": "workspace_files_workspace_id_workspace_id_fk", + "tableFrom": "workspace_files", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_chat_id_copilot_chats_id_fk": { + "name": "workspace_files_chat_id_copilot_chats_id_fk", + "tableFrom": "workspace_files", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_block_map": { + "name": "workspace_fork_block_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_workflow_id": { + "name": "parent_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_block_id": { + "name": "parent_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_workflow_id": { + "name": "child_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_block_id": { + "name": "child_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_block_map_child_ws_parent_unique": { + "name": "workspace_fork_block_map_child_ws_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_unique": { + "name": "workspace_fork_block_map_child_ws_child_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_parent_wf_idx": { + "name": "workspace_fork_block_map_child_ws_parent_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_wf_idx": { + "name": "workspace_fork_block_map_child_ws_child_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_block_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_block_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_block_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_dependent_value": { + "name": "workspace_fork_dependent_value", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workflow_id": { + "name": "target_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_block_key": { + "name": "sub_block_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_dependent_value_child_ws_wf_idx": { + "name": "workspace_fork_dependent_value_child_ws_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_dependent_value_field_unique": { + "name": "workspace_fork_dependent_value_field_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sub_block_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_dependent_value", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_promote_run": { + "name": "workspace_fork_promote_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workspace_id": { + "name": "target_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "workspace_fork_promote_direction", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_promote_run_child_ws_target_unique": { + "name": "workspace_fork_promote_run_child_ws_target_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_promote_run_target_ws_idx": { + "name": "workspace_fork_promote_run_target_ws_idx", + "columns": [ + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_promote_run_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_promote_run_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_promote_run_created_by_user_id_fk": { + "name": "workspace_fork_promote_run_created_by_user_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_resource_map": { + "name": "workspace_fork_resource_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "workspace_fork_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_resource_id": { + "name": "parent_resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_resource_id": { + "name": "child_resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_resource_map_child_ws_idx": { + "name": "workspace_fork_resource_map_child_ws_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_ws_type_idx": { + "name": "workspace_fork_resource_map_child_ws_type_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_type_parent_unique": { + "name": "workspace_fork_resource_map_child_type_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_resource_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_resource_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_resource_map_created_by_user_id_fk": { + "name": "workspace_fork_resource_map_created_by_user_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.academy_cert_status": { + "name": "academy_cert_status", + "schema": "public", + "values": ["active", "revoked", "expired"] + }, + "public.background_work_kind": { + "name": "background_work_kind", + "schema": "public", + "values": ["deployment_side_effects", "fork_content_copy", "fork_sync", "fork_rollback"] + }, + "public.background_work_status_value": { + "name": "background_work_status_value", + "schema": "public", + "values": ["pending", "processing", "completed", "completed_with_warnings", "failed"] + }, + "public.billing_blocked_reason": { + "name": "billing_blocked_reason", + "schema": "public", + "values": ["payment_failed", "dispute"] + }, + "public.billing_entity_type": { + "name": "billing_entity_type", + "schema": "public", + "values": ["user", "organization"] + }, + "public.chat_type": { + "name": "chat_type", + "schema": "public", + "values": ["mothership", "copilot"] + }, + "public.copilot_async_tool_status": { + "name": "copilot_async_tool_status", + "schema": "public", + "values": ["pending", "running", "completed", "failed", "cancelled", "delivered"] + }, + "public.copilot_run_status": { + "name": "copilot_run_status", + "schema": "public", + "values": ["active", "paused_waiting_for_tool", "resuming", "complete", "error", "cancelled"] + }, + "public.copilot_tool_permission_decision": { + "name": "copilot_tool_permission_decision", + "schema": "public", + "values": ["allow", "allow_chat", "always_allow", "skip"] + }, + "public.credential_member_role": { + "name": "credential_member_role", + "schema": "public", + "values": ["admin", "member"] + }, + "public.credential_member_status": { + "name": "credential_member_status", + "schema": "public", + "values": ["active", "pending", "revoked"] + }, + "public.credential_type": { + "name": "credential_type", + "schema": "public", + "values": ["oauth", "env_workspace", "env_personal", "service_account"] + }, + "public.data_drain_cadence": { + "name": "data_drain_cadence", + "schema": "public", + "values": ["hourly", "daily"] + }, + "public.data_drain_destination": { + "name": "data_drain_destination", + "schema": "public", + "values": ["s3", "gcs", "azure_blob", "datadog", "bigquery", "snowflake", "webhook"] + }, + "public.data_drain_run_status": { + "name": "data_drain_run_status", + "schema": "public", + "values": ["running", "success", "failed"] + }, + "public.data_drain_run_trigger": { + "name": "data_drain_run_trigger", + "schema": "public", + "values": ["cron", "manual"] + }, + "public.data_drain_source": { + "name": "data_drain_source", + "schema": "public", + "values": ["workflow_logs", "job_logs", "audit_logs", "copilot_chats", "copilot_runs"] + }, + "public.execution_large_value_reference_source": { + "name": "execution_large_value_reference_source", + "schema": "public", + "values": ["execution_log", "paused_snapshot"] + }, + "public.folder_resource_type": { + "name": "folder_resource_type", + "schema": "public", + "values": ["workflow", "file", "knowledge_base", "table"] + }, + "public.invitation_kind": { + "name": "invitation_kind", + "schema": "public", + "values": ["organization", "workspace"] + }, + "public.invitation_membership_intent": { + "name": "invitation_membership_intent", + "schema": "public", + "values": ["internal", "external"] + }, + "public.invitation_status": { + "name": "invitation_status", + "schema": "public", + "values": ["pending", "accepted", "rejected", "cancelled", "expired"] + }, + "public.permission_type": { + "name": "permission_type", + "schema": "public", + "values": ["admin", "write", "read"] + }, + "public.usage_log_category": { + "name": "usage_log_category", + "schema": "public", + "values": ["model", "fixed", "tool"] + }, + "public.usage_log_source": { + "name": "usage_log_source", + "schema": "public", + "values": [ + "workflow", + "wand", + "copilot", + "workspace-chat", + "mcp_copilot", + "mothership_block", + "knowledge-base", + "voice-input", + "enrichment" + ] + }, + "public.workspace_fork_promote_direction": { + "name": "workspace_fork_promote_direction", + "schema": "public", + "values": ["push", "pull"] + }, + "public.workspace_fork_resource_type": { + "name": "workspace_fork_resource_type", + "schema": "public", + "values": [ + "workflow", + "oauth_credential", + "service_account_credential", + "env_var", + "table", + "knowledge_base", + "knowledge_document", + "file", + "mcp_server", + "workflow_mcp_server", + "custom_tool", + "skill" + ] + }, + "public.workspace_mode": { + "name": "workspace_mode", + "schema": "public", + "values": ["personal", "organization", "grandfathered_shared"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index eea606ba98c..a36b09881a6 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -1919,6 +1919,13 @@ "when": 1785281897202, "tag": "0274_file_folder_cutover_reconcile", "breakpoints": true + }, + { + "idx": 275, + "version": "7", + "when": 1785353031084, + "tag": "0275_cold_gorilla_man", + "breakpoints": true } ] } diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 11672d356aa..1184bcd9209 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -31,6 +31,16 @@ export const tsvector = customType<{ }, }) +/** Raw binary column. Postgres `bytea` ↔ Node `Buffer` (the pg driver handles the encoding). */ +export const bytea = customType<{ + data: Buffer + driverData: Buffer +}>({ + dataType() { + return 'bytea' + }, +}) + export const user = pgTable('user', { id: text('id').primaryKey(), name: text('name').notNull(), @@ -2040,6 +2050,26 @@ export const workspaceFiles = pgTable( }) ) +/** + * Cached collaborative-document state for a workspace markdown file: the last-persisted Yjs binary and + * a hash of the markdown it was derived from. On a cold room open the seed loads this binary directly + * (the Hocuspocus load-document pattern) rather than re-converting markdown → Yjs — which avoids the + * "recreate the CRDT from a non-binary format" anti-pattern (fresh client ids / content duplication on + * reconnect) and the server-side headless-editor conversion. The row is STALE, and the seed re-converts + * from markdown, when `sourceHash` no longer matches the file's current markdown (edited externally by a + * copilot write or a direct save). One row per file; dropped by FK cascade when the file is deleted. + */ +export const workspaceFileCollabState = pgTable('workspace_file_collab_state', { + fileId: text('file_id') + .primaryKey() + .references(() => workspaceFiles.id, { onDelete: 'cascade' }), + /** `Y.encodeStateAsUpdate` of the collaborative doc at last persist — apply with `Y.applyUpdate`. */ + docState: bytea('doc_state').notNull(), + /** sha256 (hex) of the markdown `docState` was derived from — the freshness tag for cold-start. */ + sourceHash: text('source_hash').notNull(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), +}) + /** * Public share links for workspace resources. Polymorphic on `resourceType` so a * single mechanism serves files now and folders later. One row per resource From fd3485317a85c5e9696ceea31ba88986a840ae86 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 29 Jul 2026 17:25:32 -0700 Subject: [PATCH 51/53] fix(collab-doc): stream every external file write into open editors, not just edit_content (#6070) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(collab-doc): stream every external file write into open editors, not just edit_content A copilot/mothership edit to an open markdown file did not appear live in another user's editor: the live-doc merge bridge (mergeEditIntoLiveFileDoc) was wired into the edit_content tool ONLY. Every other server-side write — the file tool (/api/tools/file/manage), function_execute (/api/function/execute via writeWorkspaceFileByPath), create_file overwrite, and the PUT /content route — went straight to updateWorkspaceFileContent and skipped the merge, so the durable file changed but the open editor never updated. Confirmed from live logs (the mothership 'Prepend sentence' ran read + file + function_execute — zero apply-edit calls) and Redis (the prepended text was absent from the doc stream). Centralize the merge at the one chokepoint every external writer shares: - updateWorkspaceFileContent gains an opt-out "syncLiveDoc" (default on) and, after the durable write, merges markdown writes into any open collaborative doc (best-effort; no-op when nobody has it open). Any current OR future writer is covered automatically. - persist.ts opts out (syncLiveDoc:false) — it IS the doc→markdown projection, so merging it back would be a persist→merge→persist self-loop. - create_file opts its empty shell out (real content arrives via a later write) so an open editor never flickers to empty on overwrite; threaded through writeWorkspaceFileByPath. - edit_content drops its now-redundant explicit merge call (the chokepoint handles it). - binary writers (image/video/audio/ffmpeg/download) are naturally excluded — the merge is gated to markdown, the only format the collaborative editor renders. Also bump the api-validation route baseline 994→996 to match the true route count already on this branch (pre-existing ratchet drift from an earlier merge; NOT added by this PR). * fix(collab-doc): defer setEditable out of the render phase (flushSync warning) The collab editability-reapply effect called editor.setEditable synchronously. In collab mode isEditable flips from readiness (synced + seeded), which is driven by a Yjs config.observe firing synchronously inside Y.applyUpdate — so the effect can run while React is mid-render. TipTap's React binding commits setEditable's transaction with flushSync, which throws "flushSync was called from inside a lifecycle method. React cannot flush when React is already rendering." Defer the setEditable to a microtask (runs right after the current commit, before paint), guarding against a destroyed editor or a stale value before it fires. Only the collab path (this effect) hit the warning; the streaming/settle effect's setEditable calls run on the non-collab path where isEditable isn't driven by a mid-render Yjs observer. --- .../rich-markdown-editor.tsx | 16 ++++- apps/sim/lib/collab-doc/persist.ts | 7 +- .../copilot/tools/server/files/create-file.ts | 3 + .../tools/server/files/edit-content.ts | 15 +---- apps/sim/lib/copilot/vfs/resource-writer.ts | 9 ++- .../workspace/workspace-file-manager.ts | 28 +++++++- .../workspace-file-storage-accounting.test.ts | 64 +++++++++++++++++++ scripts/check-api-validation-contracts.ts | 4 +- 8 files changed, 127 insertions(+), 19 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 0f3ac812476..feefae1e255 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -745,7 +745,21 @@ export function LoadedRichMarkdownEditor({ */ useEffect(() => { if (!editor || !collaborationEnabled) return - if (editor.isEditable !== isEditable) editor.setEditable(isEditable) + if (editor.isEditable === isEditable) return + // Defer out of the render/commit phase. `isEditable` flips from collab readiness (synced + seeded), + // which is driven by a Yjs `config.observe` firing synchronously inside `Y.applyUpdate` — so this + // effect can run while React is mid-render. `setEditable` dispatches a TipTap transaction that the + // React binding commits with `flushSync`, which throws ("cannot flush while rendering") in that + // window. A microtask runs right after the current commit, before paint; re-check liveness/value + // since either can change before it fires. + let cancelled = false + queueMicrotask(() => { + if (cancelled || editor.isDestroyed) return + if (editor.isEditable !== isEditable) editor.setEditable(isEditable) + }) + return () => { + cancelled = true + } }, [editor, collaborationEnabled, isEditable]) /** diff --git a/apps/sim/lib/collab-doc/persist.ts b/apps/sim/lib/collab-doc/persist.ts index 2c69d875d46..c83c0c6ee60 100644 --- a/apps/sim/lib/collab-doc/persist.ts +++ b/apps/sim/lib/collab-doc/persist.ts @@ -39,7 +39,12 @@ export async function persistFileDoc( ydoc.destroy() } - await updateWorkspaceFileContent(workspaceId, fileId, userId, markdownBuffer) + // `syncLiveDoc: false`: this write IS the projection of the live doc back to markdown, so merging it + // back into that same doc would be a persist → merge → persist self-loop. The doc already holds this + // exact content; peers are already converged through the relay. + await updateWorkspaceFileContent(workspaceId, fileId, userId, markdownBuffer, undefined, { + syncLiveDoc: false, + }) // Cache the Yjs binary (tagged with the exact markdown just written) so a later cold room open loads // it directly instead of re-converting markdown → Yjs. Best-effort: the markdown IS the durable file, diff --git a/apps/sim/lib/copilot/tools/server/files/create-file.ts b/apps/sim/lib/copilot/tools/server/files/create-file.ts index 1d5e3629546..cd950eb3406 100644 --- a/apps/sim/lib/copilot/tools/server/files/create-file.ts +++ b/apps/sim/lib/copilot/tools/server/files/create-file.ts @@ -64,6 +64,9 @@ export const createFileServerTool: BaseServerTool { const contentType = args.target.mimeType || args.inferredMimeType if (args.target.mode === 'overwrite') { @@ -177,7 +183,8 @@ export async function writeWorkspaceFileByPath(args: { existing.id, args.userId, args.buffer, - contentType || existing.type + contentType || existing.type, + { syncLiveDoc: args.syncLiveDoc } ) return { diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index a660f1544da..0eac611b3f8 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -22,7 +22,7 @@ import { canonicalWorkspaceFilePath, decodeVfsPathSegments } from '@/lib/copilot import { generateRequestId } from '@/lib/core/utils/request' import { generateRestoreName } from '@/lib/core/utils/restore-name' import type { DbOrTx } from '@/lib/db/types' -import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' +import { mergeEditIntoLiveFileDoc, notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { getServePathPrefix } from '@/lib/uploads' import { deleteFile, @@ -32,6 +32,7 @@ import { uploadFile, } from '@/lib/uploads/core/storage-service' import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' +import { isMarkdownFile } from '@/lib/uploads/utils/file-utils' import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' import { isUuid, sanitizeFileName } from '@/executor/constants' import type { UserFile } from '@/executor/types' @@ -1024,7 +1025,18 @@ export async function updateWorkspaceFileContent( fileId: string, userId: string, content: Buffer, - contentType?: string + contentType?: string, + options?: { + /** + * Whether to stream this write into any open collaborative editor as a live CRDT merge. Defaults + * to `true`, so EVERY external write path (copilot tools, the file tool, the content route) reaches + * an open editor through this one chokepoint — no per-writer wiring to forget. Pass `false` only + * for a write that must NOT touch the live doc: the relay's own project-to-markdown persist (which + * would otherwise merge the doc back into itself in a loop), and empty-shell creates whose real + * content arrives via a subsequent write. + */ + syncLiveDoc?: boolean + } ): Promise { logger.info(`Updating workspace file content: ${fileId} for workspace ${workspaceId}`) @@ -1143,6 +1155,18 @@ export async function updateWorkspaceFileContent( await cleanupWorkspaceStorageObject(finalized.oldKey, 'version replacement') } + // Stream this write into any open collaborative editor as a CRDT merge, so a copilot/tool edit + // shows up live instead of the file silently changing underneath the reader. Gated to markdown (the + // only format the collaborative editor renders) and best-effort (a no-op when nobody has the file + // open; never throws). This is the single chokepoint every external writer shares — the relay's own + // persist and empty-shell creates pass `syncLiveDoc: false` to stay out of it. + if ( + options?.syncLiveDoc !== false && + isMarkdownFile({ type: nextContentType, name: finalized.file.originalName }) + ) { + await mergeEditIntoLiveFileDoc(fileId, content.toString('utf-8')) + } + const pathPrefix = getServePathPrefix() const currentFolderPath = finalized.file.folderId === fileRecord.folderId ? fileRecord.folderPath : null diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts index 9ac40d5587f..138bb8e1293 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts @@ -12,6 +12,8 @@ const { mockHeadObject, mockIncrementStorageUsageForBillingContextInTx, mockMaybeNotifyStorageLimitForBillingContext, + mockMergeEditIntoLiveFileDoc, + mockNotifyWorkspaceFilesChanged, mockResolveStorageBillingContext, mockUploadFile, } = vi.hoisted(() => ({ @@ -22,10 +24,17 @@ const { mockHeadObject: vi.fn(), mockIncrementStorageUsageForBillingContextInTx: vi.fn(), mockMaybeNotifyStorageLimitForBillingContext: vi.fn(), + mockMergeEditIntoLiveFileDoc: vi.fn(), + mockNotifyWorkspaceFilesChanged: vi.fn(), mockResolveStorageBillingContext: vi.fn(), mockUploadFile: vi.fn(), })) +vi.mock('@/lib/realtime/notify', () => ({ + mergeEditIntoLiveFileDoc: mockMergeEditIntoLiveFileDoc, + notifyWorkspaceFilesChanged: mockNotifyWorkspaceFilesChanged, +})) + vi.mock('@/lib/billing/storage', () => ({ decrementStorageUsageForBillingContextInTx: mockDecrementStorageUsageForBillingContextInTx, incrementStorageUsageForBillingContextInTx: mockIncrementStorageUsageForBillingContextInTx, @@ -105,6 +114,8 @@ describe('workspace file metadata and storage accounting', () => { mockDecrementStorageUsageForBillingContextInTx.mockResolvedValue(undefined) mockMaybeNotifyStorageLimitForBillingContext.mockResolvedValue(undefined) mockDeleteFile.mockResolvedValue(undefined) + mockMergeEditIntoLiveFileDoc.mockResolvedValue(undefined) + mockNotifyWorkspaceFilesChanged.mockResolvedValue(undefined) }) it('cleans up a newly uploaded object when atomic metadata finalization rolls back', async () => { @@ -328,4 +339,57 @@ describe('workspace file metadata and storage accounting', () => { expect(mockDeleteFile).toHaveBeenCalledTimes(1) expect(mockDeleteFile).toHaveBeenCalledWith({ key: replacementKey, context: 'workspace' }) }) + + const MD_ROW = { ...FILE_ROW, originalName: 'note.md', contentType: 'text/markdown' } + + it('streams a markdown overwrite into any open collaborative editor (the shared merge chokepoint)', async () => { + const updatedFile = { ...MD_ROW, size: 12 } + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: MD_ROW.key }) + + await updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.from('# new content', 'utf-8') + ) + + expect(mockMergeEditIntoLiveFileDoc).toHaveBeenCalledWith(MD_ROW.id, '# new content') + }) + + it('does NOT merge when syncLiveDoc is false (the relay persist / empty-shell opt-out)', async () => { + const updatedFile = { ...MD_ROW, size: 12 } + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: MD_ROW.key }) + + await updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.from('# new content', 'utf-8'), + undefined, + { syncLiveDoc: false } + ) + + expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + }) + + it('does NOT merge a non-markdown write (the collaborative editor only renders markdown)', async () => { + const updatedFile = { ...FILE_ROW, size: 10 } + dbChainMockFns.limit.mockResolvedValueOnce([FILE_ROW]).mockResolvedValueOnce([FILE_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: FILE_ROW.key }) + + await updateWorkspaceFileContent( + FILE_ROW.workspaceId, + FILE_ROW.id, + FILE_ROW.userId, + Buffer.alloc(10), + 'application/octet-stream' + ) + + expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + }) }) diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 4e7f5f6ca23..b508c48c80f 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 994, - zodRoutes: 994, + totalRoutes: 996, + zodRoutes: 996, nonZodRoutes: 0, } as const From caa0a7313cc554bfd148ae69d384ce980f17b66a Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 29 Jul 2026 17:59:30 -0700 Subject: [PATCH 52/53] fix(rich-markdown-editor): defer non-collab settle/stream mutations off the render phase (flushSync) (#6073) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(rich-markdown-editor): defer non-collab settle/stream mutations off the render phase (flushSync) The non-collaborative streaming/settle effect called editor.setContent / setEditable / setTextSelection / focus directly in the effect body. setContent mounts the custom node views synchronously through the @tiptap/react flushSync path (tiptap#3764), so when this effect runs while React is mid-render it throws "flushSync was called from inside a lifecycle method." This is the second flushSync source (the collab editability effect was the first, fixed separately); it fires on the agent-streaming-into-a-non-collab-editor surface. Defer the effect-body view mutations to a microtask via a small runOffRender helper (runs right after the current commit, before paint; no-ops if the editor was torn down). The settle block is deferred as ONE microtask so setContent -> collapse selection -> setEditable -> focus keep their order. The streaming rAF tick is left untouched — it already runs off-render, so it keeps writing content directly. queueMicrotask is TipTap's own documented remedy for this warning. 497 rich-markdown-editor tests (incl. stream-settle-selection) pass; tsc + lint + api-validation + boundary + prune green. Needs a live check: stream an agent into a non-collab markdown file and confirm it still renders smoothly. * chore(rich-markdown-editor): trim verbose flushSync-defer comments * fix(rich-markdown-editor): drop superseded settle/stream microtasks via a run token runOffRender previously only guarded editor.isDestroyed, so if React ran the next reconcile pass (a newer stream or settle) before a queued microtask flushed, the stale microtask could still apply setContent/setEditable/setTextSelection over the newer state. Tag each effect run with an incrementing token; a deferred mutation applies only when its run is still the latest (and the editor is alive). A run token fits this effect's several early-return exits better than a per-exit cleanup flag. Addresses Greptile/Cursor review. * fix(rich-markdown-editor): never drop the settle selection-collapse under a superseded run The run token drops a superseded settle's microtask, but the settle had already flipped its state flags synchronously — so a pre-empting steady-sync run took the non-settle path and never collapsed the selection, leaving a post-stream select-all painting the leaf-in-selection decoration. Track the collapse as a debt (pendingCollapseRef): whichever deferred run ultimately applies — settle or the steady-sync path — clears it, so the collapse runs exactly once on the latest content. Addresses the Cursor review finding. --- .../rich-markdown-editor.tsx | 67 ++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index feefae1e255..4290dd696fe 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -784,6 +784,8 @@ export function LoadedRichMarkdownEditor({ const pendingStreamBodyRef = useRef(null) const streamRafRef = useRef(null) const lastStreamParseAtRef = useRef(0) + const settleRunSeqRef = useRef(0) + const pendingCollapseRef = useRef(false) useEffect(() => { if (!editor) return // Collaboration and agent-streaming are disjoint surfaces: collaboration is enabled @@ -801,9 +803,29 @@ export function LoadedRichMarkdownEditor({ emitUpdate: false, }) } + // Editor view mutations flush synchronously through the @tiptap/react binding (setContent mounts + // the React node views via flushSync — tiptap#3764), so calling them directly in this effect body + // throws "flushSync ... cannot flush while rendering" when the effect runs mid-render. Defer to a + // microtask (after commit, before paint). The streaming rAF tick below already runs off-render. + // + // Tag each run: a deferred mutation applies only if it is still the latest run (this effect has + // several early-return exits, so a run-token beats a per-exit cleanup flag) and the editor is + // alive. This drops a superseded run's microtask when React ran the next pass — a newer stream or + // settle — before the microtask flushed, so it can't overwrite the newer state. + const runSeq = ++settleRunSeqRef.current + const runOffRender = (mutate: () => void) => { + queueMicrotask(() => { + if (runSeq !== settleRunSeqRef.current || editor.isDestroyed) return + mutate() + }) + } if (isStreaming) { wasStreamingRef.current = true - if (editor.isEditable) editor.setEditable(false) + if (editor.isEditable) { + runOffRender(() => { + if (editor.isEditable) editor.setEditable(false) + }) + } const body = splitFrontmatter(content).body if (body === lastSyncedBodyRef.current) return pendingStreamBodyRef.current = body @@ -852,22 +874,37 @@ export function LoadedRichMarkdownEditor({ if (isInitialSettle || wasStreamingRef.current) { wasStreamingRef.current = false settledRef.current = lockSettled(content) - syncEditorBody(splitFrontmatter(content).body) - // `setContent` maps any pre-existing selection onto the new doc rather than clearing it — a - // select-all survives as "select everything," permanently painting every divider/image with the - // `rich-leaf-in-selection` decoration (keymap.ts) until the user clicks elsewhere. This must run - // on every settle regardless of whether `setContent` ran just above: the last streaming tick - // already syncs `lastSyncedBodyRef` to the final body before settle, so `body` usually already - // equals it here — collapsing only inside that `if` would skip the common streamed-content case - // entirely. `setTextSelection` (not `.focus()`) so this never steals DOM focus from whatever the - // user is doing outside the editor. - editor.commands.setTextSelection(editor.state.doc.content.size) - editor.setEditable(canEdit && settledRef.current.verdict && collabReady) - if (isInitialSettle && autoFocus) editor.commands.focus('end') + const settledVerdict = settledRef.current.verdict + const shouldFocus = isInitialSettle && autoFocus + // A settle owes a selection collapse. Track it as a ref, not just inline in this microtask: if a + // newer run bumps the token before this microtask fires, this settle's task is dropped — but the + // debt survives, and the run that supersedes it (settle OR the steady-sync path below) clears it. + pendingCollapseRef.current = true + // One ordered microtask: set body → collapse selection → re-apply editability. The collapse is + // load-bearing and runs on every settle even when the body is unchanged — setContent maps a + // pre-existing selection onto the new doc, so a prior select-all survives as "select everything", + // permanently painting every divider/image with the rich-leaf-in-selection decoration (keymap.ts) + // until the user clicks away. setTextSelection (not .focus()) never steals DOM focus. + runOffRender(() => { + syncEditorBody(splitFrontmatter(content).body) + pendingCollapseRef.current = false + editor.commands.setTextSelection(editor.state.doc.content.size) + editor.setEditable(canEdit && settledVerdict && collabReady) + if (shouldFocus) editor.commands.focus('end') + }) return } - syncEditorBody(splitFrontmatter(content).body) - if (settledRef.current) editor.setEditable(canEdit && settledRef.current.verdict && collabReady) + const settled = settledRef.current + runOffRender(() => { + syncEditorBody(splitFrontmatter(content).body) + // Honor a collapse a superseded settle owed but never applied (its microtask was dropped when this + // run bumped the token), so a post-stream select-all can't keep the leaf-in-selection decoration. + if (pendingCollapseRef.current) { + pendingCollapseRef.current = false + editor.commands.setTextSelection(editor.state.doc.content.size) + } + if (settled) editor.setEditable(canEdit && settled.verdict && collabReady) + }) }, [ editor, content, From 525c340280b54e3d8ea58dc530d1ef12ae1ad6a2 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 29 Jul 2026 18:31:17 -0700 Subject: [PATCH 53/53] feat(tables): show live cell-selection carets in the embedded chat panel (#6081) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The table cell-selection presence room was joined only on the dedicated /tables/[id] page (useTableRoom was passed an empty id in embedded mode). Join it in embedded too, so the mothership chat resource panel shows collaborators' live cell selections and broadcasts the local one. tableId is already resolved from props in embedded (the data event stream already uses it un-gated), and authz runs on join, so this is safe. Avatars are unaffected — they render only in the !embedded Resource.Header, so the panel gets carets without avatars. --- .../[workspaceId]/tables/[tableId]/table.tsx | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index 9bb9b212df9..4ff7ca9cc19 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -231,13 +231,11 @@ export function Table({ } useTableEventStream({ tableId, workspaceId, onUsageLimitReached }) - // Live table presence (avatars + cell-selection highlights). Scoped to the - // dedicated tables page — the embedded surface passes no id, disabling it. - const { - otherUsers: presenceUsers, - remoteSelections, - emitCellSelection, - } = useTableRoom(embedded ? '' : tableId) + // Live table presence (cell-selection carets + avatars). Runs in both modes so the + // mothership chat panel shows collaborators' live selections too. The avatar stack lives + // only in the `!embedded` Resource.Header, so the embedded panel gets carets without avatars + // for free — matching the panel's own-chrome layout. + const { otherUsers: presenceUsers, remoteSelections, emitCellSelection } = useTableRoom(tableId) const [slideout, dispatch] = useReducer(slideoutReducer, { kind: 'none' }) const [showDeleteTableConfirm, setShowDeleteTableConfirm] = useState(false)