diff --git a/apps/realtime/package.json b/apps/realtime/package.json index 633ffb60827..35e45fc29e6 100644 --- a/apps/realtime/package.json +++ b/apps/realtime/package.json @@ -33,9 +33,12 @@ "@sim/workflow-types": "workspace:*", "@socket.io/redis-adapter": "8.3.0", "drizzle-orm": "^0.45.2", + "lib0": "0.2.117", "postgres": "^3.4.5", "redis": "5.10.0", "socket.io": "^4.8.1", + "y-protocols": "1.0.7", + "yjs": "13.6.31", "zod": "4.3.6" }, "devDependencies": { diff --git a/apps/realtime/src/access-revalidation.test.ts b/apps/realtime/src/access-revalidation.test.ts index 0688d5be334..4ea7999c301 100644 --- a/apps/realtime/src/access-revalidation.test.ts +++ b/apps/realtime/src/access-revalidation.test.ts @@ -50,16 +50,14 @@ function makeManager(sockets: FakeSocket[], presence: Partial[] = const manager = { io: { sockets: { sockets: socketMap } }, isReady: () => true, - getWorkflowUsers: vi.fn().mockResolvedValue(presence), - getWorkflowIdForSocket: vi.fn().mockResolvedValue(null), - removeUserFromRoom: vi - .fn() - .mockImplementation(async (_socketId: string, workflowId?: string) => workflowId ?? null), + getRoomUsers: vi.fn().mockResolvedValue(presence), + getRoomForSocket: vi.fn().mockResolvedValue(null), + removeUserFromRoom: vi.fn().mockResolvedValue(true), broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), } return manager as unknown as IRoomManager & { - getWorkflowUsers: ReturnType - getWorkflowIdForSocket: ReturnType + getRoomUsers: ReturnType + getRoomForSocket: ReturnType removeUserFromRoom: ReturnType broadcastPresenceUpdate: ReturnType } @@ -84,8 +82,11 @@ describe('access-revalidation sweep', () => { expect.objectContaining({ workflowId: 'wf-1' }) ) expect(socket.leave).toHaveBeenCalledWith('wf-1') - expect(manager.removeUserFromRoom).toHaveBeenCalledWith('sock-1', 'wf-1') - expect(manager.broadcastPresenceUpdate).toHaveBeenCalledWith('wf-1') + expect(manager.removeUserFromRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'wf-1' }, + 'sock-1' + ) + expect(manager.broadcastPresenceUpdate).toHaveBeenCalledWith({ type: 'workflow', id: 'wf-1' }) }) it('keeps a socket whose access is still valid', async () => { @@ -141,7 +142,30 @@ describe('access-revalidation sweep', () => { expect(mockResolveRole).toHaveBeenCalledWith('user-1', 'wf-1', 'read') // The security scan must stay Redis-free — presence is never consulted. - expect(manager.getWorkflowUsers).not.toHaveBeenCalled() + expect(manager.getRoomUsers).not.toHaveBeenCalled() + }) + + it('never evicts a socket joined only to a non-workflow room (files/tables/file-doc)', async () => { + // The sweep shares one io with the files/tables/file-doc handlers. Those rooms are + // namespaced (`workspace-files:ws-1`, `table:t-1`), so treating every socket.rooms + // entry as a workflow id would resolve a bogus permission → null → evict the socket + // from its files/table room every pass. Non-workflow rooms must be filtered out. + const filesSocket = makeSocket('sock-1', 'user-1', 'workspace-files:ws-1') + const tableSocket = makeSocket('sock-2', 'user-2', 'table:t-1') + const manager = makeManager([filesSocket, tableSocket]) + // Even if the role resolver would say "no access", these must never be swept. + mockResolveRole.mockResolvedValue(null) + + const sweep = startAccessRevalidationSweep(manager) + await sweep.runOnce() + sweep.stop() + + expect(mockResolveRole).not.toHaveBeenCalled() + expect(filesSocket.leave).not.toHaveBeenCalled() + expect(filesSocket.emit).not.toHaveBeenCalled() + expect(tableSocket.leave).not.toHaveBeenCalled() + expect(tableSocket.emit).not.toHaveBeenCalled() + expect(manager.removeUserFromRoom).not.toHaveBeenCalled() }) it('evicts only the revoked socket, not co-members of the room', async () => { @@ -199,28 +223,28 @@ describe('access-revalidation sweep', () => { sweep.stop() expect(manager.removeUserFromRoom).toHaveBeenCalledTimes(2) - expect(manager.broadcastPresenceUpdate).toHaveBeenCalledWith('wf-1') + expect(manager.broadcastPresenceUpdate).toHaveBeenCalledWith({ type: 'workflow', id: 'wf-1' }) }) - it('defers cleanup when removal fails with expired socket mappings', async () => { + it('drops eviction cleanup when the socket is no longer mapped to the room (no infinite retry)', async () => { const socket = makeSocket('sock-1', 'user-1', 'wf-1') const manager = makeManager([socket], [{ socketId: 'sock-1', role: 'read' }]) - // Mapping keys already expired (lookup resolves null) AND the removal fails - // (the Redis manager swallows the transport error into null) — the failed - // removal must still defer instead of reading as success. - manager.removeUserFromRoom.mockResolvedValueOnce(null) + // A healthy lookup shows the socket is no longer mapped to any workflow room (its presence + // is already gone), and removeUserFromRoom reports a no-op `false`. This is "already clean", + // not a deferrable failure — the cleanup must drop it, never re-enqueue a still-connected + // socket forever. (A genuine failure — still mapped + false — is covered by the next test.) + manager.getRoomForSocket.mockResolvedValue(null) + manager.removeUserFromRoom.mockResolvedValue(false) mockResolveRole.mockResolvedValue(null) const sweep = startAccessRevalidationSweep(manager) await sweep.runOnce() - - expect(manager.broadcastPresenceUpdate).not.toHaveBeenCalled() - await sweep.runOnce() sweep.stop() - expect(manager.removeUserFromRoom).toHaveBeenCalledTimes(2) - expect(manager.broadcastPresenceUpdate).toHaveBeenCalledWith('wf-1') + // Attempted once, then dropped — not re-enqueued across passes, and no broadcast. + expect(manager.removeUserFromRoom).toHaveBeenCalledTimes(1) + expect(manager.broadcastPresenceUpdate).not.toHaveBeenCalled() }) it('defers cleanup when the manager swallows a removal failure into null', async () => { @@ -228,8 +252,8 @@ describe('access-revalidation sweep', () => { const manager = makeManager([socket], [{ socketId: 'sock-1', role: 'read' }]) // Live mapping but the removal reports nothing removed — the Redis manager // swallows transport errors into null, so this is the only failure signal. - manager.getWorkflowIdForSocket.mockResolvedValue('wf-1') - manager.removeUserFromRoom.mockResolvedValueOnce(null) + manager.getRoomForSocket.mockResolvedValue({ type: 'workflow', id: 'wf-1' }) + manager.removeUserFromRoom.mockResolvedValueOnce(false) mockResolveRole.mockResolvedValue(null) const sweep = startAccessRevalidationSweep(manager) @@ -243,7 +267,7 @@ describe('access-revalidation sweep', () => { sweep.stop() expect(manager.removeUserFromRoom).toHaveBeenCalledTimes(2) - expect(manager.broadcastPresenceUpdate).toHaveBeenCalledWith('wf-1') + expect(manager.broadcastPresenceUpdate).toHaveBeenCalledWith({ type: 'workflow', id: 'wf-1' }) }) it('skips removal when the socket has since moved to a different workflow', async () => { @@ -251,7 +275,7 @@ describe('access-revalidation sweep', () => { const manager = makeManager([socket], [{ socketId: 'sock-1', role: 'read' }]) // Between the membership snapshot and cleanup, the socket switched to a // workflow it can still access — removal must not touch its new presence. - manager.getWorkflowIdForSocket.mockResolvedValue('wf-2') + manager.getRoomForSocket.mockResolvedValue({ type: 'workflow', id: 'wf-2' }) mockResolveRole.mockResolvedValue(null) const sweep = startAccessRevalidationSweep(manager) @@ -357,7 +381,7 @@ describe('access-revalidation sweep', () => { const manager = makeManager([socket], [{ socketId: 'sock-1', role: 'read' }]) // A Redis outage where commands hang in the offline queue instead of // failing: the cleanup lane stalls, but scans must keep running. - manager.getWorkflowIdForSocket.mockReturnValue(new Promise(() => {})) + manager.getRoomForSocket.mockReturnValue(new Promise(() => {})) mockResolveRole.mockResolvedValue(null) const sweep = startAccessRevalidationSweep(manager) diff --git a/apps/realtime/src/access-revalidation.ts b/apps/realtime/src/access-revalidation.ts index 18d63ea490d..93edaae9a0b 100644 --- a/apps/realtime/src/access-revalidation.ts +++ b/apps/realtime/src/access-revalidation.ts @@ -1,9 +1,10 @@ import { createLogger } from '@sim/logger' import type { AccessRevokedBroadcast } from '@sim/realtime-protocol/events' +import { parseRoomName, ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { sleep } from '@sim/utils/helpers' import type { AuthenticatedSocket } from '@/middleware/auth' import { ROLE_REVALIDATION_TTL_MS, resolveCurrentWorkflowRole } from '@/middleware/permissions' -import type { IRoomManager } from '@/rooms' +import { type IRoomManager, workflowRoom as wf } from '@/rooms' const logger = createLogger('AccessRevalidation') @@ -65,9 +66,14 @@ interface ScanTarget { * Collects this pod's authenticated sockets with the workflow room each has * joined, in stable socket order. * - * The workflow room is derived from the socket's own `rooms` set (pod-local, no - * Redis round-trips): a socket joins exactly one workflow room, so its rooms are - * `{ ownSocketId, workflowId }`. Only local sockets are evaluated — sockets are + * Rooms are derived from the socket's own `rooms` set (pod-local, no Redis + * round-trips). A socket may occupy several rooms of different types at once + * (workflow canvas, workspace-files browser, table, file-doc), all on the same + * io — so each name is decoded with {@link parseRoomName} and only **workflow** + * rooms are swept here. Non-workflow room names are namespaced (`type:id`) and + * resolve to a non-workflow type; sweeping them as workflow ids would resolve a + * bogus permission, come back `null`, and spuriously evict the socket from its + * files/table room every pass. Only local sockets are evaluated — sockets are * sticky to a pod, so every socket is swept by exactly one pod using that pod's * warm role cache (mirroring the per-pod reasoning of the write-path cache). */ @@ -78,7 +84,9 @@ function collectScanTargets(io: IRoomManager['io']): ScanTarget[] { if (!authed.userId) continue for (const room of socket.rooms) { if (room === socket.id) continue - targets.push({ workflowId: room, socket: authed, userId: authed.userId }) + const ref = parseRoomName(room) + if (ref?.type !== ROOM_TYPES.WORKFLOW) continue + targets.push({ workflowId: ref.id, socket: authed, userId: authed.userId }) } } return targets @@ -129,9 +137,20 @@ export function startAccessRevalidationSweep(roomManager: IRoomManager): AccessR async function cleanupEvictedSocket(socketId: string, workflowId: string): Promise { const key = `${socketId}:${workflowId}` try { + // A fully-disconnected socket already had its presence removed by the + // disconnect handler (removeSocketFromAllRooms), so there is nothing left to + // clean. Dropping here also keeps the boolean removeUserFromRoom below from + // reporting a false "not a member" for an already-gone entry and retrying it + // forever (the pre-generalization manager returned the target on a no-op). + if (!io.sockets.sockets.get(socketId)) { + pendingCleanups.delete(key) + return + } + // Unlike removeUserFromRoom, this read does not swallow transport errors, // so a Redis outage lands in the catch below and defers the cleanup. - const currentWorkflowId = await roomManager.getWorkflowIdForSocket(socketId) + const currentRoom = await roomManager.getRoomForSocket(socketId, ROOM_TYPES.WORKFLOW) + const currentWorkflowId = currentRoom?.id ?? null if (currentWorkflowId !== null && currentWorkflowId !== workflowId) { // The socket has since moved to a different workflow it can still // access; that join's room switch already removed this room's presence @@ -148,16 +167,26 @@ export function startAccessRevalidationSweep(roomManager: IRoomManager): AccessR return } - const removed = await roomManager.removeUserFromRoom(socketId, workflowId) - if (removed === null) { - // The sweep always passes the target room, and both managers report a - // performed removal by returning it — the Redis manager swallows - // transport errors into null, so null means the removal did not happen - // (even when the socket's mapping keys have already expired). - throw new Error('room-state removal not confirmed') + // A null mapping here is the normal case (the socket's mapping key may have + // expired) and does NOT mean "skip" — the eviction still removes the presence + // entry from the known target room via the explicit ref below. + const removed = await roomManager.removeUserFromRoom(wf(workflowId), socketId) + if (!removed) { + // `false` conflates two outcomes: the entry was already gone (a no-op), or a + // transport error the manager swallowed. Only retry when the socket is still mapped + // to THIS room — then a false result is a genuine, deferrable failure. When a healthy + // getRoomForSocket above returned no workflow mapping (`currentWorkflowId === null`), + // the presence entry is already gone, so the cleanup is complete: dropping it avoids + // re-enqueuing a still-connected socket forever. (A real Redis outage throws at + // getRoomForSocket and is deferred by the outer catch, never reaching here.) + if (currentWorkflowId === workflowId) { + throw new Error('room-state removal not confirmed') + } + pendingCleanups.delete(key) + return } - await roomManager.broadcastPresenceUpdate(workflowId) + await roomManager.broadcastPresenceUpdate(wf(workflowId)) pendingCleanups.delete(key) } catch (error) { pendingCleanups.set(key, { socketId, workflowId }) diff --git a/apps/realtime/src/handlers/avatar.ts b/apps/realtime/src/handlers/avatar.ts new file mode 100644 index 00000000000..349b166e270 --- /dev/null +++ b/apps/realtime/src/handlers/avatar.ts @@ -0,0 +1,29 @@ +import { db, user } from '@sim/db' +import { createLogger } from '@sim/logger' +import { eq } from 'drizzle-orm' +import type { AuthenticatedSocket } from '@/middleware/auth' + +const logger = createLogger('PresenceAvatar') + +/** + * The avatar URL for a presence entry: the socket's authenticated image when + * present, otherwise a single lookup of the user's stored image. Never throws — + * presence must not fail on an avatar lookup, so a DB error resolves to `null`. + */ +export async function resolveAvatarUrl( + socket: AuthenticatedSocket, + userId: string +): Promise { + if (socket.userImage) return socket.userImage + try { + const [record] = await db + .select({ image: user.image }) + .from(user) + .where(eq(user.id, userId)) + .limit(1) + return record?.image ?? null + } catch (error) { + logger.warn('Failed to load user avatar for presence', { userId, error }) + return null + } +} diff --git a/apps/realtime/src/handlers/connection.ts b/apps/realtime/src/handlers/connection.ts index 90eddb82464..33d90b5bfb0 100644 --- a/apps/realtime/src/handlers/connection.ts +++ b/apps/realtime/src/handlers/connection.ts @@ -1,4 +1,6 @@ import { createLogger } from '@sim/logger' +import { parseRoomName, ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { cleanupFileDocForSocket } from '@/handlers/file-doc' import { cleanupPendingSubblocksForSocket } from '@/handlers/subblocks' import { cleanupPendingVariablesForSocket } from '@/handlers/variables' import type { AuthenticatedSocket } from '@/middleware/auth' @@ -6,6 +8,14 @@ import type { IRoomManager } from '@/rooms' const logger = createLogger('ConnectionHandlers') +/** + * Room types whose presence lives in the room manager (Redis-backed), so a disconnect must + * remove the socket + broadcast a correction. The workspace-files and file-doc rooms are + * NOT here: workspace-files carries no presence (native Socket.IO membership only), and + * file-doc broadcasts its own server-authenticated roster via `cleanupFileDocForSocket`. + */ +const PRESENCE_BEARING_TYPES = new Set([ROOM_TYPES.WORKFLOW, ROOM_TYPES.TABLE]) + export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { socket.on('error', (error) => { logger.error(`Socket ${socket.id} error:`, error) @@ -15,20 +25,68 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager logger.error(`Socket ${socket.id} connection error:`, error) }) - socket.on('disconnect', async (reason) => { + // `disconnecting` (not `disconnect`): here `socket.rooms` is still populated and + // authoritative, so presence is cleaned up even if the Redis room-set key was + // evicted or TTL-expired (which would leave the manager's stored rooms empty). + socket.on('disconnecting', async (reason) => { try { + // Snapshot the live Socket.IO room membership SYNCHRONOUSLY, before any + // await: Socket.IO clears `socket.rooms` via leaveAll() as soon as the + // synchronous portion of this `disconnecting` handler returns (i.e. at the + // first await below), so reading it afterwards would see an empty set and + // the eviction fallback would be dead. + const liveRoomNames = [...socket.rooms] + // Clean up pending debounce entries for this socket to prevent memory leaks cleanupPendingSubblocksForSocket(socket.id) cleanupPendingVariablesForSocket(socket.id) + // Clear the socket's collaborative-document awareness (removes its caret for + // everyone else) and drop the room if it was the last editor. `endOfLife` drops the + // socket's join-generation entry — safe only here, on true disconnect (see cleanup). + cleanupFileDocForSocket(socket.id, roomManager.io, true) - const workflowIdHint = [...socket.rooms].find((roomId) => roomId !== socket.id) - const workflowId = await roomManager.removeUserFromRoom(socket.id, workflowIdHint) + // A socket may occupy multiple rooms (one per type). Remove it from every + // room the manager knows about. + const removedRooms = await roomManager.removeSocketFromAllRooms(socket.id) + + // Union with the snapshotted Socket.IO membership (authoritative, and it + // survives a Redis eviction/TTL lapse that would leave the manager's tracked + // rooms empty). Attempt removal for any room the manager didn't already + // remove — best-effort, since a transient Redis error can't be recovered here. + const wasInRooms = new Map() + // Only presence-bearing rooms get a corrective broadcast. Manager-removed rooms are + // presence-bearing by construction today (only workflow/table write the socket→room hash), + // but filter symmetrically with the fallback path below so a future room type that ever + // tracks presence here can't emit a bogus presence-update no client listens to. + for (const room of removedRooms) { + if (PRESENCE_BEARING_TYPES.has(room.type)) wasInRooms.set(roomName(room), room) + } + for (const name of liveRoomNames) { + // `wasInRooms.has(name)` already excludes every room the manager removed (same + // room-name key via the roomName/parseRoomName bijection). Skip room types with no + // manager-tracked presence (workspace-files, file-doc): removing there is a no-op and + // broadcasting a correction would emit a dead presence-update no client listens to. + if (name === socket.id || wasInRooms.has(name)) continue + const ref = parseRoomName(name) + if (!ref || !PRESENCE_BEARING_TYPES.has(ref.type)) continue + wasInRooms.set(name, ref) + await roomManager.removeUserFromRoom(ref, socket.id) + } + + // Broadcast a correction to every room this socket was in, EXCLUDING this + // socket — so it is never shown as a ghost collaborator even if its presence + // entry outlived a failed removal (transient Redis error; the hashes have no + // TTL). Any orphaned entry is additionally reclaimed by the next join's + // stale-presence sweep. + for (const room of wasInRooms.values()) { + await roomManager.broadcastPresenceUpdate(room, socket.id) + } - if (workflowId) { - await roomManager.broadcastPresenceUpdate(workflowId) - logger.info( - `Socket ${socket.id} disconnected from workflow ${workflowId} (reason: ${reason})` - ) + if (wasInRooms.size > 0) { + const rooms = Array.from(wasInRooms.values()) + .map((room) => `${room.type}:${room.id}`) + .join(', ') + logger.info(`Socket ${socket.id} disconnected from [${rooms}] (reason: ${reason})`) } } catch (error) { logger.error(`Error handling disconnect for socket ${socket.id}:`, error) diff --git a/apps/realtime/src/handlers/file-doc-app.ts b/apps/realtime/src/handlers/file-doc-app.ts new file mode 100644 index 00000000000..8223a5eef74 --- /dev/null +++ b/apps/realtime/src/handlers/file-doc-app.ts @@ -0,0 +1,97 @@ +import { FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' +import { env, getBaseUrl } from '@/env' + +/** + * The relay's client for the app's internal file-doc endpoints. The app owns the markdown↔Yjs + * conversion engine (TipTap + jsdom) and blob/DB access; the relay owns the live document. So for any + * operation that needs conversion, the relay delegates here over the shared `x-api-key` channel. The + * timeouts (and their ordering vs. the app-side bounds) live in the shared `FILE_DOC_TIMEOUTS`. + */ + +function postToApp(path: string, payload: unknown, timeoutMs: number): Promise { + return fetch(`${getBaseUrl()}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(timeoutMs), + }) +} + +/** + * Ask the app to build a server-authoritative seed (markdown → Yjs) for a file's collaborative + * document. Returns the Yjs update to apply, or `null` for a genuinely empty/missing file (an empty + * document is correct). THROWS on a transport failure (non-2xx / network / timeout / malformed body) + * so the caller can tell a real empty from a failure it should be allowed to retry. + */ +export async function fetchFileDocSeed( + workspaceId: string, + fileId: string +): Promise { + const response = await postToApp( + '/api/internal/file-doc/seed', + { workspaceId, fileId }, + FILE_DOC_TIMEOUTS.seedRequestMs + ) + if (!response.ok) { + throw new Error(`Seed fetch failed for file ${fileId}: ${response.status}`) + } + const body = (await response.json()) as { update?: unknown } + const update = body?.update + // A well-formed response is `{ update: base64-string | null }`. Anything else is a contract + // violation, not a "genuinely empty file" — throw so the caller retries rather than silently + // treating a malformed body as empty and stranding the room unseeded. + if (update === null) return null + if (typeof update !== 'string') { + throw new Error(`Seed fetch for file ${fileId} returned a malformed body`) + } + return new Uint8Array(Buffer.from(update, 'base64')) +} + +/** + * Ask the app to merge new markdown into the live document as a minimal Yjs diff — Stage C, so a + * copilot edit streams into open editors instead of the file changing underneath them. The relay + * ships the document's current state and applies the returned diff (which Yjs reconciles with any + * concurrent user edits). THROWS on a transport failure or malformed body. + */ +export async function fetchFileDocMerge( + fileId: string, + docState: Uint8Array, + markdown: string +): Promise { + const response = await postToApp( + '/api/internal/file-doc/merge', + { fileId, docState: Buffer.from(docState).toString('base64'), markdown }, + FILE_DOC_TIMEOUTS.mergeRequestMs + ) + if (!response.ok) { + throw new Error(`Merge fetch failed for file ${fileId}: ${response.status}`) + } + const body = (await response.json()) as { update?: unknown } + if (typeof body?.update !== 'string') { + throw new Error(`Merge fetch for file ${fileId} returned a malformed body`) + } + return new Uint8Array(Buffer.from(body.update, 'base64')) +} + +/** + * Ask the app to project a live collaborative document back to durable markdown and write it to the + * file (Yjs → markdown, through the exact editor engine). This is the server-authoritative durable + * path — called debounced while the doc is edited and when the last collaborator leaves — that + * replaces the editor's client autosave, so a server/copilot edit can't be clobbered by a stale + * keystroke. THROWS on a transport failure so the caller can log/retry on the next debounce. + */ +export async function fetchFileDocPersist( + workspaceId: string, + fileId: string, + userId: string, + docState: Uint8Array +): Promise { + const response = await postToApp( + '/api/internal/file-doc/persist', + { workspaceId, fileId, userId, docState: Buffer.from(docState).toString('base64') }, + FILE_DOC_TIMEOUTS.persistRequestMs + ) + if (!response.ok) { + throw new Error(`Persist failed for file ${fileId}: ${response.status}`) + } +} diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts new file mode 100644 index 00000000000..7a6aba13015 --- /dev/null +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -0,0 +1,394 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' + +/** + * One shared in-memory Redis backing per test, so several {@link FileDocStore} instances (modelling + * several ECS tasks) all talk to the "same Redis". A minimal fake of just the stream/lock ops the + * store uses. + */ +interface Backing { + streams: Map }[]> + kv: Map + seq: number + /** Number of upcoming xAdd calls to fail with a transient error (to exercise publish retry). */ + failXAdd: number +} + +const state = vi.hoisted(() => ({ backing: null as Backing | null })) + +const seqOf = (id: string) => Number(id.split('-')[0]) + +function makeClient(): any { + const b = () => { + if (!state.backing) throw new Error('backing not initialized') + return state.backing + } + const client: any = { + connect: async () => {}, + quit: async () => {}, + on: () => client, + duplicate: () => makeClient(), + xAdd: async (key: string, _star: string, fields: Record) => { + if (b().failXAdd > 0) { + b().failXAdd-- + throw new Error('transient xAdd failure') + } + const id = `${++b().seq}-0` + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { ...fields } }) + b().streams.set(key, arr) + return id + }, + xRange: async (key: string) => (b().streams.get(key) ?? []).map((e) => ({ ...e })), + xLen: async (key: string) => (b().streams.get(key) ?? []).length, + xTrim: async (key: string, _strategy: string, minid: string) => { + const arr = b().streams.get(key) ?? [] + b().streams.set( + key, + arr.filter((e) => seqOf(e.id) >= seqOf(minid)) + ) + }, + xRead: async (streams: { key: string; id: string }[]) => { + const res: { name: string; messages: { id: string; message: Record }[] }[] = + [] + for (const { key, id } of streams) { + const after = (b().streams.get(key) ?? []).filter((e) => seqOf(e.id) > seqOf(id)) + if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) + } + if (res.length) return res + await new Promise((r) => setTimeout(r, 5)) + return null + }, + set: async (key: string, val: string, opts?: { NX?: boolean }) => { + if (opts?.NX && b().kv.has(key)) return null + b().kv.set(key, val) + return 'OK' + }, + del: async (key: string) => { + b().kv.delete(key) + return 1 + }, + eval: async (script: string, opts: { keys: string[]; arguments: string[] }) => { + const [key] = opts.keys + // Atomic seed-if-empty (SEED_IF_EMPTY_SCRIPT): append the entry iff the stream is empty, in one + // synchronous step — mirroring Redis's atomic Lua execution, so two concurrent evals can never both + // append (the second sees a non-empty stream). + if (script.includes('xlen')) { + const [field, value] = opts.arguments + const arr = b().streams.get(key) ?? [] + if (arr.length > 0) return 0 + const id = `${++b().seq}-0` + arr.push({ id, message: { [field]: value } }) + b().streams.set(key, arr) + return 1 + } + // Compare-and-delete Lua (RELEASE_LOCK_SCRIPT): del only if the stored value matches the token. + const [token] = opts.arguments + if (b().kv.get(key) === token) { + b().kv.delete(key) + return 1 + } + return 0 + }, + expire: async () => 1, + } + return client +} + +vi.mock('redis', () => ({ createClient: () => makeClient() })) + +import { FileDocStore } from '@/handlers/file-doc-store' + +const REDIS_URL = 'redis://fake' +const NAME = 'workspace-file-doc:file-1' + +function docWithText(text: string): Y.Doc { + const doc = new Y.Doc() + doc.getText('body').insert(0, text) + return doc +} + +/** The delta a doc emits when `text` is inserted — what the relay would `publish`. */ +function updateFor(text: string): Uint8Array { + const doc = docWithText(text) + const update = Y.encodeStateAsUpdate(doc) + doc.destroy() + return update +} + +let stores: FileDocStore[] = [] +async function newStore(): Promise { + const store = new FileDocStore(REDIS_URL) + await store.init() + stores.push(store) + return store +} + +describe('FileDocStore', () => { + beforeEach(() => { + state.backing = { streams: new Map(), kv: new Map(), seq: 0, failXAdd: 0 } + stores = [] + }) + + afterEach(async () => { + await Promise.all(stores.map((s) => s.shutdown())) + }) + + it('elects exactly one seeder across tasks (no split-brain seed)', async () => { + const a = await newStore() + const b = await newStore() + // shouldSeed returns a lock token (truthy) for the winner, null for the loser. + const [aTok, bTok] = await Promise.all([a.shouldSeed(NAME), b.shouldSeed(NAME)]) + expect([aTok, bTok].filter(Boolean)).toHaveLength(1) + }) + + it('does not re-seed once the stream already has content (stale lock)', async () => { + const a = await newStore() + const token = await a.shouldSeed(NAME) + expect(token).toBeTruthy() + // A seeds and releases its lock. + a.publish(NAME, updateFor('hello')) + await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) + await a.releaseSeedLock(NAME, token as string) + // A different task must NOT seed again — the lock is free but the stream is non-empty. + const b = await newStore() + expect(await b.shouldSeed(NAME)).toBeNull() + }) + + it('getStreamState reconstructs the shared document from the stream', async () => { + const a = await newStore() + a.publish(NAME, updateFor('shared content')) + let state: Uint8Array | null = null + await vi.waitFor(async () => { + state = await a.getStreamState(NAME) + expect(state).not.toBeNull() + }) + const doc = new Y.Doc() + Y.applyUpdate(doc, state!) + expect(doc.getText('body').toString()).toBe('shared content') + doc.destroy() + }) + + it('attachRoom catches a fresh task up to the current shared state', async () => { + const a = await newStore() + a.publish(NAME, updateFor('already here')) + await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) + + // A second task opens the same file: its doc must load the existing content, not start empty. + const b = await newStore() + const doc = new Y.Doc() + await b.attachRoom(NAME, doc) + expect(doc.getText('body').toString()).toBe('already here') + doc.destroy() + }) + + it('converges a peer task via the tailer after attach', async () => { + const a = await newStore() + const b = await newStore() + const bDoc = new Y.Doc() + await b.attachRoom(NAME, bDoc) + + // A publishes an edit; B's multiplexed reader must apply it to B's attached doc. + a.publish(NAME, updateFor('from task A')) + await vi.waitFor(() => expect(bDoc.getText('body').toString()).toBe('from task A'), { + timeout: 2000, + }) + bDoc.destroy() + }) + + it('compaction never trims peer entries the compacting task has not yet integrated', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(Y.encodeStateAsUpdate(new Y.Doc())).toString('base64') + + // Two peer edits published by ANOTHER task that this task's tailer has not read yet. + const peerDoc = new Y.Doc() + const peerUpdates: Uint8Array[] = [] + peerDoc.on('update', (u: Uint8Array) => peerUpdates.push(u)) + peerDoc.getText('body').insert(0, 'PEER1') + peerDoc.getText('body').insert(5, 'PEER2') + + // Backing: 400 already-integrated (no-op) entries this task's doc reflects, then the 2 un-integrated + // peer entries. Enough entries to cross COMPACT_THRESHOLD. + const entries = Array.from({ length: 400 }, (_, i) => ({ + id: `${i + 1}-0`, + message: { u: noop }, + })) + entries.push({ id: '401-0', message: { u: Buffer.from(peerUpdates[0]).toString('base64') } }) + entries.push({ id: '402-0', message: { u: Buffer.from(peerUpdates[1]).toString('base64') } }) + state.backing!.streams.set(streamKey, entries) + state.backing!.seq = 402 + + const a = await newStore() + // This task has integrated only up to entry 400 (all no-ops) — its local doc is empty and lags the + // two peer entries. Inject that lagging room directly. + ;(a as any).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0 }) + await (a as any).maybeCompact(NAME) + + // A fresh catch-up must still reconstruct the peer content — compaction must not have trimmed 401/402. + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(doc.getText('body').toString()).toBe('PEER1PEER2') + doc.destroy() + + // The appended snapshot entry must carry the snapshot marker, so a fresh catch-up task treats it as + // edited content (not a bare seed) and persists on last-disconnect. + const stream = state.backing!.streams.get(streamKey)! + expect(stream[stream.length - 1].message.s).toBe('1') + }) + + it('retries a transient append failure so the edit is not lost from the shared log', async () => { + const a = await newStore() + state.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed + a.publish(NAME, updateFor('resilient')) + await vi.waitFor( + async () => { + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(doc.getText('body').toString()).toBe('resilient') + doc.destroy() + }, + { timeout: 2000 } + ) + }) + + it('streamHasContent fences a seed apply against an already-seeded stream', async () => { + const a = await newStore() + expect(await a.streamHasContent(NAME)).toBe(false) + a.publish(NAME, updateFor('seeded')) + await vi.waitFor(async () => expect(await a.streamHasContent(NAME)).toBe(true)) + }) + + it('serializes merges across tasks via the merge lock', async () => { + const a = await newStore() + const b = await newStore() + const aTok = await a.acquireMergeSlot(NAME, 5_000) + expect(aTok).toBeTruthy() + // A holds it → B is refused until A releases. + expect(await b.acquireMergeSlot(NAME, 5_000)).toBeNull() + // A stale-holder release with the WRONG token must NOT free A's lock (compare-and-delete). + await b.releaseMergeSlot(NAME, 'wrong-token') + expect(await b.acquireMergeSlot(NAME, 5_000)).toBeNull() + // A releases with its real token → B can now acquire. + await a.releaseMergeSlot(NAME, aTok as string) + const bTok = await b.acquireMergeSlot(NAME, 5_000) + expect(bTok).toBeTruthy() + await b.releaseMergeSlot(NAME, bTok as string) + }) + + it('is disabled without a REDIS_URL and behaves single-replica', async () => { + const store = new FileDocStore(undefined) + expect(store.enabled).toBe(false) + // Seeds locally (returns a sentinel token), never touches a stream. + expect(await store.shouldSeed(NAME)).toBeTruthy() + expect(await store.getStreamState(NAME)).toBeNull() + const doc = new Y.Doc() + await store.attachRoom(NAME, doc) // no-op, no throw + expect(doc.getText('body').toString()).toBe('') + doc.destroy() + }) + + it('seedIfEmpty writes the seed once and reports it, then refuses a non-empty stream', async () => { + const a = await newStore() + expect(await a.seedIfEmpty(NAME, updateFor('first'))).toBe(true) + // A second seed attempt (any task) must be refused — the stream already holds content. + const b = await newStore() + expect(await b.seedIfEmpty(NAME, updateFor('second'))).toBe(false) + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(doc.getText('body').toString()).toBe('first') + doc.destroy() + }) + + it('atomic seed prevents split-brain even when the seed lock expired mid-seed', async () => { + // The exact split-brain precondition from the concurrency audit: the seed lock is only an efficiency + // optimization, so if it lapses (TTL) while the stream is still empty, TWO tasks can both hold a + // token and both try to seed with DIFFERENT docs (distinct Yjs client ids). The atomic seedIfEmpty + // must still let only one land — otherwise the union duplicates content. + const a = await newStore() + const b = await newStore() + const tokenA = await a.shouldSeed(NAME) + expect(tokenA).toBeTruthy() + // Simulate A's lock expiring mid-seed so B also wins the freed lock over a still-empty stream. + state.backing!.kv.delete(`filedoc:seedlock:${NAME}`) + const tokenB = await b.shouldSeed(NAME) + expect(tokenB).toBeTruthy() + // Both tasks now race to seed with distinct client ids. + const [seededA, seededB] = await Promise.all([ + a.seedIfEmpty(NAME, updateFor('SEED-A')), + b.seedIfEmpty(NAME, updateFor('SEED-B')), + ]) + expect([seededA, seededB].filter(Boolean)).toHaveLength(1) + // Exactly one seed is in the stream — the reconstructed text is a single seed, never a duplicated + // union of both (e.g. 'SEED-ASEED-B'). + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(['SEED-A', 'SEED-B']).toContain(doc.getText('body').toString()) + doc.destroy() + }) + + it('a peer edit published during attachRoom catch-up is not lost', async () => { + // Author two INCREMENTAL edits from one doc so they converge to 'basepeer' (not an independent union). + const author = new Y.Doc() + const updates: Uint8Array[] = [] + author.on('update', (u: Uint8Array) => updates.push(u)) + author.getText('body').insert(0, 'base') + author.getText('body').insert(4, 'peer') + + const a = await newStore() + a.publish(NAME, updates[0]) // 'base' + await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) + + // Task B attaches; while its synchronous catch-up runs, task A publishes the second edit. The tailer + // resumes from the id catch-up stopped at, so the edit converges rather than falling into a gap. + const b = await newStore() + const bDoc = new Y.Doc() + const attach = b.attachRoom(NAME, bDoc) + a.publish(NAME, updates[1]) // 'peer' appended + await attach + await vi.waitFor(() => expect(bDoc.getText('body').toString()).toBe('basepeer'), { + timeout: 2000, + }) + bDoc.destroy() + author.destroy() + }) + + it('concurrent compaction on two tasks preserves the full document', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(Y.encodeStateAsUpdate(new Y.Doc())).toString('base64') + + // Two peer edits neither compacting task has integrated (id > each task's lastId). + const peerDoc = new Y.Doc() + const peerUpdates: Uint8Array[] = [] + peerDoc.on('update', (u: Uint8Array) => peerUpdates.push(u)) + peerDoc.getText('body').insert(0, 'PEER1') + peerDoc.getText('body').insert(5, 'PEER2') + + const entries = Array.from({ length: 400 }, (_, i) => ({ + id: `${i + 1}-0`, + message: { u: noop }, + })) + entries.push({ id: '401-0', message: { u: Buffer.from(peerUpdates[0]).toString('base64') } }) + entries.push({ id: '402-0', message: { u: Buffer.from(peerUpdates[1]).toString('base64') } }) + state.backing!.streams.set(streamKey, entries) + state.backing!.seq = 402 + + // Two tasks whose local docs lag at DIFFERENT points (400 and 401): both cross the threshold and + // compact concurrently. Each must only trim what its own snapshot subsumes, so the union of both + // snapshots plus the un-integrated peer entries still reconstructs the whole doc. + const a = await newStore() + const b = await newStore() + const docA = new Y.Doc() + Y.applyUpdate(docA, peerUpdates[0]) // A integrated up to 401 + ;(a as any).rooms.set(NAME, { doc: docA, lastId: '401-0', publishes: 0 }) + ;(b as any).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0 }) + await Promise.all([(a as any).maybeCompact(NAME), (b as any).maybeCompact(NAME)]) + + const doc = new Y.Doc() + Y.applyUpdate(doc, (await a.getStreamState(NAME))!) + expect(doc.getText('body').toString()).toBe('PEER1PEER2') + doc.destroy() + }) +}) diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts new file mode 100644 index 00000000000..e4bd138612c --- /dev/null +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -0,0 +1,560 @@ +/** + * Shared, multi-replica Yjs backend for the collaborative file-document relay, over Redis Streams. + * + * The relay keeps an in-memory {@link Y.Doc} per open file (for the sync handshake, awareness, and + * copilot merges), but on a horizontally-scaled deployment (multiple ECS tasks, autoscaling) that + * per-process doc is NOT authoritative on its own: two tasks each seeding the same file from markdown + * would mint independent Yjs client ids and union into duplicated content (split-brain), and a task + * only ever sees the edits of ITS OWN clients. This module makes every task converge on ONE CRDT per + * file by treating a Redis Stream as the shared, ordered, replayable log of Yjs updates — the union of + * a stream's entries IS the document. It is the "shared Yjs backend (y-redis / Hocuspocus)" the relay's + * single-replica model always deferred, built natively for our Socket.IO transport on the Redis the + * Socket.IO adapter already runs. + * + * How it fits the relay's message flow (see `file-doc.ts`): + * - Doc-sync messages no longer ride the Socket.IO Redis ADAPTER cross-pod. Instead each applied + * update is {@link publish}ed to the stream; every task's multiplexed reader + * applies it to its local doc (origin {@link REDIS_ORIGIN}) and fans it out to ITS OWN clients. So a + * client receives each update exactly once, from its own task's local broadcast — no adapter + * amplification, and every task's doc stays converged. (Awareness/presence stay on the adapter: they + * are ephemeral and need no convergence or replay.) + * - {@link attachRoom} does a synchronous catch-up read from the head of the stream when a task first + * opens a file, so a late-joining task (the normal case under autoscaling) loads the current shared + * state before its first client syncs. Catch-up + tail are seamless: the tailer resumes from the + * exact id catch-up stopped at. + * - The one-time seed is written via the atomic {@link seedIfEmpty} (append-iff-empty in one Redis + * step), so exactly one task ever writes the seed cluster-wide (the fix for split-brain) — even if two + * tasks race. {@link shouldSeed} is a Redis lock + empty-stream check layered on top ONLY as an + * efficiency gate (so tasks don't all run the seed fetch); correctness does not depend on it. + * + * When `REDIS_URL` is unset (single-pod dev) the store is DISABLED and every method degrades to the + * relay's original single-replica behavior: seed locally, no stream, no tailer. + * + * @module + */ +import { createLogger } from '@sim/logger' +import { FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { backoffWithJitter } from '@sim/utils/retry' +import { createClient, type RedisClientType } from 'redis' +import * as Y from 'yjs' + +const logger = createLogger('FileDocStore') + +/** + * Compare-and-delete: release a lock ONLY if this task still holds it (its token still the value), so a + * lock that expired and was re-acquired by another task is never stolen by the original holder's release. + */ +const RELEASE_LOCK_SCRIPT = + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end" + +/** + * Atomic seed: append the seed entry ONLY if the stream is still empty, in one Redis-side step. This is + * the real split-brain guard — two tasks racing (even both past an expired seed lock) can never both + * write a seed (each would mint a distinct Yjs client id → duplicated content), because the emptiness + * check and the append happen atomically with no check-then-append window. The seed lock is only an + * efficiency optimization (avoid two seed fetches); correctness does not depend on it staying held. + * Returns 1 if THIS call wrote the seed, 0 if the stream already had content. + */ +const SEED_IF_EMPTY_SCRIPT = + "if redis.call('xlen', KEYS[1]) == 0 then redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]); return 1 else return 0 end" + +/** + * The transaction origin the store stamps on updates it applies from the stream. The relay's + * `doc.on('update')` handler uses it to distinguish an update that ARRIVED from a peer (fan out to + * local clients, but do NOT re-publish — it is already in the stream) from a local edit (fan out AND + * publish). It must be a non-string sentinel so it is never mistaken for a socket id. + */ +export const REDIS_ORIGIN = Symbol('file-doc-redis') + +/** + * Origin for a COMPACTED SNAPSHOT applied from the stream. A snapshot folds the seed + all prior edits + * into one entry, so a fresh task catching up from it would otherwise never see a separate post-seed + * edit frame and would treat the doc as unedited. The relay's edit-tracker uses this origin to mark the + * doc edited (a snapshot only exists after the stream crossed the compaction threshold, i.e. real edits + * happened). Behaves like {@link REDIS_ORIGIN} otherwise (already in the stream — never re-published). + */ +export const REDIS_SNAPSHOT_ORIGIN = Symbol('file-doc-redis-snapshot') + +const STREAM_PREFIX = 'filedoc:stream:' +const SEED_LOCK_PREFIX = 'filedoc:seedlock:' +const COMPACT_LOCK_PREFIX = 'filedoc:compactlock:' +const PERSIST_LOCK_PREFIX = 'filedoc:persistlock:' +const MERGE_LOCK_PREFIX = 'filedoc:mergelock:' + +/** The field each stream entry carries — a base64 Yjs update. */ +const UPDATE_FIELD = 'u' +/** Marks a stream entry as a compaction SNAPSHOT (folds seed + edits), so the tailer applies it with + * {@link REDIS_SNAPSHOT_ORIGIN}. Present only on snapshot entries. */ +const SNAPSHOT_FIELD = 's' + +/** Sentinel token a DISABLED store returns from a lock acquire, so single-replica callers proceed + * without special-casing; {@link FileDocStore.releaseLock} treats it as a no-op. Not a real UUID, so it + * can never collide with a {@link generateId} token. */ +const DISABLED_LOCK_TOKEN = '__disabled__' + +/** How long a blocking multiplexed read waits before re-snapshotting the live room set. Also bounds + * how long a room attached mid-block waits for its first cross-task update (updates are not lost — the + * next read resumes from its last id — only briefly delayed). */ +const READ_BLOCK_MS = 1_000 +/** Idle poll cadence when NO room is open on this task, so a freshly-attached room is picked up fast + * without busy-spinning an empty task. */ +const IDLE_POLL_MS = 250 +/** Max entries drained per stream per read. */ +const READ_COUNT = 200 +/** Compact a stream once it exceeds this many entries (snapshot + trim). */ +const COMPACT_THRESHOLD = 400 +/** Check whether compaction is due only every Nth local publish, to avoid an XLEN per keystroke. */ +const COMPACT_CHECK_EVERY = 64 +/** Compaction critical section (snapshot + xAdd + xTrim) is fast; a generous TTL covers a slow Redis + * round-trip without risking expiry mid-compact. Released via compare-and-delete regardless. */ +const COMPACT_LOCK_TTL_MS = 10_000 +/** Retry a failed stream append this many times before giving up, so a transient Redis blip doesn't + * silently drop an edit from the shared log (which no peer would then ever see). */ +const PUBLISH_MAX_RETRIES = 3 +/** The seed lock spans the app seed fetch (hard-bounded at `seedRequestMs = 8s`) + the atomic seed + * append. It is only an EFFICIENCY optimization — it stops two tasks both running the seed fetch — and is + * sized to comfortably exceed the fetch bound while staying near the client readiness deadline (12s) so a + * dead seeder's lock frees when clients would recover anyway. Double-seed is prevented even if the lock + * expires mid-seed, because the seed is written via the atomic {@link SEED_IF_EMPTY_SCRIPT} + * (append-iff-empty), NOT the lock — correctness never depends on the lock staying held. */ +const SEED_LOCK_TTL_MS = FILE_DOC_TIMEOUTS.seedRequestMs + 4_000 +/** How long a stream survives with no heartbeat — long enough that an occupied-but-idle doc never + * loses its shared state (the heartbeat refreshes it while any task holds the room). */ +const STREAM_TTL_SEC = 600 +/** Refresh every occupied stream's TTL on this cadence, so a live doc's stream never expires. */ +const HEARTBEAT_MS = 60_000 + +const streamKey = (name: string) => `${STREAM_PREFIX}${name}` + +/** + * Decode one stream entry's base64 Yjs update and apply it to `doc`. A malformed entry is logged and + * SKIPPED — never thrown — so one bad frame can neither wedge the tailer nor abort a headless + * stream-fold. Shared by the tailer/catch-up (applies with {@link REDIS_ORIGIN}) and the merge-base + * reconstruction (no origin — a throwaway doc), so the two can never diverge on how an entry is read. + */ +function applyEntryToDoc( + doc: Y.Doc, + id: string, + message: Record, + origin?: unknown +): void { + const encoded = message[UPDATE_FIELD] + if (!encoded) return + try { + Y.applyUpdate(doc, new Uint8Array(Buffer.from(encoded, 'base64')), origin) + } catch (error) { + logger.warn('FileDocStore dropping malformed stream entry', { + id, + error: getErrorMessage(error), + }) + } +} + +/** One locally-open room the store tracks: its doc and the last stream id applied to it. */ +interface StoreRoom { + doc: Y.Doc + /** The id of the last stream entry applied to `doc`; the tailer resumes strictly after it. */ + lastId: string + /** Local publish count, to pace compaction checks. */ + publishes: number +} + +/** + * The Redis-Streams shared Yjs backend. A single instance per process. `enabled` is false when there + * is no `REDIS_URL`, in which case every method is a no-op and the relay runs single-replica. + */ +export class FileDocStore { + readonly enabled: boolean + /** Command connection: XADD / locks / XLEN / XTRIM / EXPIRE. */ + private write: RedisClientType | null = null + /** Dedicated connection for blocking XREAD (a blocking command monopolizes its connection). */ + private read: RedisClientType | null = null + private readonly rooms = new Map() + private running = false + private heartbeat: ReturnType | null = null + + constructor(private readonly redisUrl: string | undefined) { + this.enabled = Boolean(redisUrl) + } + + /** Connect the two Redis clients and start the multiplexed reader + TTL heartbeat. Idempotent. */ + async init(): Promise { + if (!this.enabled || this.running || !this.redisUrl) return + const options = { + url: this.redisUrl, + socket: { + reconnectStrategy: (retries: number) => { + if (retries > 10) return new Error('FileDocStore Redis reconnection failed') + return Math.min(retries * 100, 3000) + }, + }, + } + this.write = createClient(options) + this.read = this.write.duplicate() + this.write.on('error', (err) => logger.error('FileDocStore write client error:', err)) + this.read.on('error', (err) => logger.error('FileDocStore read client error:', err)) + await Promise.all([this.write.connect(), this.read.connect()]) + this.running = true + void this.runReader() + this.heartbeat = setInterval(() => void this.refreshTtls(), HEARTBEAT_MS) + logger.info('FileDocStore ready — shared Yjs backend over Redis Streams enabled') + } + + /** Stop the reader/heartbeat and close both clients. */ + async shutdown(): Promise { + this.running = false + if (this.heartbeat) clearInterval(this.heartbeat) + this.heartbeat = null + await Promise.all([this.write?.quit().catch(() => {}), this.read?.quit().catch(() => {})]) + this.write = null + this.read = null + } + + /** + * Register a locally-opened room and load the shared state into its doc: read the whole stream from + * the head, apply every entry (origin {@link REDIS_ORIGIN}), and remember the last id so the tailer + * resumes exactly after it. A brand-new file has an empty stream and loads nothing (it is seeded + * shortly after, via {@link shouldSeed}). No-op when disabled. + */ + async attachRoom(name: string, doc: Y.Doc): Promise { + if (!this.enabled || !this.write) return + // Register BEFORE the async read so a concurrent publish/tailer for this room can't be missed — + // the tailer resumes from `lastId`, which the catch-up advances. + const room: StoreRoom = { doc, lastId: '0', publishes: 0 } + this.rooms.set(name, room) + try { + const entries = await this.write.xRange(streamKey(name), '-', '+') + for (const entry of entries) { + // The room can be detached + its doc destroyed while catch-up is in flight (a fast open→close); + // stop touching it the moment that happens. + if (this.rooms.get(name) !== room) return + this.applyEntry(room, entry.id, entry.message) + } + await this.write.expire(streamKey(name), STREAM_TTL_SEC) + } catch (error) { + logger.warn(`FileDocStore catch-up failed for ${name}`, { error: getErrorMessage(error) }) + } + } + + /** Deregister a room the relay is destroying, so the tailer stops touching its (about-to-be-destroyed) doc. */ + detachRoom(name: string): void { + this.rooms.delete(name) + } + + /** + * Append a locally-applied update to the shared stream so every task converges, AWAITING the write + * and retrying a transient failure ({@link PUBLISH_MAX_RETRIES}) so a Redis blip can't silently drop + * an edit from the shared log. Only the `xAdd` is retried; the TTL refresh + compaction check are + * post-write best-effort and never re-trigger the append. Throws if the append ultimately fails. + */ + private async appendUpdate(name: string, update: Uint8Array): Promise { + if (!this.write) return + const encoded = Buffer.from(update).toString('base64') + for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { + try { + await this.write.xAdd(streamKey(name), '*', { [UPDATE_FIELD]: encoded }) + break + } catch (error) { + if (attempt === PUBLISH_MAX_RETRIES) { + logger.error(`FileDocStore append failed for ${name}`, { error: getErrorMessage(error) }) + throw error + } + // Snappy backoff — a stream append is a fast op; a transient blip clears in tens of ms. + // `backoffWithJitter` is 1-indexed, so pass the 1-based attempt number. + await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) + } + } + await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + const room = this.rooms.get(name) + if (room && ++room.publishes % COMPACT_CHECK_EVERY === 0) void this.maybeCompact(name) + } + + /** + * Fire-and-forget append for the hot keystroke path (`doc.on('update')`): converges peers without + * blocking the relay. Retries internally; never throws. No-op when disabled. + */ + publish(name: string, update: Uint8Array): void { + if (!this.enabled || !this.write) return + void this.appendUpdate(name, update).catch(() => {}) // already logged inside appendUpdate + } + + /** + * Awaitable append for callers that must know the update is durably in the stream before proceeding + * — the copilot merge, so the cross-task merge lock is not released before the diff is committed + * (else the next task would diff a stale base). Throws on ultimate failure. No-op when disabled. + */ + async publishAndWait(name: string, update: Uint8Array): Promise { + if (!this.enabled || !this.write) return + await this.appendUpdate(name, update) + } + + /** + * Atomically seed the stream iff it is still empty (see {@link SEED_IF_EMPTY_SCRIPT}). This is what + * actually prevents split-brain double-seeding: the emptiness check and the append happen in one + * Redis-side step, so — unlike a separate {@link streamHasContent} fence + {@link publishAndWait} — + * there is no check-then-append window, and two tasks racing (even both past an expired seed lock) can + * never both write a seed. Returns true if THIS call wrote the seed (apply it locally), false if the + * stream was already seeded (the tailer will deliver the peer's seed — do NOT apply a second one). + * Retries a transient Redis error like {@link appendUpdate}; throws if it ultimately fails. Disabled → + * true (single-replica: seed locally, no stream). + */ + async seedIfEmpty(name: string, update: Uint8Array): Promise { + if (!this.enabled || !this.write) return true + const encoded = Buffer.from(update).toString('base64') + for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { + try { + const wrote = await this.write.eval(SEED_IF_EMPTY_SCRIPT, { + keys: [streamKey(name)], + arguments: [UPDATE_FIELD, encoded], + }) + await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + return wrote === 1 + } catch (error) { + if (attempt === PUBLISH_MAX_RETRIES) { + logger.error(`FileDocStore seed failed for ${name}`, { error: getErrorMessage(error) }) + throw error + } + await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) + } + } + return false + } + + /** + * Whether the file's stream already holds content — an EFFICIENCY recheck in {@link shouldSeed} that + * skips the seed fetch when a prior holder already seeded (the split-brain guard itself is the atomic + * {@link SEED_IF_EMPTY_SCRIPT}, not this check). Treats `true` as "already seeded", so it fails CLOSED: + * a Redis `xLen` error returns `true` (cannot confirm empty → skip the redundant fetch; the atomic seed + * would no-op anyway). `false` only when genuinely empty, or when disabled (single-replica). + */ + async streamHasContent(name: string): Promise { + if (!this.enabled || !this.write) return false + try { + return (await this.write.xLen(streamKey(name))) > 0 + } catch (error) { + logger.warn(`FileDocStore streamHasContent failed for ${name}`, { + error: getErrorMessage(error), + }) + return true + } + } + + /** + * Acquire a distributed lock with a unique ownership TOKEN (`SET key NX PX`). Returns the + * token to release with, or `null` if not won. Fails CLOSED (null) on a Redis error — a lock we can't + * prove we hold must not be treated as held. The special sentinel {@link DISABLED_LOCK_TOKEN} lets a + * disabled store return a truthy token so callers proceed single-replica without special-casing. + */ + private async acquireLock(key: string, ttlMs: number): Promise { + if (!this.enabled || !this.write) return DISABLED_LOCK_TOKEN + const token = generateId() + try { + return (await this.write.set(key, token, { NX: true, PX: ttlMs })) === 'OK' ? token : null + } catch (error) { + logger.warn(`FileDocStore lock ${key} failed`, { error: getErrorMessage(error) }) + return null + } + } + + /** Release a lock via compare-and-delete, so it is only dropped if we still hold our token. */ + private async releaseLock(key: string, token: string): Promise { + if (!this.write || token === DISABLED_LOCK_TOKEN) return + await this.write.eval(RELEASE_LOCK_SCRIPT, { keys: [key], arguments: [token] }).catch(() => {}) + } + + /** + * Decide whether THIS task should run the (expensive) seed fetch + write for a file. Returns a lock + * TOKEN when this task wins the seed lock and the stream still looks empty; `null` otherwise. This is an + * EFFICIENCY gate — it stops every task that opens the file at once from each fetching the seed. It does + * NOT by itself guarantee a single seed: exactly-once is enforced by the atomic {@link seedIfEmpty} the + * token-holder then calls (the split-brain guard), so a lock that expires mid-seed cannot cause a + * double-seed. Release the token with {@link releaseSeedLock}. Disabled → always a token (single-replica: + * seed locally). + */ + async shouldSeed(name: string): Promise { + const token = await this.acquireLock(`${SEED_LOCK_PREFIX}${name}`, SEED_LOCK_TTL_MS) + if (!token || token === DISABLED_LOCK_TOKEN) return token + // The lock could be free yet the stream already seeded (a prior holder seeded then its lock + // expired). Re-check so we skip the redundant seed fetch — the atomic seedIfEmpty would no-op anyway, + // but this avoids the wasted app round-trip. + if (await this.streamHasContent(name)) { + await this.releaseSeedLock(name, token) + return null + } + return token + } + + /** + * Build the file's current shared state from the stream, headless (no registered room), for a merge + * that must reach the live doc regardless of which task holds it. Returns the encoded Yjs state, or + * `null` when the stream is empty — i.e. no doc is (or was recently) live, so there is nothing to + * merge into and the caller should fall back to a direct file write. Disabled → always null. + */ + async getStreamState(name: string): Promise { + if (!this.enabled || !this.write) return null + const entries = await this.write.xRange(streamKey(name), '-', '+') + if (entries.length === 0) return null + const doc = new Y.Doc() + try { + for (const entry of entries) applyEntryToDoc(doc, entry.id, entry.message) + return Y.encodeStateAsUpdate(doc) + } finally { + doc.destroy() + } + } + + /** Release the seed lock (compare-and-delete) once the seed has been published or a seed attempt failed. */ + async releaseSeedLock(name: string, token: string): Promise { + await this.releaseLock(`${SEED_LOCK_PREFIX}${name}`, token) + } + + /** + * A best-effort TTL dedup WINDOW (NOT a lock): claim the right to run a debounced persist for the next + * `ttlMs`, so concurrent tasks editing the same file don't each write a redundant blob version. It is + * never released — it simply expires after `ttlMs`, gating the debounced persist to ~once per window + * cluster-wide. Fails OPEN (returns true on a Redis error): a redundant persist is a harmless + * idempotent write, so it must never block a real one. The final last-collaborator flush does NOT gate + * on this — it must always write. + */ + async tryClaimPersistWindow(name: string, ttlMs: number): Promise { + if (!this.enabled || !this.write) return true + try { + const won = await this.write.set(`${PERSIST_LOCK_PREFIX}${name}`, '1', { + NX: true, + PX: ttlMs, + }) + return won === 'OK' + } catch { + return true + } + } + + /** + * Try to claim the cross-task right to merge new content into this file. The relay already serializes + * merges per task; this extends that across tasks so two copilot edits to the same file landing on + * different tasks don't each diff the SAME shared base and publish conflicting full-document rewrites. + * The loser waits and retries so it diffs against the winner's RESULT (correct sequential merge). + * Returns a lock TOKEN (proceed) when disabled or once won; `null` otherwise (fails CLOSED on error, so + * a merge never races when exclusivity can't be proven). Release with {@link releaseMergeSlot}. + */ + async acquireMergeSlot(name: string, ttlMs: number): Promise { + return this.acquireLock(`${MERGE_LOCK_PREFIX}${name}`, ttlMs) + } + + async releaseMergeSlot(name: string, token: string): Promise { + await this.releaseLock(`${MERGE_LOCK_PREFIX}${name}`, token) + } + + private applyEntry(room: StoreRoom, id: string, message: Record): void { + room.lastId = id + // A compaction snapshot folds seed + edits into one frame; stamp it so the relay's edit-tracker + // treats a fresh catch-up from it as edited (a snapshot only exists once real edits accumulated). + const origin = message[SNAPSHOT_FIELD] ? REDIS_SNAPSHOT_ORIGIN : REDIS_ORIGIN + applyEntryToDoc(room.doc, id, message, origin) + } + + /** + * The single multiplexed tail loop: block-read every locally-open room's stream from its last id and + * apply new entries. One blocking connection for the whole process regardless of open-file count. + */ + private async runReader(): Promise { + while (this.running && this.read) { + const snapshot = new Map(this.rooms) + if (snapshot.size === 0) { + await sleep(IDLE_POLL_MS) + continue + } + try { + const res = await this.read.xRead( + [...snapshot].map(([name, room]) => ({ key: streamKey(name), id: room.lastId })), + { BLOCK: READ_BLOCK_MS, COUNT: READ_COUNT } + ) + if (!res) continue + for (const stream of res) { + const name = stream.name.slice(STREAM_PREFIX.length) + const room = this.rooms.get(name) + // Skip if detached mid-read, OR replaced by a close→reopen (a DIFFERENT StoreRoom): applying + // entries read against the OLD room's lastId to the new one could regress its lastId (harmless + // but wasteful re-delivery). The new room caught itself up via xRange already. + if (!room || room !== snapshot.get(name)) continue + for (const entry of stream.messages) this.applyEntry(room, entry.id, entry.message) + } + } catch (error) { + if (!this.running) break + logger.warn('FileDocStore reader error; retrying', { error: getErrorMessage(error) }) + await sleep(500) + } + } + } + + /** + * Snapshot-then-trim compaction: append a full-state snapshot and drop the older deltas it subsumes, + * so the stream stays bounded while a fresh task can still catch up from the head. Lock-guarded so + * only one task compacts a given stream at a time (concurrent snapshot+trim would race). Trims only up + * to what the snapshot provably contains — never un-integrated peer entries (see below). + */ + private async maybeCompact(name: string): Promise { + if (!this.write) return + const room = this.rooms.get(name) + if (!room) return + try { + if ((await this.write.xLen(streamKey(name))) < COMPACT_THRESHOLD) return + const key = `${COMPACT_LOCK_PREFIX}${name}` + const token = await this.acquireLock(key, COMPACT_LOCK_TTL_MS) + if (!token) return + try { + // Capture the snapshot AND the id it covers in one synchronous step (no await between): the + // snapshot is `room.doc`, which holds exactly what this task's tailer has integrated — every + // entry up to `room.lastId`. Entries a peer task published AFTER that (id > lastId) are NOT in + // the snapshot and this task's blocking reader may not have seen them yet, so we must NOT trim + // them — only entries the snapshot provably subsumes (id <= lastId). Trimming to the freshly + // appended snapshot id instead would silently drop those un-integrated peer entries. + const upTo = room.lastId + const snapshot = Buffer.from(Y.encodeStateAsUpdate(room.doc)).toString('base64') + // Mark it a snapshot so a fresh catch-up task treats it as edited content, not a bare seed. + await this.write.xAdd(streamKey(name), '*', { + [UPDATE_FIELD]: snapshot, + [SNAPSHOT_FIELD]: '1', + }) + // MINID keeps entries with id >= upTo: the snapshot, any un-integrated peer entries, and + // `upTo` itself (redundant with the snapshot, harmless); it drops only the folded older deltas. + await this.write.xTrim(streamKey(name), 'MINID', upTo) + } finally { + await this.releaseLock(key, token) + } + } catch (error) { + logger.warn(`FileDocStore compaction failed for ${name}`, { error: getErrorMessage(error) }) + } + } + + private async refreshTtls(): Promise { + if (!this.write) return + for (const name of this.rooms.keys()) { + await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + } + } +} + +let store: FileDocStore | null = null + +/** + * Initialize the process-wide store from the realtime server bootstrap (alongside the socket adapter). + * Authoritative: if a disabled placeholder was lazily created by an early {@link getFileDocStore} call, + * this REPLACES it with the real, connected store — so the bootstrap can never silently no-op. A second + * call once already initialized is a no-op. + */ +export async function initFileDocStore(redisUrl: string | undefined): Promise { + if (store?.enabled) return store + store = new FileDocStore(redisUrl) + await store.init() + return store +} + +/** The process-wide store. Returns a disabled instance if init was never called (e.g. in unit tests). */ +export function getFileDocStore(): FileDocStore { + if (!store) store = new FileDocStore(undefined) + return store +} diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts new file mode 100644 index 00000000000..83c4c0e144a --- /dev/null +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -0,0 +1,931 @@ +/** + * @vitest-environment node + */ +import { + FILE_DOC_EVENTS, + FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SEED, +} from '@sim/realtime-protocol/file-doc' +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as awarenessProtocol from 'y-protocols/awareness' +import * as syncProtocol from 'y-protocols/sync' +import * as Y from 'yjs' +import type { IRoomManager } from '@/rooms' + +const { mockAuthorizeRoom, mockFetchFileDocSeed, mockFetchFileDocMerge, mockFetchFileDocPersist } = + vi.hoisted(() => ({ + mockAuthorizeRoom: vi.fn(), + mockFetchFileDocSeed: vi.fn(), + mockFetchFileDocMerge: vi.fn(), + mockFetchFileDocPersist: vi.fn(), + })) + +vi.mock('@sim/platform-authz/rooms', () => ({ + authorizeRoom: mockAuthorizeRoom, +})) + +vi.mock('@/handlers/file-doc-app', () => ({ + fetchFileDocSeed: mockFetchFileDocSeed, + fetchFileDocMerge: mockFetchFileDocMerge, + fetchFileDocPersist: mockFetchFileDocPersist, +})) + +import { + applyMarkdownToLiveFileDoc, + cleanupFileDocForSocket, + flushAllFileDocRooms, + setupWorkspaceFileDocHandlers, +} from '@/handlers/file-doc' + +type Handler = (payload?: unknown) => Promise | void + +const ROOM_NAME = 'workspace-file-doc:file-1' + +interface SentMessage { + target: string + except?: string + event: string + payload: unknown +} + +/** An `io` mock that records every server-originated emit with its target/except. */ +function createIo() { + const sent: SentMessage[] = [] + /** Records `io.in(socketId).socketsLeave(room)` — a socket forced out of a room from outside. */ + const left: { socketId: string; room: string }[] = [] + const to = vi.fn((target: string) => ({ + except: (exclude: string) => ({ + emit: (event: string, payload: unknown) => + sent.push({ target, except: exclude, event, payload }), + }), + emit: (event: string, payload: unknown) => sent.push({ target, event, payload }), + })) + const inFn = vi.fn((socketId: string) => ({ + socketsLeave: (room: string) => { + left.push({ socketId, room }) + }, + })) + // Doc-sync frames fan out via `io.local.to(...)` (cross-task delivery rides the Redis stream, not the + // adapter). With the store disabled in tests, `local` is the whole room — mirror `to` so those emits + // are recorded identically. Awareness/presence still use `io.to(...)`. + return { io: { to, in: inFn, local: { to } } as unknown as IRoomManager['io'], sent, left } +} + +/** Every socket id a test created, so `afterEach` can drop their rooms without a + * hardcoded list drifting out of sync with the tests. */ +const createdSocketIds = new Set() + +function createSocket(id: string, overrides?: Record) { + createdSocketIds.add(id) + const handlers: Record = {} + const socket = { + id, + userId: 'user-1', + userName: 'Test User', + // Set so the server's roster resolves the avatar from the socket (never the DB). + userImage: 'avatar.png', + disconnected: false, + on: vi.fn((event: string, handler: Handler) => { + handlers[event] = handler + }), + emit: vi.fn(), + join: vi.fn(), + leave: vi.fn(), + ...overrides, + } + return { handlers, socket } +} + +function createRoomManager( + io: IRoomManager['io'], + overrides?: Partial +): IRoomManager { + return { + isReady: vi.fn().mockReturnValue(true), + io, + ...overrides, + } as unknown as IRoomManager +} + +function setup(id: string, io: IRoomManager['io'], socketOverrides?: Record) { + const { socket, handlers } = createSocket(id, socketOverrides) + setupWorkspaceFileDocHandlers( + socket as unknown as Parameters[0], + createRoomManager(io) + ) + return { socket, handlers } +} + +const FILE_DOC_FIELD = 'default' + +/** Let a fire-and-forget `void ensureServerSeed(...)` chain settle (mock resolves synchronously). */ +async function flushMicrotasks(): Promise { + // Enough to drain the fire-and-forget seed chain (shouldSeed → fetch → fence → publish → apply). + for (let i = 0; i < 8; i++) await Promise.resolve() +} + +/** + * An encoded Yjs update shaped like the server seed builder's output: some content in the shared + * `default` type plus the {@link FILE_DOC_SEED} flag, so applying it marks the doc seeded. + */ +function encodedSeedUpdate(content: string): Uint8Array { + const doc = new Y.Doc() + doc.getText(FILE_DOC_FIELD).insert(0, content) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + return Y.encodeStateAsUpdate(doc) +} + +/** Apply a server sync reply frame (`[SYNC tag][sync message]`) into a fresh client doc. */ +function applySyncReply(frameBytes: Uint8Array, doc: Y.Doc): void { + const decoder = decoding.createDecoder(frameBytes) + decoding.readVarUint(decoder) // skip the message-type tag + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), doc, null) +} + +/** Frame a Yjs message with its type tag, exactly as the client provider would. */ +function frame(type: number, write: (encoder: encoding.Encoder) => void): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, type) + write(encoder) + return encoding.toUint8Array(encoder) +} + +/** Build a real awareness frame carrying a single client's state. */ +function awarenessFrame(clientId: number, name: string): { frame: Uint8Array; clientId: number } { + const doc = new Y.Doc() + // Force a specific clientID so the test can bind/spoof deliberately. + doc.clientID = clientId + const awareness = new awarenessProtocol.Awareness(doc) + awareness.setLocalStateField('user', { name }) + const update = awarenessProtocol.encodeAwarenessUpdate(awareness, [clientId]) + return { + frame: frame(FILE_DOC_MESSAGE_TYPE.AWARENESS, (e) => encoding.writeVarUint8Array(e, update)), + clientId, + } +} + +function joinSuccessFileId(socket: { emit: ReturnType }) { + const calls = socket.emit.mock.calls.filter( + (call: unknown[]) => call[0] === FILE_DOC_EVENTS.JOIN_SUCCESS + ) + const last = calls[calls.length - 1] + return (last?.[1] as { fileId: string } | undefined)?.fileId +} + +describe('setupWorkspaceFileDocHandlers', () => { + beforeEach(() => { + vi.clearAllMocks() + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'write', + }) + // Default: the server seed builder returns no content (empty file). Tests that + // exercise seeding override this per-case with an encoded Yjs update. + mockFetchFileDocSeed.mockResolvedValue(null) + // Default: the merge builder returns a valid no-op (empty-doc) update. Tests exercising copilot + // merges override it. + mockFetchFileDocMerge.mockResolvedValue(Y.encodeStateAsUpdate(new Y.Doc())) + }) + + afterEach(() => { + // The room store is module-global; drop every room the test's sockets opened. + const { io } = createIo() + // Simulate a full disconnect between tests (`endOfLife`) so the module-global join-generation + // map is cleared and never bleeds a counter into the next test. + for (const id of createdSocketIds) cleanupFileDocForSocket(id, io, true) + createdSocketIds.clear() + }) + + it('rejects join when the socket is not authenticated', async () => { + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io, { userId: undefined, userName: undefined }) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'AUTHENTICATION_REQUIRED', retryable: false }) + ) + }) + + it('rejects join with a retryable error when realtime is unavailable', async () => { + const { io } = createIo() + const { socket, handlers } = createSocket('socket-1') + setupWorkspaceFileDocHandlers( + socket as unknown as Parameters[0], + createRoomManager(io, { isReady: vi.fn().mockReturnValue(false) }) + ) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) + ) + }) + + it('rejects a payload missing the file id or client id before authorizing', async () => { + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: '', clientId: 1 }) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'INVALID_PAYLOAD', retryable: false }) + ) + expect(mockAuthorizeRoom).not.toHaveBeenCalled() + }) + + it('requires write permission and reports 404 as NOT_FOUND', async () => { + mockAuthorizeRoom.mockResolvedValue({ allowed: false, status: 404, workspacePermission: null }) + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(mockAuthorizeRoom).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'write', + room: { type: 'workspace-file-doc', id: 'file-1' }, + }) + ) + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'NOT_FOUND', retryable: false }) + ) + }) + + it('does NOT persist a seeded-but-unedited doc on last disconnect (no clobber of a concurrent write)', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() // let the seed apply + + // Last collaborator leaves without ever editing — projecting this seed back over the file could + // clobber a concurrent copilot write, so the final flush must NOT persist. + cleanupFileDocForSocket('socket-1', io, true) + await flushMicrotasks() + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + }) + + it('persists on last disconnect once a genuine user edit has landed', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + + // A real user edit (socket-origin sync update) marks the doc dirty. + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'user typed this') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(edit)) + ) + ) + await flushMicrotasks() + + cleanupFileDocForSocket('socket-1', io, true) + await flushMicrotasks() + expect(mockFetchFileDocPersist).toHaveBeenCalled() + }) + + it('flushAllFileDocRooms persists open EDITED rooms (graceful shutdown), skips unedited', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + + // Seed-only room: a graceful-shutdown flush must NOT persist it. + mockFetchFileDocPersist.mockClear() + await flushAllFileDocRooms() + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + + // After a real user edit, the same flush persists (edits would otherwise be lost on deploy). + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'typed') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(edit)) + ) + ) + await flushMicrotasks() + mockFetchFileDocPersist.mockClear() + await flushAllFileDocRooms() + expect(mockFetchFileDocPersist).toHaveBeenCalled() + }) + + it('joins the room, sends sync step 1, and seeds the document from the server', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# From server')) + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(socket.join).toHaveBeenCalledWith(ROOM_NAME) + expect(joinSuccessFileId(socket)).toBe('file-1') + + // A binary sync-step-1 message (type tag 0) is sent to kick off the handshake. + const syncMessage = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + expect((syncMessage?.[1] as Uint8Array)[0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + + // The server seeds authoritatively from the file's stored markdown, keyed by (workspaceId, fileId). + await flushMicrotasks() + expect(mockFetchFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1') + + // The seeded state is served to a client that syncs: request step 2 and decode it. + socket.emit.mockClear() + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + const reply = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# From server') + }) + + it('seeds the document only once from the server across concurrent joiners of the same file', async () => { + // Keep the first seed fetch IN FLIGHT so the doc is still unseeded when the second socket joins: + // that forces the dedup onto `serverSeedStarted` (the in-flight guard) rather than `isDocSeeded`. + let resolveSeed: (v: Uint8Array | null) => void = () => {} + mockFetchFileDocSeed.mockReturnValueOnce(new Promise((resolve) => (resolveSeed = resolve))) + const { io } = createIo() + const a = setup('socket-a', io) + const b = setup('socket-b', io) + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + // Second join happened with the fetch still pending; only after this does the seed land. + expect(mockFetchFileDocSeed).toHaveBeenCalledTimes(1) + resolveSeed(encodedSeedUpdate('# From server')) + await flushMicrotasks() + expect(mockFetchFileDocSeed).toHaveBeenCalledTimes(1) + }) + + it('marks an empty/absent-file doc seeded so clients still reach readiness', async () => { + // A genuinely absent file yields a null seed (a read error would throw, not return null). The + // relay must still flip `initialContentLoaded` so the client's `synced && seeded` gate opens. + mockFetchFileDocSeed.mockResolvedValue(null) + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + + socket.emit.mockClear() + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + const reply = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('') + }) + + it('makes one seed attempt and releases the guard on failure so a later join retries', async () => { + mockFetchFileDocSeed + .mockRejectedValueOnce(new Error('transport blip')) + .mockResolvedValueOnce(encodedSeedUpdate('# Recovered')) + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + + // First join: a single attempt that fails — no in-room retry loop. + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + expect(mockFetchFileDocSeed).toHaveBeenCalledTimes(1) + + // The guard was released, so a subsequent join re-attempts and this time the seed lands. + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + await flushMicrotasks() + expect(mockFetchFileDocSeed).toHaveBeenCalledTimes(2) + + socket.emit.mockClear() + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + const reply = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# Recovered') + }) + + it('does not seed a room that was dropped while the seed fetch was in flight', async () => { + let resolveSeed: (v: Uint8Array | null) => void = () => {} + mockFetchFileDocSeed.mockReturnValueOnce(new Promise((resolve) => (resolveSeed = resolve))) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + // The only owner leaves → the room (and its doc) is destroyed while the fetch is still pending. + cleanupFileDocForSocket('socket-1', io, true) + // Resolving now must not touch the destroyed doc or throw (liveness re-check after the await). + resolveSeed(encodedSeedUpdate('# Too late')) + await expect(flushMicrotasks()).resolves.toBeUndefined() + }) + + it('still seeds when content was synced into the doc before the seed returned', async () => { + // Defensive: the guard is `isDocSeeded`, NOT doc-emptiness. In practice a fresh client never + // writes ahead of the seed (@tiptap/y-tiptap suppresses the empty-paragraph placeholder and real + // edits are readiness-gated), but even if some update landed content in the doc before the seed + // fetch resolved, the seed must still apply and set the flag — or the client's + // `synced && initialContentLoaded` gate would never open. + let resolveSeed: (v: Uint8Array | null) => void = () => {} + mockFetchFileDocSeed.mockReturnValueOnce(new Promise((resolve) => (resolveSeed = resolve))) + const { io } = createIo() + const { socket, handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + // The client syncs a placeholder update — content in the doc, but no seed flag. + const placeholder = new Y.Doc() + placeholder.getText(FILE_DOC_FIELD).insert(0, 'x') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(placeholder)) + ) + ) + resolveSeed(encodedSeedUpdate('# Seeded')) + await flushMicrotasks() + + socket.emit.mockClear() + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + const reply = socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toContain('# Seeded') + }) + + it('merges a copilot edit into a seeded live room and relays it to editors', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# Original')) + const { io, sent } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() // seed lands → room is seeded/live + + // The app returns a diff (here, an update introducing text) for the relay to apply. + const diff = new Y.Doc() + diff.getText(FILE_DOC_FIELD).insert(0, 'copilot content') + mockFetchFileDocMerge.mockResolvedValue(Y.encodeStateAsUpdate(diff)) + sent.length = 0 + + const result = await applyMarkdownToLiveFileDoc('file-1', '# Rewritten by copilot') + + expect(result).toBe('applied') + expect(mockFetchFileDocMerge).toHaveBeenCalledWith( + 'file-1', + expect.any(Uint8Array), + '# Rewritten by copilot' + ) + // Applying the diff fires doc.on('update') → the merge is broadcast to the whole room. + expect(sent.some((m) => m.event === FILE_DOC_EVENTS.MESSAGE && m.target === ROOM_NAME)).toBe( + true + ) + }) + + it('reports no-live-room (and does not call the app) when the file has no seeded room', async () => { + const result = await applyMarkdownToLiveFileDoc('file-1', '# anything') + expect(result).toBe('no-live-room') + expect(mockFetchFileDocMerge).not.toHaveBeenCalled() + }) + + it('serializes concurrent merges for the same file (second waits for the first)', async () => { + mockFetchFileDocSeed.mockResolvedValue(encodedSeedUpdate('# Original')) + const { io } = createIo() + const { handlers } = setup('socket-1', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + + // First merge is left in flight; the second must not start its own fetch until the first finishes. + const noOpUpdate = Y.encodeStateAsUpdate(new Y.Doc()) + let resolveFirst: (v: Uint8Array) => void = () => {} + mockFetchFileDocMerge + .mockReturnValueOnce(new Promise((resolve) => (resolveFirst = resolve))) + .mockResolvedValueOnce(noOpUpdate) + + const first = applyMarkdownToLiveFileDoc('file-1', '# One') + const second = applyMarkdownToLiveFileDoc('file-1', '# Two') + await flushMicrotasks() + expect(mockFetchFileDocMerge).toHaveBeenCalledTimes(1) // second is queued behind the first + + resolveFirst(noOpUpdate) + await first + await second + // Only after the first resolved did the second run — and it snapshotted the post-first state. + expect(mockFetchFileDocMerge).toHaveBeenCalledTimes(2) + }) + + it('relays a document update to the rest of the room, excluding the sender', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io) + const b = setup('socket-b', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + sent.length = 0 + + const clientDoc = new Y.Doc() + clientDoc.getText('default').insert(0, 'hello') + const update = Y.encodeStateAsUpdate(clientDoc) + a.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeUpdate(e, update)) + ) + + const relayed = sent.find((m) => m.event === FILE_DOC_EVENTS.MESSAGE) + expect(relayed?.target).toBe(ROOM_NAME) + expect(relayed?.except).toBe('socket-a') + expect((relayed?.payload as Uint8Array)[0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + }) + + it('relays an owned awareness update to the room, excluding the sender', async () => { + const { io, sent } = createIo() + const { frame: awFrame, clientId } = awarenessFrame(4242, 'Ada') + const a = setup('socket-a', io) + const b = setup('socket-b', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + sent.length = 0 + + a.handlers[FILE_DOC_EVENTS.MESSAGE](awFrame) + + const relayed = sent.find( + (m) => + m.event === FILE_DOC_EVENTS.MESSAGE && + (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS + ) + expect(relayed?.except).toBe('socket-a') + }) + + it('drops an awareness frame that spoofs another client id', async () => { + const { io, sent } = createIo() + // socket-a binds client id 100 at join, but sends awareness for client 999. + const { frame: spoof } = awarenessFrame(999, 'Mallory') + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 100 }) + sent.length = 0 + + a.handlers[FILE_DOC_EVENTS.MESSAGE](spoof) + + const relayed = sent.find( + (m) => + m.event === FILE_DOC_EVENTS.MESSAGE && + (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS + ) + expect(relayed).toBeUndefined() + }) + + it("rejects a DIFFERENT user binding a peer's client id (spoof)", async () => { + const { io } = createIo() + const a = setup('socket-a', io) + const b = setup('socket-b', io, { userId: 'attacker' }) + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + + expect(b.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'CLIENT_ID_IN_USE' }) + ) + expect(b.socket.join).not.toHaveBeenCalled() + }) + + it('reclaims a client id for the SAME user reconnecting (reused Yjs client id)', async () => { + const { io } = createIo() + // The same user's dropped socket still owns client id 7 (its disconnect + // cleanup has not run yet) when it reconnects on a new socket reusing id 7. + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + const b = setup('socket-b', io) // same default userId 'user-1' + + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + + expect(joinSuccessFileId(b.socket)).toBe('file-1') + expect(b.socket.join).toHaveBeenCalledWith(ROOM_NAME) + }) + + it('clears a departed caret when a socket rejoins the room with a new client id', async () => { + const { io, sent } = createIo() + const { frame: awFrame } = awarenessFrame(500, 'A') + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 500 }) + a.handlers[FILE_DOC_EVENTS.MESSAGE](awFrame) + sent.length = 0 + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 501 }) + + // The old client (500) caret removal is broadcast to the room. + const removal = sent.find( + (m) => + m.event === FILE_DOC_EVENTS.MESSAGE && + (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS + ) + expect(removal).toBeDefined() + }) + + it('preserves the existing caret when a rebind to a foreign client id is rejected', async () => { + const { io, sent } = createIo() + const { frame: awFrame } = awarenessFrame(10, 'A') + const a = setup('socket-a', io) + const b = setup('socket-b', io, { userId: 'user-b' }) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 10 }) + a.handlers[FILE_DOC_EVENTS.MESSAGE](awFrame) // a publishes its caret for client 10 + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 20 }) + sent.length = 0 + + // socket-a (owns 10) tries to rebind to 20, owned by a different user → reject. + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 20 }) + + expect(a.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'CLIENT_ID_IN_USE' }) + ) + // The rejected rebind must NOT have removed a's existing caret (no awareness + // removal broadcast fires). + const removal = sent.find( + (m) => + m.event === FILE_DOC_EVENTS.MESSAGE && + (m.payload as Uint8Array)[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS + ) + expect(removal).toBeUndefined() + }) + + it('drops a malformed frame without throwing', async () => { + const { io } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(() => + a.handlers[FILE_DOC_EVENTS.MESSAGE](new Uint8Array([255, 254, 253, 200])) + ).not.toThrow() + }) + + it('drops the document when the last editor leaves, re-seeding a fresh joiner from the server', async () => { + const { io } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await flushMicrotasks() + cleanupFileDocForSocket('socket-a', io) + + // The room was dropped with its last owner: a fresh joiner starts a new document, so the server + // is asked to seed it again (a stale in-memory doc is never reused across an empty gap). + mockFetchFileDocSeed.mockClear() + const b = setup('socket-b', io) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + await flushMicrotasks() + + expect(b.socket.join).toHaveBeenCalledWith(ROOM_NAME) + expect(mockFetchFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1') + }) + + it('aborts a join superseded by a newer join during authorization (no cross-binding)', async () => { + const { io } = createIo() + let resolveFirst: (v: unknown) => void = () => {} + mockAuthorizeRoom + .mockReturnValueOnce(new Promise((resolve) => (resolveFirst = resolve))) + .mockResolvedValueOnce({ allowed: true, status: 200, workspacePermission: 'write' }) + const s = setup('socket-a', io) + + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) + resolveFirst({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + // The socket is bound only to the newer file, never cross-bound to file-1. + expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') + expect(s.socket.join).not.toHaveBeenCalledWith('workspace-file-doc:file-1') + }) + + it('does not register a socket that disconnected during authorization', async () => { + const { io, sent } = createIo() + let resolveAuth: (v: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValueOnce(new Promise((resolve) => (resolveAuth = resolve))) + const s = setup('socket-a', io) + + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + s.socket.disconnected = true + cleanupFileDocForSocket('socket-a', io, true) // disconnect cleanup — no-op, nothing registered yet + resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + expect(s.socket.join).not.toHaveBeenCalled() + // No room leaked: a fresh joiner starts a new document and joins cleanly. + const b = setup('socket-b', io) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(b.socket.join).toHaveBeenCalledWith(ROOM_NAME) + expect(joinSuccessFileId(b.socket)).toBe('file-1') + }) + + it('does not abort an in-flight join when a leave for a different file arrives', async () => { + const { io } = createIo() + let resolveAuth: (v: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValueOnce(new Promise((resolve) => (resolveAuth = resolve))) + const s = setup('socket-a', io) + + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) + // A stale leave for a DIFFERENT file must not invalidate the in-flight join. + s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + expect(joinSuccessFileId(s.socket)).toBe('file-2') + expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') + }) + + it('does not reset the join generation on a leave, so an in-flight join still binds', async () => { + const { io } = createIo() + const s = setup('socket-a', io) + + // file-1 join completes; the socket is registered in file-1. + await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + // file-2 join goes in-flight (authorize deferred). + let resolveAuth: (v: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValueOnce(new Promise((resolve) => (resolveAuth = resolve))) + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) + + // A deferred leave for the prior file-1 lands while file-2's join awaits authorization. Its + // cleanup must NOT reset the monotonic join generation, or file-2's guard would see an emptied + // map (`undefined !== generation`) and abort the join the client actually wants. + s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + + resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + expect(joinSuccessFileId(s.socket)).toBe('file-2') + expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') + }) + + it('cancels an in-flight join when the client leaves that same file (no ghost owner)', async () => { + const { io, sent } = createIo() + let resolveAuth: (v: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValueOnce(new Promise((resolve) => (resolveAuth = resolve))) + const s = setup('socket-a', io) + + // Join file-1 is awaiting authorization when the client leaves file-1 (fast open→close). + const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + await pending + + // The stale join must not register: no success, no room join, and no presence broadcast that + // would leave a ghost collaborator until disconnect. + expect(s.socket.join).not.toHaveBeenCalled() + expect(joinSuccessFileId(s.socket)).toBeUndefined() + expect(sent.some((m) => m.event === FILE_DOC_EVENTS.PRESENCE)).toBe(false) + }) + + it('scopes LEAVE to the named file (a leave for a different file is a no-op)', async () => { + const { io } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + a.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'other' }) + expect(a.socket.leave).not.toHaveBeenCalledWith(ROOM_NAME) + + a.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + expect(a.socket.leave).toHaveBeenCalledWith(ROOM_NAME) + }) + + it('replies with a sync step 2 to the sender on a sync step 1 frame', async () => { + const { io } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + // Give the server doc some content so a step-1 request yields a non-empty step 2. + const seeded = new Y.Doc() + seeded.getText('default').insert(0, 'hi') + a.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(seeded)) + ) + ) + a.socket.emit.mockClear() + + a.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) + ) + + const reply = a.socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + expect((reply?.[1] as Uint8Array)[0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + }) + + it('leaves the previous document when a socket switches files', async () => { + const { io, sent } = createIo() + const s = setup('socket-a', io) + await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) + + expect(s.socket.leave).toHaveBeenCalledWith('workspace-file-doc:file-1') + expect(s.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-2') + + // file-1's room was dropped (socket-a was its only owner): a fresh joiner of file-1 starts a new + // document, so the server is asked to seed it again. + await flushMicrotasks() + mockFetchFileDocSeed.mockClear() + const b = setup('socket-b', io) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + await flushMicrotasks() + expect(b.socket.join).toHaveBeenCalledWith('workspace-file-doc:file-1') + expect(mockFetchFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1') + }) + + it('fully evicts a reclaimed prior socket so it can no longer write to the doc', async () => { + const { io, sent, left } = createIo() + const a = setup('socket-a', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) + const b = setup('socket-b', io) // same default user-1 + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 7 }) // reclaims client id 7 + + // The stale prior socket is forced out of the Socket.IO room... + expect(left).toContainEqual({ socketId: 'socket-a', room: ROOM_NAME }) + + // ...and its room mapping is cleared, so a later document (SYNC) frame from it is dropped + // (handleMessage's SYNC path gates on socketToRoomName): nothing is applied or relayed. + sent.length = 0 + const doc = new Y.Doc() + doc.getText('t').insert(0, 'x') + const updateFrame = frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => + syncProtocol.writeUpdate(e, Y.encodeStateAsUpdate(doc)) + ) + a.handlers[FILE_DOC_EVENTS.MESSAGE](updateFrame) + expect(sent.some((m) => m.event === FILE_DOC_EVENTS.MESSAGE)).toBe(false) + }) + + it('does not drop the current document when a switch is rejected for a foreign client id', async () => { + const { io } = createIo() + const a = setup('socket-a', io) // user-1 + const other = setup('socket-c', io, { userId: 'user-b' }) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 10 }) // a owns 10 in file-1 + await other.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 99 }) // user-b owns 99 in file-2 + a.socket.leave.mockClear() + a.socket.join.mockClear() + + // a tries to switch to file-2 but requests client id 99, owned by a DIFFERENT user → reject. + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 99 }) + + expect(a.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'CLIENT_ID_IN_USE' }) + ) + // The rejected switch must leave file-1 intact — a is not torn out of its current document. + expect(a.socket.leave).not.toHaveBeenCalledWith('workspace-file-doc:file-1') + expect(a.socket.join).not.toHaveBeenCalledWith('workspace-file-doc:file-2') + }) + + it('broadcasts a server-authenticated presence roster on join, one entry per session', async () => { + const { io, sent } = createIo() + const a = setup('socket-a', io, { userId: 'user-a', userName: 'Ada', userImage: 'ada.png' }) + const b = setup('socket-b', io, { userId: 'user-b', userName: 'Bob', userImage: 'bob.png' }) + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + const roster = sent.filter((m) => m.event === FILE_DOC_EVENTS.PRESENCE).at(-1)?.payload as { + fileId: string + users: Array<{ socketId: string; userId: string; userName: string; avatarUrl: string | null }> + } + expect(roster.fileId).toBe('file-1') + // Identity is each socket's authenticated session — not any client-supplied value. + expect([...roster.users].sort((x, y) => x.userId.localeCompare(y.userId))).toEqual([ + { socketId: 'socket-a', userId: 'user-a', userName: 'Ada', avatarUrl: 'ada.png' }, + { socketId: 'socket-b', userId: 'user-b', userName: 'Bob', avatarUrl: 'bob.png' }, + ]) + }) + + it('keeps a per-session entry for two sockets of the SAME user (no server-side user dedup)', async () => { + const { io, sent } = createIo() + // Two tabs of one account: the client self-excludes its own socket, so the roster must carry + // BOTH sessions or a client could never see the other tab as present. + const a = setup('socket-a', io, { userId: 'user-a', userName: 'Ada', userImage: 'ada.png' }) + const b = setup('socket-b', io, { userId: 'user-a', userName: 'Ada', userImage: 'ada.png' }) + + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await b.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + const roster = sent.filter((m) => m.event === FILE_DOC_EVENTS.PRESENCE).at(-1)?.payload as { + users: Array<{ socketId: string; userId: string }> + } + expect([...roster.users].map((u) => u.socketId).sort()).toEqual(['socket-a', 'socket-b']) + }) +}) diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts new file mode 100644 index 00000000000..509d312c338 --- /dev/null +++ b/apps/realtime/src/handlers/file-doc.ts @@ -0,0 +1,909 @@ +/** + * Collaborative document editing (live carets + text selection) for a single + * file's rich-text editor. This is the standard Yjs "websocket server" relay — + * an authoritative in-memory {@link Y.Doc} + {@link awarenessProtocol.Awareness} + * per file — carried over the shared, already-authenticated Socket.IO connection + * and the room abstraction, rather than a separate ws server. Clients speak the + * `y-protocols` sync + awareness protocols; the server applies and relays them. + * + * Multi-replica safe. The in-memory {@link Y.Doc} is NOT authoritative on its own — every task + * converges on one CRDT per file through the shared Redis-Streams backend in {@link file-doc-store}: + * each applied update is published to the file's stream and every task's tailer applies it to its own + * doc and fans it out to its own clients, so two tasks can never split-brain. Consequently doc-sync + * messages are broadcast LOCALLY ({@link io.local}) and cross-task delivery rides the stream — NOT the + * Socket.IO adapter (that would double-deliver). Awareness/presence stay on the adapter: they are + * ephemeral and need neither convergence nor replay. When `REDIS_URL` is unset the store is disabled + * and this falls back to the original single-replica behavior (local doc, local seed, local fan-out). + * + * Durability: the live doc is projected back to the file's markdown server-side — debounced while it is + * edited and flushed when the last collaborator leaves — via the app's `/persist` endpoint (the app + * owns the conversion engine). This replaces the editor's client autosave, closing the copilot + * clobber-window, and the Redis stream is the crash buffer between flushes. The markdown file remains + * the long-term source of truth; the Redis stream is ephemeral (TTL'd, heartbeat-refreshed while live). + * + * @module + */ +import { createLogger } from '@sim/logger' +import { + FILE_DOC_EVENTS, + FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SEED, + FILE_DOC_TIMEOUTS, + type FileDocPresenceUser, + type JoinFileDocPayload, + type LeaveFileDocPayload, + toFileDocBytes, +} from '@sim/realtime-protocol/file-doc' +import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import type { Server } from 'socket.io' +import * as awarenessProtocol from 'y-protocols/awareness' +import * as syncProtocol from 'y-protocols/sync' +import * as Y from 'yjs' +import { resolveAvatarUrl } from '@/handlers/avatar' +import { fetchFileDocMerge, fetchFileDocPersist, fetchFileDocSeed } from '@/handlers/file-doc-app' +import { getFileDocStore, REDIS_ORIGIN, REDIS_SNAPSHOT_ORIGIN } from '@/handlers/file-doc-store' +import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' +import type { AuthenticatedSocket } from '@/middleware/auth' +import type { IRoomManager } from '@/rooms' + +const logger = createLogger('FileDocHandlers') + +/** + * The transaction origin the server stamps on a SEED apply, so `doc.on('update')` can broadcast + share + * it (peers still need the seed) but skip the debounced persist — the seed IS the file's current + * content, so writing it straight back would only churn a redundant blob version. + */ +const SEED_ORIGIN = Symbol('file-doc-seed') + +/** Debounce window for the server-side project-to-markdown persist while a doc is actively edited. */ +const PERSIST_DEBOUNCE_MS = 5_000 +/** Max-wait cap on the persist debounce: a CONTINUOUS edit burst keeps resetting the 5s debounce and + * would otherwise never persist until an idle pause, so force a flush at least this often — bounding how + * many edits are unpersisted (in the stream only) if the task dies mid-burst. */ +const PERSIST_MAX_WAIT_MS = 20_000 + +/** Cross-task merge lock. The TTL must exceed the whole critical section it guards — stream fold + + * `fetchFileDocMerge` (bounded at `mergeRequestMs`) + the awaited publish — so the lock never expires + * mid-merge and lets a second task race the same base; hence `mergeRequestMs` plus generous headroom. + * The waiter retries at {@link MERGE_LOCK_RETRY_MS} for LONGER than the TTL, so a live holder always + * releases (or a dead holder's lock expires) within the window. Release is compare-and-delete on an + * ownership token, so even a pathological over-TTL hold never deletes another task's re-acquired lock. */ +const MERGE_LOCK_TTL_MS = FILE_DOC_TIMEOUTS.mergeRequestMs + 5_000 +const MERGE_LOCK_RETRY_MS = 200 +const MERGE_LOCK_RETRIES = Math.ceil( + (MERGE_LOCK_TTL_MS + FILE_DOC_TIMEOUTS.mergeRequestMs) / MERGE_LOCK_RETRY_MS +) + +/** A socket's presence ownership within a room. */ +interface FileDocOwner { + /** + * The awareness clientID the socket declared at join. It owns exactly this one + * and may only publish/remove awareness for it, so an authenticated peer cannot + * forge or clear another collaborator's presence. + */ + clientId: number + /** The owning user — used to tell a reconnect (same user reusing its Yjs client + * id) from a spoof (a different user binding a peer's id). */ + userId: string + /** Server-authenticated display identity for the presence roster (from the socket's + * session, never the client-set awareness — so a peer cannot spoof it). */ + userName: string + avatarUrl: string | null +} + +interface FileDocRoom { + /** The `workspace_files.id` this room edits. */ + fileId: string + doc: Y.Doc + awareness: awarenessProtocol.Awareness + /** socketId → its presence ownership. */ + owners: Map + /** True once the server-side seed fetch has started, so concurrent joins don't each fetch. + * Reset on a fetch FAILURE so a later join can retry (a genuinely empty file stays empty). */ + serverSeedStarted: boolean + /** The workspace this file belongs to, captured at join — needed to persist back to markdown. */ + workspaceId: string | null + /** The last collaborator to edit here, for persist attribution (blob metadata) only. */ + lastEditorUserId: string | null + /** + * True once a genuine edit (a user edit — local OR a peer's, relayed via the tailer) has been applied + * here AFTER seeding. Persistence is gated on it so a doc that was only seeded is NEVER projected back + * over the file: the seed is captured from possibly-stale markdown and must not clobber a concurrent + * external write, whereas real edits are not otherwise durable and must be persisted by whichever task + * is last to leave — even one that only tailed the edits. + */ + edited: boolean + /** Whether this room has observed its doc become seeded — so a post-seed update counts as an edit but + * the seed transition itself does not. See the `doc.on('update')` edit-tracking below. */ + seededObserved: boolean + /** The pending debounced persist timer, if any. */ + persistTimer: ReturnType | null + /** Absolute time (ms) by which a debounced persist must fire even under continuous editing (the + * max-wait cap); null when no persist is pending. */ + persistDeadline: number | null +} + +/** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */ +const fileDocRooms = new Map() +/** socketId → its current file-doc room name (a socket edits at most one doc). */ +const socketToRoomName = new Map() +/** + * socketId → a monotonic join generation. A JOIN bumps it on arrival and, after + * the async authorization, proceeds only if the generation is still its own — so + * a newer JOIN (a fast document switch) or a disconnect (which drops the entry in + * cleanup) that occurred during authorization aborts the now-stale JOIN. Without + * this, an out-of-order authorize completion could bind the socket to the wrong + * document, or a disconnect-during-authorize could register a dead socket and + * leak its room. + */ +const joinGeneration = new Map() + +interface AwarenessChange { + added: number[] + updated: number[] + removed: number[] +} + +const fileDocRoom = (fileId: string): RoomRef => ({ + type: ROOM_TYPES.WORKSPACE_FILE_DOC, + id: fileId, +}) + +/** + * A `y-protocols` transaction/awareness origin is the emitting socket id (a + * string) when it came from a client, and something else (`null` / `'local'` / + * `'timeout'`) for server-internal changes. Returns the socket id to exclude + * from a relay, or `null` to broadcast to the whole room. + */ +function originSocketId(origin: unknown): string | null { + return typeof origin === 'string' ? origin : null +} + +/** + * Broadcast an AWARENESS frame to the room ACROSS tasks via the Socket.IO Redis adapter. Awareness + * (cursors/selection) is ephemeral and needs no convergence or replay, so the adapter's cross-task + * fan-out is exactly right for it. + */ +function broadcast(io: Server, name: string, payload: Uint8Array, exceptSocketId: string | null) { + const channel = exceptSocketId ? io.to(name).except(exceptSocketId) : io.to(name) + channel.emit(FILE_DOC_EVENTS.MESSAGE, payload) +} + +/** + * Broadcast a DOC-SYNC frame to this task's LOCAL clients only. Cross-task delivery rides the shared + * Redis stream (each task's tailer applies the update and runs its OWN local fan-out), so using the + * adapter here would double-deliver and amplify. With no adapter (single-pod dev) `io.local` is the + * whole room, so behavior is unchanged. + */ +function broadcastLocal( + io: Server, + name: string, + payload: Uint8Array, + exceptSocketId: string | null +) { + const channel = exceptSocketId ? io.local.to(name).except(exceptSocketId) : io.local.to(name) + channel.emit(FILE_DOC_EVENTS.MESSAGE, payload) +} + +/** + * Schedule a debounced server-side persist of the live doc back to durable markdown. Coalesces rapid + * edits; a no-op until the room knows its workspace (set at join). A {@link PERSIST_MAX_WAIT_MS} + * max-wait caps the debounce so a continuous burst still persists periodically. The final flush on + * last-disconnect is separate ({@link flushPersist} with `final`). + */ +function schedulePersist(name: string, room: FileDocRoom): void { + if (!room.workspaceId || !room.lastEditorUserId) return + const now = Date.now() + if (room.persistDeadline === null) room.persistDeadline = now + PERSIST_MAX_WAIT_MS + if (room.persistTimer) clearTimeout(room.persistTimer) + const delay = Math.max(0, Math.min(PERSIST_DEBOUNCE_MS, room.persistDeadline - now)) + room.persistTimer = setTimeout(() => { + room.persistTimer = null + room.persistDeadline = null + void flushPersist(name, room, false) + }, delay) +} + +/** + * Project the live doc to markdown and write it durably via the app. `final` (last collaborator + * leaving) always writes; a debounced mid-edit flush first claims a best-effort cross-task dedup WINDOW + * (a TTL key that just expires, so at most ~one persist per window cluster-wide) so concurrent tasks + * editing the same file don't each write a redundant blob version. Best-effort: never throws (a failure + * is retried on the next debounce; the stream holds the state meanwhile). + * + * Persists the AUTHORITATIVE shared state (the stream), not this task's local doc: a copilot merge — or + * a peer's edit — published by another task may not be integrated into `room.doc` yet (and the stream + * holds content even when THIS task's doc was never locally seeded), so a last-disconnect flush can't + * clobber the durable file with a lagging projection. The local doc is captured SYNCHRONOUSLY as a + * fallback before any await, so a `void flushPersist(name, room, true)` fired immediately before the + * caller destroys `room.doc` never encodes a destroyed doc, and the disabled path stays authoritative. + */ +async function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise { + // Never project a doc no user actually edited back over the file (see {@link FileDocRoom.edited}). + if (!room.edited || !room.workspaceId || !room.lastEditorUserId) return + const store = getFileDocStore() + // Synchronous fallback capture — before any await, since the caller may destroy `room.doc` the moment + // this yields. Only meaningful once seeded; used only when the authoritative stream state is absent. + const localState = isDocSeeded(room.doc) ? Y.encodeStateAsUpdate(room.doc) : null + try { + if (!final && !(await store.tryClaimPersistWindow(name, FILE_DOC_TIMEOUTS.persistRequestMs))) + return + let docState = localState + if (store.enabled) { + try { + docState = (await store.getStreamState(name)) ?? localState + } catch (streamError) { + // A transient Redis read must NOT drop the write when we already hold a valid local snapshot — + // else the last-disconnect flush loses the session's edits as the room is torn down. + if (!localState) throw streamError + logger.warn(`Stream state unavailable for file ${room.fileId}; persisting local snapshot`, { + error: getErrorMessage(streamError), + }) + } + } + if (!docState) return // nothing seeded/authoritative to persist yet + await fetchFileDocPersist(room.workspaceId, room.fileId, room.lastEditorUserId, docState) + } catch (error) { + logger.warn(`Persist failed for file ${room.fileId}`, { error: getErrorMessage(error) }) + } +} + +/** + * Broadcast the room's collaborator roster to everyone in it, for the avatar stack. One entry + * PER SESSION (socket) — the client excludes its own socket and dedupes the remainder per user + * for display, so a second tab of the same account still registers as present (mirroring the + * canvas presence model). Deduping here instead would drop the current user's other sessions + * asymmetrically (only one socket survives), so each client could never reliably self-exclude. + * Identity comes from each owner's server-authenticated session — never the client-set awareness + * — so a peer cannot spoof or suppress an entry. + */ +function broadcastFileDocPresence(io: Server, name: string, room: FileDocRoom) { + const users: FileDocPresenceUser[] = [] + for (const [socketId, owner] of room.owners) { + users.push({ + socketId, + userId: owner.userId, + userName: owner.userName, + avatarUrl: owner.avatarUrl, + }) + } + io.to(name).emit(FILE_DOC_EVENTS.PRESENCE, { fileId: room.fileId, users }) +} + +/** Whether the client has recorded that it seeded the document's initial content. */ +function isDocSeeded(doc: Y.Doc): boolean { + return doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag) === true +} + +/** + * Decode the client IDs an awareness update carries, without applying it, to + * check a frame only touches its sender's own presence. Mirrors the wire format + * of `awarenessProtocol.encodeAwarenessUpdate`: a count, then per client a + * varUint id, a varUint clock, and a varString state. + */ +function awarenessUpdateClientIds(update: Uint8Array): number[] { + const decoder = decoding.createDecoder(update) + const count = decoding.readVarUint(decoder) + const ids: number[] = [] + for (let i = 0; i < count; i++) { + ids.push(decoding.readVarUint(decoder)) + decoding.readVarUint(decoder) // clock + decoding.readVarUint8Array(decoder) // state bytes — advanced past, only ids matter + } + return ids +} + +/** + * Drop a room's document + awareness once it has no owners on THIS task, so an idle file holds no + * memory. Before dropping, flush the converged doc back to durable markdown (the last collaborator on + * this task leaving) and detach from the shared stream. A later joiner re-creates it — catching up + * from the stream if the doc is still live on another task, or re-seeding from markdown otherwise. + */ +function destroyRoomIfIdle(name: string) { + const room = fileDocRooms.get(name) + if (!room || room.owners.size > 0) return + room.persistDeadline = null + if (room.persistTimer) { + clearTimeout(room.persistTimer) + room.persistTimer = null + } + // Final durable flush BEFORE teardown — `flushPersist` encodes the doc synchronously (before the + // destroy below) and awaits the write in the background. Best-effort; never throws. + void flushPersist(name, room, true) + getFileDocStore().detachRoom(name) + room.awareness.destroy() + room.doc.destroy() + fileDocRooms.delete(name) +} + +/** + * Flush every open, edited room's converged doc to durable markdown, AWAITING the writes. Called on + * graceful shutdown (rolling deploy / scale-in) so edits since the last debounce aren't left only in the + * ephemeral stream — the per-socket disconnect flush is fire-and-forget and would race `process.exit`. + * Best-effort and bounded by each persist's own timeout; never throws. Rooms are NOT torn down here (the + * process is exiting); only their durable state is secured. + */ +export async function flushAllFileDocRooms(): Promise { + const flushes: Promise[] = [] + for (const [name, room] of fileDocRooms) { + if (room.edited) flushes.push(flushPersist(name, room, true)) + } + await Promise.all(flushes) +} + +/** + * Seed a room's document server-side, once, on the first join: ask the app to build the seed (the + * file's current markdown → Yjs, through the exact editor engine) and apply it, which relays the + * content to every connected client via `doc.on('update')`. No client is elected to import content. + * + * `isDocSeeded` is the sufficient guard: content only ever reaches the doc alongside the seed flag + * (this seed, or a client's offline fallback), so an unseeded doc is genuinely empty and safe to seed. + * A genuinely empty/missing file returns `null` (a read error throws instead), so still set the flag — + * an empty doc must reach readiness, not wait forever. After the fetch, re-check the room is still + * live and unseeded (an owner may have left, or a client seeded it, while the fetch was in flight). + * + * Recovery on failure is deliberately simple — no in-room retry loop: a single attempt bounded by a + * timeout shorter than the client's readiness deadline, then release the guard. A transient failure + * is re-attempted by the next join/reconnect; a persistent one lets the connected client's readiness + * deadline lapse into its read-only fallback. (An in-room backoff retry can outlast that client + * deadline, so it would keep trying a doc the client has already given up on — worse, not better.) + */ +async function ensureServerSeed( + name: string, + room: FileDocRoom, + workspaceId: string +): Promise { + if (room.serverSeedStarted || isDocSeeded(room.doc)) return + room.serverSeedStarted = true + const store = getFileDocStore() + // Exactly one task across the cluster builds the seed; the others receive it via the stream (the fix + // for split-brain seeding). Returns a lock token here (single-pod: a sentinel token). + const token = await store.shouldSeed(name) + if (!token) { + // A peer is seeding (or already did). Release our guard so a later join can retry if the seed never + // arrives (e.g. the seeder died); the stream / this doc being seeded makes a retry safe. + room.serverSeedStarted = false + return + } + // We hold the seed lock — release it on EVERY exit from here (one `finally`, impossible to leak). + try { + const update = await fetchFileDocSeed(workspaceId, room.fileId) + if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return + // Build the seed (file content + seed flag, or just the flag for an empty/missing file) and write it + // to the shared stream ATOMICALLY, iff the stream is still empty. This — NOT the seed lock — is the + // split-brain guard: two tasks racing (even both past an expired lock) can never both seed, because + // the emptiness check and the append are one Redis-side step. Publish-before-apply: the doc is marked + // seeded (via the local apply) only once the seed is durably in the stream, so a failed write leaves + // the doc unseeded and the stream empty for a clean retry. SEED_ORIGIN keeps `doc.on('update')` from + // re-publishing it. + const seedUpdate = update ?? emptySeedUpdate() + const didSeed = await store.seedIfEmpty(name, seedUpdate) + if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return + if (didSeed) { + Y.applyUpdate(room.doc, seedUpdate, SEED_ORIGIN) + } else { + // A peer seeded first: its seed arrives via the tailer, so we must NOT apply our own — a second, + // different-client-id seed IS the split-brain. Clear the guard so a later join can retry if that + // peer seed somehow never lands (e.g. a fail-closed `xLen` error made `shouldSeed` skip a genuinely + // empty stream); a real peer-seed makes the retry a no-op. + room.serverSeedStarted = false + } + } catch (error) { + logger.warn(`Server seed failed for file ${room.fileId} (workspace ${workspaceId})`, error) + room.serverSeedStarted = false + } finally { + await store.releaseSeedLock(name, token) + } +} + +/** The seed update for an empty/missing file: just the `initialContentLoaded` flag, so an empty doc + * still reaches readiness (and its emptiness is durably shared like any seed). */ +function emptySeedUpdate(): Uint8Array { + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + try { + return Y.encodeStateAsUpdate(doc) + } finally { + doc.destroy() + } +} + +/** Serializes live merges per file so overlapping calls never race the same doc (see below). */ +const fileDocMergeChains = new Map>() + +/** + * Apply new markdown into a file's LIVE collaborative document (Stage C — copilot writing into an open + * doc). Ships the document's current state to the app to build a minimal Yjs diff, applies it — which + * fires `doc.on('update')` and relays the merge to every connected editor, reconciled with any + * concurrent user edits — and reports whether it landed. + * + * Merges for the same file are SERIALIZED — within a task by the {@link fileDocMergeChains} promise + * chain, and ACROSS tasks by a Redis merge lock (below) — so each diff is computed against the previous + * merge's result, never the same stale base concurrently. + * + * Multi-task: the diff is computed against, and published to, the file's SHARED stream state — so the + * merge reaches the live doc no matter which task holds it (the apply-edit HTTP call can land on any + * task). Every task's tailer then applies it and fans it out to its own clients. Because the merge + * always lands in the stream while the stream exists, the stream can never go stale relative to a + * copilot direct file write. + * + * Returns `'no-live-room'` when there is no shared state to merge against (no doc is or was recently + * live): the caller (copilot) writes the file directly and the next open seeds from that markdown. + */ +export function applyMarkdownToLiveFileDoc( + fileId: string, + markdown: string +): Promise<'applied' | 'no-live-room'> { + const name = roomName(fileDocRoom(fileId)) + const prior = fileDocMergeChains.get(name) ?? Promise.resolve() + // `.catch` so a failed prior merge doesn't reject this one — each merge is independent. + const run = prior.catch(() => {}).then(() => mergeMarkdownIntoRoom(name, fileId, markdown)) + fileDocMergeChains.set( + name, + run.finally(() => { + if (fileDocMergeChains.get(name) === run) fileDocMergeChains.delete(name) + }) + ) + return run +} + +async function mergeMarkdownIntoRoom( + name: string, + fileId: string, + markdown: string +): Promise<'applied' | 'no-live-room'> { + const store = getFileDocStore() + + if (store.enabled) { + // Serialize merges to this file ACROSS tasks — the per-file chain above only covers this process. + // Two copilot edits to the same file landing on different tasks must not diff the SAME shared base + // and publish conflicting full-document rewrites. Retry LONGER than the lock TTL, so a live holder + // always releases (or its lock expires) first and we acquire — never merging against a shared base + // while a peer holds the lock. If somehow still unavailable, skip the live merge (copilot's durable + // file write stands) rather than race. + let token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + for (let i = 0; !token && i < MERGE_LOCK_RETRIES; i++) { + await sleep(MERGE_LOCK_RETRY_MS) + token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + } + if (!token) { + logger.warn(`Merge lock unavailable for file ${fileId}; skipping live merge`) + return 'no-live-room' + } + try { + // Compute the diff against the committed SHARED state and PUBLISH it — every task with the doc + // live (including this one, via its own tailer) applies it and fans it out to its clients, so the + // merge reaches the live doc no matter which task the apply-edit call landed on. An empty stream + // means no doc is (or was recently) live → nothing to merge into. AWAIT the publish so the diff is + // durably in the stream before we release the lock (else the next task would diff a stale base). + const base = await store.getStreamState(name) + if (!base) return 'no-live-room' + const diff = await fetchFileDocMerge(fileId, base, markdown) + await store.publishAndWait(name, diff) + return 'applied' + } finally { + await store.releaseMergeSlot(name, token) + } + } + + // Single-replica fallback: apply straight to the local authoritative doc. + const room = fileDocRooms.get(name) + if (!room || room.owners.size === 0 || !isDocSeeded(room.doc)) return 'no-live-room' + const update = await fetchFileDocMerge(fileId, Y.encodeStateAsUpdate(room.doc), markdown) + // The room may have been dropped while the diff was being built; never touch a destroyed doc. + if (fileDocRooms.get(name) !== room) return 'no-live-room' + // No transaction origin → `doc.on('update')` relays to the WHOLE room (every editor sees copilot). + Y.applyUpdate(room.doc, update) + return 'applied' +} + +/** + * Get (or lazily create) the authoritative document for a room, wiring the two + * relay handlers exactly once: document updates and awareness changes are + * broadcast to the room, excluding the origin socket (it already applied them). + */ +function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { + const name = roomName(ref) + const existing = fileDocRooms.get(name) + if (existing) return existing + + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + // The server holds no cursor of its own; it only relays clients' awareness. + awareness.setLocalState(null) + + const room: FileDocRoom = { + fileId: ref.id, + doc, + awareness, + owners: new Map(), + serverSeedStarted: false, + workspaceId: null, + lastEditorUserId: null, + edited: false, + seededObserved: false, + persistTimer: null, + persistDeadline: null, + } + // Register synchronously BEFORE the async catch-up so a concurrent join sees this room, not a second. + fileDocRooms.set(name, room) + + doc.on('update', (update: Uint8Array, origin: unknown) => { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, update) + // Fan out to THIS task's clients only (excluding the origin socket if local). Cross-task delivery + // rides the shared stream — every task's tailer applies + runs its own local fan-out. + broadcastLocal(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) + // Share every locally-originated update to the stream so peers converge. Skip updates that already + // came FROM the stream (REDIS_ORIGIN / REDIS_SNAPSHOT_ORIGIN) and SEED_ORIGIN — the seed is published + // EXPLICITLY and AWAITED under the seed lock (so it lands before the lock releases), which a + // fire-and-forget publish here couldn't guarantee. + if (origin !== REDIS_ORIGIN && origin !== REDIS_SNAPSHOT_ORIGIN && origin !== SEED_ORIGIN) + getFileDocStore().publish(name, update) + // Edit tracking for persistence. Mark the doc dirty on any update applied AFTER it was seeded — a + // local user edit (socket origin) OR a peer's edit relayed via the tailer (REDIS_ORIGIN) — so + // whichever task is last to leave persists real edits, even one that only tailed them. A compaction + // snapshot on catch-up (REDIS_SNAPSHOT_ORIGIN) also counts: it folds real edits into one frame, so a + // fresh task catching up purely from it must not treat the doc as unedited. The seed transition + // itself is never counted, so a seeded-but-unedited doc is never projected back over the file. NOTE: + // in the multi-replica path a copilot merge is NOT excluded — it round-trips through the stream as + // REDIS_ORIGIN, indistinguishable from a peer edit, so it marks `edited`. That only ever causes an + // extra idempotent persist of content copilot already wrote directly (safe over-persist, never a lost + // edit); the single-replica path applies the merge locally with no origin and does not count it. + const seededBefore = room.seededObserved + if (isDocSeeded(room.doc)) room.seededObserved = true + if ( + originSocketId(origin) || + origin === REDIS_SNAPSHOT_ORIGIN || + (seededBefore && origin === REDIS_ORIGIN) + ) + room.edited = true + // Debounce a persist for LOCAL user edits only (peers debounce their own). + if (originSocketId(origin)) schedulePersist(name, room) + }) + + awareness.on('update', ({ added, updated, removed }: AwarenessChange, origin: unknown) => { + const changed = added.concat(updated, removed) + if (changed.length === 0) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(awareness, changed) + ) + broadcast(io, name, encoding.toUint8Array(encoder), originSocketId(origin)) + }) + + // Load the shared state into the doc and start tailing the stream (fire-and-forget: content streams + // in via `doc.on('update')` as it lands, mirroring the fire-and-forget seed below). Disabled → no-op. + void getFileDocStore().attachRoom(name, doc) + + return room +} + +function emitJoinError( + socket: AuthenticatedSocket, + fileId: unknown, + error: string, + code: string, + retryable: boolean +) { + socket.emit(FILE_DOC_EVENTS.JOIN_ERROR, { + fileId: typeof fileId === 'string' ? fileId : '', + error, + code, + retryable, + }) +} + +function handleMessage(socket: AuthenticatedSocket, data: unknown) { + const name = socketToRoomName.get(socket.id) + if (!name) return + const room = fileDocRooms.get(name) + if (!room) return + + const bytes = toFileDocBytes(data) + if (!bytes) return + + // A malformed frame from any client must never escape as a process-level + // exception; drop it and keep the relay running. + try { + const decoder = decoding.createDecoder(bytes) + const messageType = decoding.readVarUint(decoder) + + switch (messageType) { + case FILE_DOC_MESSAGE_TYPE.SYNC: { + // Attribute a server-side persist of the resulting edit to the actual editor (blob metadata). + const editor = room.owners.get(socket.id)?.userId + if (editor) room.lastEditorUserId = editor + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + // `socket.id` is the transaction origin, so the doc's `update` handler + // excludes this sender when relaying the applied update to the room. + syncProtocol.readSyncMessage(decoder, encoder, room.doc, socket.id) + // A reply longer than the 1-byte type tag is a sync step 2 (or step 1) + // destined for the sender only; applied updates fan out via `doc.on`. + if (encoding.length(encoder) > 1) { + socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + break + } + case FILE_DOC_MESSAGE_TYPE.AWARENESS: { + const update = decoding.readVarUint8Array(decoder) + // Enforce presence ownership: a socket may only publish/remove awareness + // for the clientID it bound at join, so a peer cannot spoof or clear + // another collaborator's caret. + const owned = room.owners.get(socket.id)?.clientId + if (owned === undefined || awarenessUpdateClientIds(update).some((id) => id !== owned)) { + logger.warn('Dropping awareness frame for an unowned client id', { socketId: socket.id }) + return + } + awarenessProtocol.applyAwarenessUpdate(room.awareness, update, socket.id) + break + } + default: + logger.warn('Unknown file-doc message type', { messageType }) + } + } catch (error) { + logger.warn('Dropping malformed file-doc frame', { socketId: socket.id, error }) + } +} + +/** + * Remove a socket from its file-doc room: clear its awareness state (so its caret + * disappears for everyone else) and drop the room's document when the last + * collaborator leaves. + * Exported for the disconnect handler; safe to call for a socket in no room. + */ +export function cleanupFileDocForSocket(socketId: string, io: Server, endOfLife = false): void { + // The join-generation counter is monotonic for the socket's WHOLE life and must survive a room + // switch/leave: resetting it here would let the next join reuse a low number that a still + // in-flight earlier join also holds, so that stale join passes the generation guard and rebinds + // the socket to the wrong document. Drop it ONLY when the socket is truly gone (disconnect), + // which is also the only place the map would otherwise leak. An in-flight join is already + // aborted on disconnect by the `socket.disconnected` check, and on a switch by a newer join + // bumping the generation — neither needs this delete. + if (endOfLife) joinGeneration.delete(socketId) + + const name = socketToRoomName.get(socketId) + if (!name) return + socketToRoomName.delete(socketId) + + const room = fileDocRooms.get(name) + if (!room) return + + const owner = room.owners.get(socketId) + room.owners.delete(socketId) + if (owner !== undefined) { + // Fires the awareness `update` handler with a non-socket origin → the removal + // is broadcast to every remaining client, so the departed caret vanishes. + awarenessProtocol.removeAwarenessStates(room.awareness, [owner.clientId], null) + // Refresh the roster for whoever remains (server-authenticated identity). + broadcastFileDocPresence(io, name, room) + } + + destroyRoomIfIdle(name) +} + +/** + * Registers the collaborative file-document handlers on a socket. Room id is the + * file id; joining requires workspace `write` (editing a document). Mirrors the + * workspace-files join shape (auth → readiness → validate → authorize → join), + * then runs the Yjs sync/awareness handshake. + * + * The avatar roster is derived from this room's own `owners` map and broadcast as + * `FILE_DOC_EVENTS.PRESENCE` — NOT the Redis-backed room-manager presence the workflow / + * table rooms use — because the file-doc room already owns an authoritative in-memory Y.Doc + * pinned to a single replica, so the session identity is right here with no extra store. + */ +export function setupWorkspaceFileDocHandlers( + socket: AuthenticatedSocket, + roomManager: IRoomManager +) { + const io = roomManager.io + // The file this socket currently intends to edit (set when a join starts). A leave targeting it + // — or an unscoped leave — advances the join generation to cancel an in-flight join, so a join + // awaiting authorization can't complete after the client left and register a ghost owner. A + // leave for a DIFFERENT file must NOT cancel it (a document switch), mirroring workspace-files. + let currentFileId: string | null = null + + socket.on(FILE_DOC_EVENTS.JOIN, async ({ fileId, clientId }: JoinFileDocPayload) => { + // Hoisted so the catch can tell whether this join was superseded (a switch to another file) + // before surfacing a retryable error for the abandoned one. + let generation: number | undefined + try { + const userId = socket.userId + const userName = socket.userName + + if (!userId || !userName) { + emitJoinError(socket, fileId, 'Authentication required', 'AUTHENTICATION_REQUIRED', false) + return + } + if (!roomManager.isReady()) { + emitJoinError(socket, fileId, 'Realtime unavailable', 'ROOM_MANAGER_UNAVAILABLE', true) + return + } + if ( + typeof fileId !== 'string' || + fileId.length === 0 || + // A Yjs clientID is a uint32; reject NaN/Infinity/negative/non-integer so a malformed id + // can't become a bogus ownership key. + !Number.isInteger(clientId) || + clientId < 0 + ) { + emitJoinError(socket, fileId, 'Invalid join payload', 'INVALID_PAYLOAD', false) + return + } + + // Claim this JOIN's generation before the async authorize below, and record the file the + // socket now intends to edit so a leave for it can cancel this join if it's still in-flight. + generation = (joinGeneration.get(socket.id) ?? 0) + 1 + joinGeneration.set(socket.id, generation) + currentFileId = fileId + + const room = fileDocRoom(fileId) + const name = roomName(room) + + const authorized = await resolveRoomJoinAuth({ + userId, + room, + action: 'write', + logger, + logLabel: `file-doc room for ${userId}`, + messages: { + verifyFailed: 'Failed to verify workspace access', + notFound: 'File not found', + accessDenied: 'Access denied to file', + }, + emitError: ({ error, code, retryable }) => + emitJoinError(socket, fileId, error, code, retryable), + }) + if (!authorized) return + + // Server-authenticated identity for the presence roster (never trusts the client-set + // awareness). Resolved here so the generation guard below also covers this await. + const avatarUrl = await resolveAvatarUrl(socket, userId) + + // Abort a JOIN superseded during authorization/identity resolution: the socket + // disconnected, or a newer JOIN (a document switch) bumped the generation. Registering + // here would leak a dead socket's room or bind the socket to the wrong document. + if (socket.disconnected || joinGeneration.get(socket.id) !== generation) return + + const entry = getOrCreateRoom(io, room) + + // A client id must be owned by at most one user, or a peer could bind an active + // collaborator's id and pass the per-frame ownership check to spoof/clear its caret. + // Distinguish a reconnect from a spoof by the owning user: the same user reclaiming its + // own client id (a dropped socket reconnecting reuses the Yjs client id, and its prior + // socket may not be cleaned up yet) takes over the stale binding; a DIFFERENT user is + // rejected. This runs BEFORE any teardown of the socket's current binding below, so a + // rejected rebind — even during a document switch — leaves the socket's existing document + // and caret untouched. + for (const [otherSid, owner] of entry.owners) { + if (owner.clientId !== clientId || otherSid === socket.id) continue + if (owner.userId !== userId) { + emitJoinError(socket, fileId, 'Client id already in use', 'CLIENT_ID_IN_USE', false) + return + } + // Fully evict the stale prior socket of the same user — owner + caret AND its room + // mapping + Socket.IO membership — so it can no longer send document (sync) frames: + // handleMessage's SYNC path gates on socketToRoomName, not owners. Done inline rather + // than via cleanupFileDocForSocket, which could destroyRoomIfIdle the room we're joining. + entry.owners.delete(otherSid) + awarenessProtocol.removeAwarenessStates(entry.awareness, [owner.clientId], null) + socketToRoomName.delete(otherSid) + io.in(otherSid).socketsLeave(name) + } + + // Only now that the rebind is guaranteed to succeed, leave a previously-joined document if + // switching (a socket edits at most one). A duplicate join of the SAME room falls through + // and simply re-runs the sync handshake, idempotently. + const currentName = socketToRoomName.get(socket.id) + if (currentName && currentName !== name) { + socket.leave(currentName) + cleanupFileDocForSocket(socket.id, io) + } + + // Accepted: a same socket rebinding to a NEW client id clears its old caret + // so it doesn't linger as a ghost after the binding is overwritten. + const previous = entry.owners.get(socket.id) + if (previous !== undefined && previous.clientId !== clientId) { + awarenessProtocol.removeAwarenessStates(entry.awareness, [previous.clientId], null) + } + + entry.owners.set(socket.id, { clientId, userId, userName, avatarUrl }) + socketToRoomName.set(socket.id, name) + socket.join(name) + + // Capture what the server-side persist needs: the workspace to write back to, and the current + // user for attribution (refreshed to the actual editor on each edit in `handleMessage`). + if (authorized.workspaceId) entry.workspaceId = authorized.workspaceId + entry.lastEditorUserId = userId + + socket.emit(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId }) + // Server-authenticated roster → everyone in the room, including this joiner. + broadcastFileDocPresence(io, name, entry) + + // Begin the sync handshake: send the server's state (sync step 1). The + // client replies with its updates and requests the server's in return. + const syncEncoder = encoding.createEncoder() + encoding.writeVarUint(syncEncoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep1(syncEncoder, entry.doc) + socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(syncEncoder)) + + // Send existing awareness so the new client immediately sees others' carets. + const states = entry.awareness.getStates() + if (states.size > 0) { + const awarenessEncoder = encoding.createEncoder() + encoding.writeVarUint(awarenessEncoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array( + awarenessEncoder, + awarenessProtocol.encodeAwarenessUpdate(entry.awareness, Array.from(states.keys())) + ) + socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(awarenessEncoder)) + } + + // Seed the document server-side (once). Fire-and-forget: the join completes immediately and + // the seed relays to this socket via `doc.on('update')` the moment it lands. + if (authorized.workspaceId) void ensureServerSeed(name, entry, authorized.workspaceId) + + logger.info(`User ${userId} joined file-doc room ${fileId}`) + } catch (error) { + logger.error('Error joining file-doc room:', error) + try { + const name = roomName(fileDocRoom(fileId)) + socket.leave(name) + // Roll back ONLY this join's target room. cleanupFileDocForSocket keys off socketToRoomName, + // which — if the join failed before rebinding to the target (e.g. a switch that threw during + // client-id reclaim) — still points at the socket's PRIOR, valid document. Running it then + // would tear down a document the socket is validly in. So only run it when the binding + // already points at the target; otherwise the socket never registered as an owner of this + // room and the only leftover is a freshly-created empty room, dropped below. + if (socketToRoomName.get(socket.id) === name) cleanupFileDocForSocket(socket.id, io) + destroyRoomIfIdle(name) + } catch {} + // Suppress the client-facing error when this join was already superseded (a switch to another + // file, or a disconnect): the rollback above still ran, but a retryable error naming the + // abandoned file could make a client re-join it and cancel the newer one (matches the sibling + // handlers). + if ( + socket.disconnected || + (generation !== undefined && joinGeneration.get(socket.id) !== generation) + ) + return + emitJoinError(socket, fileId, 'Failed to join file document', 'JOIN_FAILED', true) + } + }) + + socket.on(FILE_DOC_EVENTS.MESSAGE, (data: unknown) => handleMessage(socket, data)) + + socket.on(FILE_DOC_EVENTS.LEAVE, (payload?: LeaveFileDocPayload) => { + try { + // Cancel an in-flight join whose file the client is now leaving (or an unscoped leave): a + // join still awaiting authorization would otherwise complete after the client left, register + // as an owner, and broadcast a ghost collaborator until disconnect. Guard on the current + // file intent so a stale/deferred leave for a DIFFERENT file can't abort the join the client + // has since switched to (bumping the generation blindly caused that regression in #5941). + if (!payload?.fileId || payload.fileId === currentFileId) { + joinGeneration.set(socket.id, (joinGeneration.get(socket.id) ?? 0) + 1) + currentFileId = null + } + // Tear down membership only for a REGISTERED room; a leave that raced ahead of an in-flight + // join (nothing registered yet) has already cancelled it above. + const name = socketToRoomName.get(socket.id) + if (!name) return + // Scope the leave to the named file when provided: a deferred leave from a + // prior document must not evict the socket from one it has since opened. + if (payload?.fileId && roomName(fileDocRoom(payload.fileId)) !== name) return + socket.leave(name) + cleanupFileDocForSocket(socket.id, io) + } catch (error) { + logger.error('Error leaving file-doc room:', error) + } + }) +} diff --git a/apps/realtime/src/handlers/index.ts b/apps/realtime/src/handlers/index.ts index 6ded2e54741..8dd71093673 100644 --- a/apps/realtime/src/handlers/index.ts +++ b/apps/realtime/src/handlers/index.ts @@ -1,9 +1,13 @@ +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { setupConnectionHandlers } from '@/handlers/connection' +import { setupWorkspaceFileDocHandlers } from '@/handlers/file-doc' import { setupOperationsHandlers } from '@/handlers/operations' import { setupPresenceHandlers } from '@/handlers/presence' import { setupSubblocksHandlers } from '@/handlers/subblocks' +import { setupTablesHandlers } from '@/handlers/tables' import { setupVariablesHandlers } from '@/handlers/variables' import { setupWorkflowHandlers } from '@/handlers/workflow' +import { setupWorkspaceInvalidationRoom } from '@/handlers/workspace-invalidation-room' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -13,5 +17,10 @@ export function setupAllHandlers(socket: AuthenticatedSocket, roomManager: IRoom setupSubblocksHandlers(socket, roomManager) setupVariablesHandlers(socket, roomManager) setupPresenceHandlers(socket, roomManager) + // Presence-free, workspace-scoped live-list rooms (share one implementation). + setupWorkspaceInvalidationRoom(socket, roomManager, ROOM_TYPES.WORKSPACE_FILES) + setupWorkspaceInvalidationRoom(socket, roomManager, ROOM_TYPES.WORKSPACE_TABLES) + setupWorkspaceFileDocHandlers(socket, roomManager) + setupTablesHandlers(socket, roomManager) setupConnectionHandlers(socket, roomManager) } diff --git a/apps/realtime/src/handlers/operations.ts b/apps/realtime/src/handlers/operations.ts index eef51847718..d3d603887db 100644 --- a/apps/realtime/src/handlers/operations.ts +++ b/apps/realtime/src/handlers/operations.ts @@ -9,6 +9,7 @@ import { type VariableOperation, WORKFLOW_OPERATIONS, } from '@sim/realtime-protocol/constants' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { WorkflowOperationSchema } from '@sim/realtime-protocol/schemas' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' @@ -16,7 +17,7 @@ import { ZodError } from 'zod' import { persistWorkflowOperation } from '@/database/operations' import type { AuthenticatedSocket } from '@/middleware/auth' import { checkWorkflowOperationPermission } from '@/middleware/permissions' -import type { IRoomManager, UserSession } from '@/rooms' +import { type IRoomManager, type UserSession, workflowRoom as wf } from '@/rooms' const logger = createLogger('OperationsHandlers') @@ -44,7 +45,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager let session: UserSession | null = null try { - workflowId = await roomManager.getWorkflowIdForSocket(socket.id) + workflowId = (await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW))?.id ?? null session = await roomManager.getUserSession(socket.id) } catch (error) { logger.error('Error loading session for workflow operation:', error) @@ -65,7 +66,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager let hasRoom = false try { - hasRoom = await roomManager.hasWorkflowRoom(workflowId) + hasRoom = await roomManager.hasRoom(wf(workflowId)) } catch (error) { logger.error('Error checking workflow room:', error) emitOperationError( @@ -98,14 +99,16 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager const operationTimestamp = isPositionUpdate ? timestamp : Date.now() // Get user presence for permission checking - const users = await roomManager.getWorkflowUsers(workflowId) + const users = await roomManager.getRoomUsers(wf(workflowId)) const userPresence = users.find((u) => u.socketId === socket.id) // Skip permission checks for non-committed position updates (broadcasts only, no persistence) if (isPositionUpdate && !commitPositionUpdate) { // Update last activity if (userPresence) { - await roomManager.updateUserActivity(workflowId, socket.id, { lastActivity: Date.now() }) + await roomManager.updateUserActivity(wf(workflowId), socket.id, { + lastActivity: Date.now(), + }) } } else { // Check permissions from cached role for all other operations @@ -123,7 +126,9 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager return } - await roomManager.updateUserActivity(workflowId, socket.id, { lastActivity: Date.now() }) + await roomManager.updateUserActivity(wf(workflowId), socket.id, { + lastActivity: Date.now(), + }) // Re-validate the workspace role against the DB (cached per pod for a short // window) so revoked or downgraded collaborators lose write access live. @@ -198,7 +203,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager timestamp: operationTimestamp, userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) if (operationId) { socket.emit('operation-confirmed', { @@ -244,7 +249,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager timestamp: operationTimestamp, userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) if (operationId) { socket.emit('operation-confirmed', { operationId, serverTimestamp: Date.now() }) @@ -277,7 +282,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) const broadcastData = { operation, @@ -317,7 +322,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) const broadcastData = { operation, @@ -354,7 +359,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -386,7 +391,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -415,7 +420,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -447,7 +452,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -479,7 +484,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -511,7 +516,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -540,7 +545,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) socket.to(workflowId).emit('workflow-operation', { operation, @@ -569,7 +574,7 @@ export function setupOperationsHandlers(socket: AuthenticatedSocket, roomManager userId: session.userId, }) - await roomManager.updateRoomLastModified(workflowId) + await roomManager.updateRoomLastModified(wf(workflowId)) const broadcastData = { operation, diff --git a/apps/realtime/src/handlers/presence.ts b/apps/realtime/src/handlers/presence.ts index 13aadc22f34..78b53176e2f 100644 --- a/apps/realtime/src/handlers/presence.ts +++ b/apps/realtime/src/handlers/presence.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import type { AuthenticatedSocket } from '@/middleware/auth' import type { IRoomManager } from '@/rooms' @@ -7,16 +8,16 @@ const logger = createLogger('PresenceHandlers') export function setupPresenceHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { socket.on('cursor-update', async ({ cursor }) => { try { - const workflowId = await roomManager.getWorkflowIdForSocket(socket.id) + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) const session = await roomManager.getUserSession(socket.id) - if (!workflowId || !session) return + if (!room || !session) return // Update cursor in room state - await roomManager.updateUserActivity(workflowId, socket.id, { cursor }) + await roomManager.updateUserActivity(room, socket.id, { cursor }) - // Broadcast to other users in the room - socket.to(workflowId).emit('cursor-update', { + // Broadcast to other users in the room (workflow room name is the bare id) + socket.to(room.id).emit('cursor-update', { socketId: socket.id, userId: session.userId, userName: session.userName, @@ -30,16 +31,16 @@ export function setupPresenceHandlers(socket: AuthenticatedSocket, roomManager: socket.on('selection-update', async ({ selection }) => { try { - const workflowId = await roomManager.getWorkflowIdForSocket(socket.id) + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) const session = await roomManager.getUserSession(socket.id) - if (!workflowId || !session) return + if (!room || !session) return // Update selection in room state - await roomManager.updateUserActivity(workflowId, socket.id, { selection }) + await roomManager.updateUserActivity(room, socket.id, { selection }) - // Broadcast to other users in the room - socket.to(workflowId).emit('selection-update', { + // Broadcast to other users in the room (workflow room name is the bare id) + socket.to(room.id).emit('selection-update', { socketId: socket.id, userId: session.userId, userName: session.userName, diff --git a/apps/realtime/src/handlers/room-join-auth.ts b/apps/realtime/src/handlers/room-join-auth.ts new file mode 100644 index 00000000000..a7c2a9aa24c --- /dev/null +++ b/apps/realtime/src/handlers/room-join-auth.ts @@ -0,0 +1,55 @@ +import type { createLogger } from '@sim/logger' +import { authorizeRoom } from '@sim/platform-authz/rooms' +import type { RoomRef } from '@sim/realtime-protocol/rooms' + +type Authorized = Awaited> + +interface ResolveRoomJoinAuthParams { + userId: string + room: RoomRef + action: 'read' | 'write' + logger: ReturnType + /** Included in the warn log on an authorize throw, e.g. `table room for ${userId}`. */ + logLabel: string + messages: { verifyFailed: string; notFound: string; accessDenied: string } + /** Emits the handler's own JOIN_ERROR shape (event name + id key differ per handler). */ + emitError: (args: { error: string; code: string; retryable: boolean }) => void +} + +/** + * Runs the shared authorize→allowed slice of a room join: authorizes the room and checks + * the result, emitting the handler-specific JOIN_ERROR on failure. Returns the authorized + * result on success, or `null` when it has already emitted an error and the caller must return. + * + * Deliberately excludes the auth/readiness/id-validation preamble and the join-generation + * capture/recheck — those differ per handler, and for file-doc the generation capture sits + * mid-preamble. This helper is always invoked strictly between a handler's generation capture + * and its post-authorize recheck; it contains exactly the one `await authorizeRoom` that the + * recheck was designed to cover and returns before any state mutation, so it never straddles + * that seam. Pass each handler's own `logger` so the log namespace/request-id context is kept. + */ +export async function resolveRoomJoinAuth( + params: ResolveRoomJoinAuthParams +): Promise { + const { userId, room, action, logger, logLabel, messages, emitError } = params + + let authorized: Authorized + try { + authorized = await authorizeRoom({ userId, room, action }) + } catch (error) { + logger.warn(`Error authorizing ${logLabel}:`, error) + emitError({ error: messages.verifyFailed, code: 'VERIFY_ACCESS_FAILED', retryable: true }) + return null + } + + if (!authorized.allowed) { + emitError({ + error: authorized.status === 404 ? messages.notFound : messages.accessDenied, + code: authorized.status === 404 ? 'NOT_FOUND' : 'ACCESS_DENIED', + retryable: false, + }) + return null + } + + return authorized +} diff --git a/apps/realtime/src/handlers/subblocks.ts b/apps/realtime/src/handlers/subblocks.ts index b2f94b6fb98..93e7411d148 100644 --- a/apps/realtime/src/handlers/subblocks.ts +++ b/apps/realtime/src/handlers/subblocks.ts @@ -3,12 +3,13 @@ import { workflow, workflowBlocks } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { assertWorkflowMutable, WorkflowLockedError } from '@sim/platform-authz/workflow' import { SUBBLOCK_OPERATIONS } from '@sim/realtime-protocol/constants' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { getErrorMessage } from '@sim/utils/errors' import { isWorkflowBlockProtected } from '@sim/workflow-types/workflow' import { and, eq } from 'drizzle-orm' import type { AuthenticatedSocket } from '@/middleware/auth' import { checkWorkflowOperationPermission } from '@/middleware/permissions' -import type { IRoomManager } from '@/rooms' +import { type IRoomManager, workflowRoom as wf } from '@/rooms' const logger = createLogger('SubblocksHandlers') @@ -69,7 +70,8 @@ export function setupSubblocksHandlers(socket: AuthenticatedSocket, roomManager: } try { - const sessionWorkflowId = await roomManager.getWorkflowIdForSocket(socket.id) + const sessionWorkflowId = + (await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW))?.id ?? null const session = await roomManager.getUserSession(socket.id) if (!sessionWorkflowId || !session) { @@ -106,7 +108,7 @@ export function setupSubblocksHandlers(socket: AuthenticatedSocket, roomManager: return } - const hasRoom = await roomManager.hasWorkflowRoom(workflowId) + const hasRoom = await roomManager.hasRoom(wf(workflowId)) if (!hasRoom) { logger.debug(`Ignoring subblock update: workflow room not found`, { socketId: socket.id, @@ -117,7 +119,7 @@ export function setupSubblocksHandlers(socket: AuthenticatedSocket, roomManager: return } - const users = await roomManager.getWorkflowUsers(workflowId) + const users = await roomManager.getRoomUsers(wf(workflowId)) const userPresence = users.find((user) => user.socketId === socket.id) if (!userPresence) { socket.emit('operation-forbidden', { @@ -182,7 +184,7 @@ export function setupSubblocksHandlers(socket: AuthenticatedSocket, roomManager: } // Update user activity - await roomManager.updateUserActivity(workflowId, socket.id, { lastActivity: Date.now() }) + await roomManager.updateUserActivity(wf(workflowId), socket.id, { lastActivity: Date.now() }) // Server-side debounce/coalesce by workflowId+blockId+subblockId const debouncedKey = `${workflowId}:${blockId}:${subblockId}` diff --git a/apps/realtime/src/handlers/tables.test.ts b/apps/realtime/src/handlers/tables.test.ts new file mode 100644 index 00000000000..518c5ee376a --- /dev/null +++ b/apps/realtime/src/handlers/tables.test.ts @@ -0,0 +1,351 @@ +/** + * @vitest-environment node + */ +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { TABLE_PRESENCE_EVENTS } from '@sim/realtime-protocol/table-presence' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { IRoomManager } from '@/rooms' + +const { mockAuthorizeRoom } = vi.hoisted(() => ({ + mockAuthorizeRoom: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { select: vi.fn() }, + user: { image: 'image' }, +})) + +vi.mock('@sim/platform-authz/rooms', () => ({ + authorizeRoom: mockAuthorizeRoom, +})) + +import { setupTablesHandlers } from '@/handlers/tables' + +const TABLE_ROOM = { type: ROOM_TYPES.TABLE, id: 'table-1' } + +function createSocket(overrides?: Record) { + const handlers: Record Promise | void> = {} + const toEmit = vi.fn() + const socket = { + id: 'socket-1', + userId: 'user-1', + userName: 'Test User', + userImage: 'avatar.png', + on: vi.fn((event: string, handler: (payload: unknown) => Promise | void) => { + handlers[event] = handler + }), + emit: vi.fn(), + join: vi.fn(), + leave: vi.fn(), + to: vi.fn().mockReturnValue({ emit: toEmit }), + ...overrides, + } + return { handlers, socket, toEmit } +} + +function createRoomManager(overrides?: Partial): IRoomManager { + return { + isReady: vi.fn().mockReturnValue(true), + getRoomForSocket: vi.fn().mockResolvedValue(null), + getRoomsForSocket: vi.fn().mockResolvedValue([]), + removeUserFromRoom: vi.fn().mockResolvedValue(false), + removeSocketFromAllRooms: vi.fn().mockResolvedValue([]), + broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), + getRoomUsers: vi.fn().mockResolvedValue([]), + hasRoom: vi.fn().mockResolvedValue(false), + deleteRoom: vi.fn().mockResolvedValue(undefined), + addUserToRoom: vi.fn().mockResolvedValue(undefined), + getUserSession: vi.fn().mockResolvedValue(null), + updateUserActivity: vi.fn().mockResolvedValue(undefined), + updateRoomLastModified: vi.fn().mockResolvedValue(undefined), + emitToRoom: vi.fn(), + getUniqueUserCount: vi.fn().mockResolvedValue(1), + getTotalActiveConnections: vi.fn().mockResolvedValue(0), + shutdown: vi.fn().mockResolvedValue(undefined), + initialize: vi.fn().mockResolvedValue(undefined), + io: { + in: vi.fn().mockReturnValue({ socketsLeave: vi.fn().mockResolvedValue(undefined) }), + }, + ...overrides, + } as unknown as IRoomManager +} + +type SetupArg = Parameters[0] + +describe('setupTablesHandlers', () => { + beforeEach(() => { + vi.clearAllMocks() + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', + }) + }) + + it('rejects join when the socket is not authenticated', async () => { + const { socket, handlers } = createSocket({ userId: undefined, userName: undefined }) + setupTablesHandlers(socket as unknown as SetupArg, createRoomManager()) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + + expect(socket.emit).toHaveBeenCalledWith(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId: 'table-1', + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + }) + + it('rejects join with a retryable error when realtime is unavailable', async () => { + const { socket, handlers } = createSocket() + setupTablesHandlers( + socket as unknown as SetupArg, + createRoomManager({ isReady: vi.fn().mockReturnValue(false) }) + ) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + TABLE_PRESENCE_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) + ) + }) + + it('rejects join when table access is denied', async () => { + mockAuthorizeRoom.mockResolvedValue({ + allowed: false, + status: 403, + workspaceId: 'ws-1', + workspacePermission: null, + }) + const { socket, handlers } = createSocket() + setupTablesHandlers(socket as unknown as SetupArg, createRoomManager()) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + TABLE_PRESENCE_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) + ) + }) + + it('joins the table room and broadcasts presence on success', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1', tabSessionId: 'tab-1' }) + + expect(socket.join).toHaveBeenCalledWith('table:table-1') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + TABLE_ROOM, + 'socket-1', + expect.objectContaining({ userId: 'user-1', role: 'admin' }) + ) + expect(socket.emit).toHaveBeenCalledWith( + TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, + expect.objectContaining({ tableId: 'table-1', socketId: 'socket-1' }) + ) + expect(roomManager.broadcastPresenceUpdate).toHaveBeenCalledWith(TABLE_ROOM) + }) + + it('persists and relays a cell selection to the namespaced room (id + cell only)', async () => { + const { socket, handlers, toEmit } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue(TABLE_ROOM), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const cell = { + anchor: { rowId: 'row-1', columnId: 'col-a' }, + focus: { rowId: 'row-1', columnId: 'col-a' }, + editing: true, + } + await handlers[TABLE_PRESENCE_EVENTS.CELL_SELECTION]({ cell }) + + expect(roomManager.updateUserActivity).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1', { cell }) + // Namespaced room → broadcast targets roomName(room), not the bare id. + expect(socket.to).toHaveBeenCalledWith('table:table-1') + // The delta carries only the socket id + cell — identity comes from the roster. + expect(toEmit).toHaveBeenCalledWith(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { + socketId: 'socket-1', + cell, + }) + }) + + it('drops a malformed cell selection without storing or relaying it', async () => { + const { socket, handlers, toEmit } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue(TABLE_ROOM), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.CELL_SELECTION]({ cell: { anchor: 'x"]' } }) + + expect(roomManager.updateUserActivity).not.toHaveBeenCalled() + expect(toEmit).not.toHaveBeenCalled() + }) + + it('strips unknown/oversized fields from an otherwise-valid selection before storing or relaying', async () => { + const { socket, handlers, toEmit } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue(TABLE_ROOM), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.CELL_SELECTION]({ + cell: { + anchor: { rowId: 'row-1', columnId: 'col-a', junk: 'x'.repeat(10_000) }, + focus: { rowId: 'row-1', columnId: 'col-a' }, + editing: true, + bloat: 'x'.repeat(100_000), + }, + }) + + // Only the whitelisted fields survive — a hostile peer can't amplify an oversized object. + const expected = { + anchor: { rowId: 'row-1', columnId: 'col-a' }, + focus: { rowId: 'row-1', columnId: 'col-a' }, + editing: true, + } + expect(roomManager.updateUserActivity).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1', { + cell: expected, + }) + expect(toEmit).toHaveBeenCalledWith(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { + socketId: 'socket-1', + cell: expected, + }) + }) + + it('skips a superseded queued join on a fast table switch', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + // Two joins enqueued back-to-back (A then B). B bumps the generation synchronously, so A's + // queued run no-ops at its start check — only B commits. Because JOINs are serialized on one + // op chain, A's and B's Redis writes can never interleave (no map-clobber, no stranding). + handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-A' }) + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) + + expect(socket.join).toHaveBeenCalledWith('table:table-B') + expect(socket.join).not.toHaveBeenCalledWith('table:table-A') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-B' }, + 'socket-1', + expect.anything() + ) + expect(roomManager.addUserToRoom).not.toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-A' }, + expect.anything(), + expect.anything() + ) + }) + + it('aborts an in-flight join when a leave for that table arrives during authorize', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + let releaseAuth: (value: unknown) => void = () => {} + const pending = new Promise((resolve) => { + releaseAuth = resolve + }) + mockAuthorizeRoom.mockReturnValueOnce(pending) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const joinPromise = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + // Client navigates away while the join is still awaiting authorization. + await handlers[TABLE_PRESENCE_EVENTS.LEAVE]({ tableId: 'table-1' }) + releaseAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + // The cancelled join must touch no room state — the socket is not stranded. + expect(socket.join).not.toHaveBeenCalled() + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + expect(roomManager.broadcastPresenceUpdate).not.toHaveBeenCalledWith(TABLE_ROOM) + }) + + it('aborts an in-flight join when an unscoped leave arrives during authorize', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + let releaseAuth: (value: unknown) => void = () => {} + const pending = new Promise((resolve) => { + releaseAuth = resolve + }) + mockAuthorizeRoom.mockReturnValueOnce(pending) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const joinPromise = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + // A leave with no table id (view teardown) must also cancel the in-flight join. + await handlers[TABLE_PRESENCE_EVENTS.LEAVE](undefined) + releaseAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + expect(socket.join).not.toHaveBeenCalled() + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + }) + + it('does not abort an in-flight join when a leave targets a different table', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + let releaseAuth: (value: unknown) => void = () => {} + const pending = new Promise((resolve) => { + releaseAuth = resolve + }) + mockAuthorizeRoom.mockReturnValueOnce(pending) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + const joinPromise = handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-B' }) + // A stale/deferred leave for a table the client already left must NOT cancel the B join. + await handlers[TABLE_PRESENCE_EVENTS.LEAVE]({ tableId: 'table-A' }) + releaseAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + expect(socket.join).toHaveBeenCalledWith('table:table-B') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: ROOM_TYPES.TABLE, id: 'table-B' }, + 'socket-1', + expect.anything() + ) + }) + + it('leaves the table room on leave', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue(TABLE_ROOM), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.LEAVE]({ tableId: 'table-1' }) + + expect(socket.leave).toHaveBeenCalledWith('table:table-1') + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1') + expect(roomManager.broadcastPresenceUpdate).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1') + }) + + it('rolls back the Socket.IO membership when a join fails mid-commit', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + // socket.join lands first, then the presence write throws — the socket is now in the + // Socket.IO room with no matching socket→room map entry, unreclaimable by any later op. + addUserToRoom: vi.fn().mockRejectedValue(new Error('redis down')), + }) + setupTablesHandlers(socket as unknown as SetupArg, roomManager) + + await handlers[TABLE_PRESENCE_EVENTS.JOIN]({ tableId: 'table-1' }) + + // The catch must always roll back the partial membership, not skip it. + expect(socket.leave).toHaveBeenCalledWith('table:table-1') + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith(TABLE_ROOM, 'socket-1') + expect(socket.emit).toHaveBeenCalledWith( + TABLE_PRESENCE_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'JOIN_FAILED' }) + ) + }) +}) diff --git a/apps/realtime/src/handlers/tables.ts b/apps/realtime/src/handlers/tables.ts new file mode 100644 index 00000000000..2474063c9a0 --- /dev/null +++ b/apps/realtime/src/handlers/tables.ts @@ -0,0 +1,314 @@ +import { createLogger } from '@sim/logger' +import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import { + type JoinTablePayload, + TABLE_PRESENCE_EVENTS, + type TableCellRef, + type TableCellSelection, +} from '@sim/realtime-protocol/table-presence' +import { resolveAvatarUrl } from '@/handlers/avatar' +import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' +import type { AuthenticatedSocket } from '@/middleware/auth' +import type { IRoomManager, UserPresence } from '@/rooms' +import { filterVisiblePresence, sweepStalePresence } from '@/rooms/presence-visibility' + +const logger = createLogger('TablePresenceHandlers') + +/** Longest accepted row/column id — real ids are UUIDs/short ids; this bounds a hostile payload. */ +const MAX_CELL_ID_LENGTH = 200 + +/** The table presence room ref for a table id. */ +const tableRoom = (tableId: string): RoomRef => ({ type: ROOM_TYPES.TABLE, id: tableId }) + +function isCellRef(value: unknown): value is TableCellRef { + if (typeof value !== 'object' || value === null) return false + const ref = value as { rowId?: unknown; columnId?: unknown } + return ( + typeof ref.rowId === 'string' && + ref.rowId.length <= MAX_CELL_ID_LENGTH && + typeof ref.columnId === 'string' && + ref.columnId.length <= MAX_CELL_ID_LENGTH + ) +} + +/** + * Validate + whitelist an untrusted peer's selection before it is stored and + * rebroadcast (it ultimately flows into a DOM query on every viewer). Returns the + * normalized selection — `null` for a legitimately cleared selection — or `undefined` + * for anything malformed, so the caller drops it. Only the known fields survive, so a + * hostile client can't amplify an oversized object through the room. + */ +function normalizeCellSelection(cell: unknown): TableCellSelection | undefined { + if (cell === null) return null + if (typeof cell !== 'object') return undefined + const candidate = cell as { anchor?: unknown; focus?: unknown; editing?: unknown } + if (!isCellRef(candidate.anchor) || !isCellRef(candidate.focus)) return undefined + return { + anchor: { rowId: candidate.anchor.rowId, columnId: candidate.anchor.columnId }, + focus: { rowId: candidate.focus.rowId, columnId: candidate.focus.columnId }, + ...(candidate.editing === true ? { editing: true } : {}), + } +} + +/** + * Live cell-selection presence for the table grid. Mirrors the workspace-files + * join flow but is table-scoped (room id = tableId) with a bidirectional + * cell-selection channel — the grid analog of the workflow cursor/selection + * relay. Table *data* still flows through the one-way durable event stream + * (`lib/table/events.ts`); this socket carries only ephemeral presence. + * + * Table rooms are namespaced (`table:${id}`), so every broadcast targets + * `roomName(room)`, never the bare `room.id` (which the workflow handler can use + * only because a workflow room's name equals its id). + */ +export function setupTablesHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { + // Monotonic per-socket generation: each JOIN/LEAVE bumps it synchronously on arrival, and a + // queued or in-flight op that finds a newer generation aborts — a fast table switch A→B thus + // cancels A the instant B arrives. + let joinGeneration = 0 + // The table the socket currently intends to be in (set when a join is enqueued). A leave + // targeting it — or an unscoped leave — bumps the generation to cancel that join; a leave for a + // DIFFERENT table must NOT (a table switch), mirroring workspace-files. + let currentTableId: string | null = null + // Serialize this socket's room mutations (JOIN + LEAVE) so their multi-step async Redis commits + // can never interleave: two concurrent joins would otherwise race on the single-valued + // socket→room map (a late addUserToRoom clobbering a newer join's entry). This restores the + // atomic-commit property the synchronous sibling handlers (file-doc, workspace-files) get for + // free. CELL_SELECTION is NOT chained — it only touches presence activity, never the map. + let opChain: Promise = Promise.resolve() + + socket.on(TABLE_PRESENCE_EVENTS.JOIN, ({ tableId, tabSessionId }: JoinTablePayload) => { + // Validate the id BEFORE claiming a generation, so a malformed join can't advance + // joinGeneration and cancel a legitimate in-flight join for another table. + if (typeof tableId !== 'string' || tableId.length === 0) { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId: typeof tableId === 'string' ? tableId : '', + error: 'Invalid table id', + code: 'INVALID_PAYLOAD', + retryable: false, + }) + return + } + const joinAttempt = (joinGeneration += 1) + currentTableId = tableId + opChain = opChain + .then(() => runJoin(tableId, tabSessionId, joinAttempt)) + .catch((error) => logger.error('Error joining table room:', error)) + // Returned so callers awaiting this op (e.g. tests) can await its completion; Socket.IO + // ignores a handler's return value. + return opChain + }) + + async function runJoin(tableId: string, tabSessionId: string | undefined, joinAttempt: number) { + // True once this JOIN has been superseded — a newer JOIN/LEAVE bumped joinGeneration, or the + // socket disconnected. Because ops are serialized, no other op mutates room state while this + // one runs, so only two checks are needed: skip a superseded queued op (here), and one final + // check right before the membership commit. + const superseded = () => joinGeneration !== joinAttempt || socket.disconnected + if (superseded()) return + try { + const userId = socket.userId + const userName = socket.userName + + if (!userId || !userName) { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId, + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + return + } + + if (!roomManager.isReady()) { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId, + error: 'Realtime unavailable', + code: 'ROOM_MANAGER_UNAVAILABLE', + retryable: true, + }) + return + } + + const room = tableRoom(tableId) + + const authorized = await resolveRoomJoinAuth({ + userId, + room, + action: 'read', + logger, + logLabel: `table room for ${userId}`, + messages: { + verifyFailed: 'Failed to verify table access', + notFound: 'Table not found', + accessDenied: 'Access denied to table', + }, + emitError: ({ error, code, retryable }) => + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { tableId, error, code, retryable }), + }) + if (!authorized) return + + // Server-authenticated avatar for the presence roster. + const avatarUrl = await resolveAvatarUrl(socket, userId) + + // Leave a previously-joined table room if switching tables. No generation guard is needed + // around this: serialization guarantees no concurrent op committed to a different room + // during the lookup, so `currentRoom` is the socket's genuine prior room, safe to leave. + const currentRoom = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) + if (currentRoom && currentRoom.id !== tableId) { + socket.leave(roomName(currentRoom)) + await roomManager.removeUserFromRoom(currentRoom, socket.id) + await roomManager.broadcastPresenceUpdate(currentRoom) + } + + // Reclaim presence orphaned by an ungraceful disconnect (no `disconnecting` + // event fires on a pod crash; the room hashes have no TTL). Returns the roster it + // read so the same-tab dedup below reuses it instead of issuing a second read. + const roster = await sweepStalePresence(roomManager, room) + + // Clean up the same user's stale socket from the same tab (a reconnect that raced + // the old socket's disconnect), so presence shows one entry. Reuses the sweep's + // roster snapshot; re-removing an already-swept entry is a harmless no-op. + if (tabSessionId) { + for (const existing of roster) { + if ( + existing.socketId !== socket.id && + existing.userId === userId && + existing.tabSessionId === tabSessionId + ) { + await roomManager.removeUserFromRoom(room, existing.socketId) + await roomManager.io.in(existing.socketId).socketsLeave(roomName(room)) + } + } + } + + // Final re-check before the membership commit: a LEAVE or a newer JOIN enqueued during the + // awaits above bumped the generation, or the socket disconnected. Abort before registering. + if (superseded()) return + + socket.join(roomName(room)) + + const presence: UserPresence = { + userId, + room, + userName, + socketId: socket.id, + tabSessionId, + joinedAt: Date.now(), + lastActivity: Date.now(), + role: authorized.workspacePermission ?? 'read', + avatarUrl, + } + + // If the socket disconnects during this commit (disconnect cleanup runs off the op chain), + // this write can land after it, leaving a stale presence entry. Benign and self-correcting: + // filterVisiblePresence hides it and sweepStalePresence reclaims it (same as the siblings). + await roomManager.addUserToRoom(room, socket.id, presence) + + // Filter the join ack to live members so a new joiner never briefly sees a + // ghost from an entry the sweep hasn't reclaimed yet. + const presenceUsers = await filterVisiblePresence( + roomManager.io, + room, + await roomManager.getRoomUsers(room) + ) + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, { + tableId, + socketId: socket.id, + presenceUsers, + }) + + // Post-success, purely decorative: notify peers. The user is already joined and acked, so a + // Redis blip here must not surface as a join failure — swallow it (the next healthy broadcast + // reconciles peers). Kept OUT of the rollback catch below, which is only for pre-success failures. + try { + await roomManager.broadcastPresenceUpdate(room) + } catch (error) { + logger.warn(`Post-join presence broadcast failed for table room ${tableId}`, error) + } + + logger.info(`User ${userId} (${userName}) joined table room ${tableId}`) + } catch (error) { + logger.error('Error joining table room:', error) + // Roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` that + // landed without a matching `addUserToRoom` (a throw in between) would otherwise leave the + // socket stranded in the Socket.IO room, unreclaimable by any later op. A failure between the + // commit and the success ack rolls back too and surfaces a retryable error, so the client + // retries rather than hanging. Safe to run even when superseded — serialization means the + // newer op hasn't committed yet, so this touches only this join's own (this-table) state. + try { + const room = tableRoom(tableId) + socket.leave(roomName(room)) + await roomManager.removeUserFromRoom(room, socket.id) + } catch { + // Best-effort rollback — the original join failure is the one surfaced below, so a + // secondary cleanup error must not mask it or throw out of the error handler. + } + // Suppress the client-facing error when this join was already superseded: the client has moved + // to a newer table, and a retryable error naming the abandoned one could make it re-join and + // supersede the newer join. The rollback above still runs. + if (superseded()) return + socket.emit(TABLE_PRESENCE_EVENTS.JOIN_ERROR, { + tableId, + error: 'Failed to join table', + code: 'JOIN_FAILED', + retryable: true, + }) + } + } + + socket.on(TABLE_PRESENCE_EVENTS.LEAVE, (payload?: { tableId?: string }) => { + // Cancel an in-flight/queued join whose table the client is now leaving (or an unscoped + // leave). Scope to the current table intent so a stale/deferred leave for a DIFFERENT table + // can't cancel the join the client has since switched to. Bumped synchronously here — before + // the teardown is enqueued — so it cancels a running join at its next generation check. + if (!payload?.tableId || payload.tableId === currentTableId) { + joinGeneration += 1 + currentTableId = null + } + opChain = opChain + .then(() => runLeave(payload)) + .catch((error) => logger.error('Error leaving table room:', error)) + return opChain + }) + + async function runLeave(payload?: { tableId?: string }) { + try { + if (!roomManager.isReady()) return + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) + if (!room) return + // Scope the leave to a specific table when the client provides one: a deferred leave from a + // prior view must not evict the socket from a room it has since switched into. + if (payload?.tableId && payload.tableId !== room.id) return + socket.leave(roomName(room)) + await roomManager.removeUserFromRoom(room, socket.id) + await roomManager.broadcastPresenceUpdate(room, socket.id) + } catch (error) { + logger.error('Error leaving table room:', error) + } + } + + socket.on(TABLE_PRESENCE_EVENTS.CELL_SELECTION, async ({ cell }: { cell: unknown }) => { + try { + // Drop a malformed/oversized selection from an untrusted peer before it is stored + // or rebroadcast (`undefined` = invalid; `null` = a legitimately cleared selection). + const selection = normalizeCellSelection(cell) + if (selection === undefined) return + + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.TABLE) + if (!room) return + + // Persist so a later joiner sees this viewer's current selection in the join ack. + await roomManager.updateUserActivity(room, socket.id, { cell: selection }) + + // Relay to peers (namespaced room → roomName, not room.id). Peers already know this + // socket's identity from the presence roster, so the delta carries only id + cell. + socket.to(roomName(room)).emit(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { + socketId: socket.id, + cell: selection, + }) + } catch (error) { + logger.error(`Error handling table cell selection for socket ${socket.id}:`, error) + } + }) +} diff --git a/apps/realtime/src/handlers/variables.ts b/apps/realtime/src/handlers/variables.ts index 7a4303e70b3..0d6eaac2e3a 100644 --- a/apps/realtime/src/handlers/variables.ts +++ b/apps/realtime/src/handlers/variables.ts @@ -4,11 +4,12 @@ import { workflow } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { assertWorkflowMutable, WorkflowLockedError } from '@sim/platform-authz/workflow' import { VARIABLE_OPERATIONS } from '@sim/realtime-protocol/constants' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { getErrorMessage } from '@sim/utils/errors' import { eq } from 'drizzle-orm' import type { AuthenticatedSocket } from '@/middleware/auth' import { checkWorkflowOperationPermission } from '@/middleware/permissions' -import type { IRoomManager } from '@/rooms' +import { type IRoomManager, workflowRoom as wf } from '@/rooms' const logger = createLogger('VariablesHandlers') @@ -61,7 +62,8 @@ export function setupVariablesHandlers(socket: AuthenticatedSocket, roomManager: } try { - const sessionWorkflowId = await roomManager.getWorkflowIdForSocket(socket.id) + const sessionWorkflowId = + (await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW))?.id ?? null const session = await roomManager.getUserSession(socket.id) if (!sessionWorkflowId || !session) { @@ -98,7 +100,7 @@ export function setupVariablesHandlers(socket: AuthenticatedSocket, roomManager: return } - const hasRoom = await roomManager.hasWorkflowRoom(workflowId) + const hasRoom = await roomManager.hasRoom(wf(workflowId)) if (!hasRoom) { logger.debug(`Ignoring variable update: workflow room not found`, { socketId: socket.id, @@ -109,7 +111,7 @@ export function setupVariablesHandlers(socket: AuthenticatedSocket, roomManager: return } - const users = await roomManager.getWorkflowUsers(workflowId) + const users = await roomManager.getRoomUsers(wf(workflowId)) const userPresence = users.find((user) => user.socketId === socket.id) if (!userPresence) { socket.emit('operation-forbidden', { @@ -174,7 +176,7 @@ export function setupVariablesHandlers(socket: AuthenticatedSocket, roomManager: } // Update user activity - await roomManager.updateUserActivity(workflowId, socket.id, { lastActivity: Date.now() }) + await roomManager.updateUserActivity(wf(workflowId), socket.id, { lastActivity: Date.now() }) const debouncedKey = `${workflowId}:${variableId}:${field}` const existing = pendingVariableUpdates.get(debouncedKey) diff --git a/apps/realtime/src/handlers/workflow.test.ts b/apps/realtime/src/handlers/workflow.test.ts index ad649f88f53..552db8125c5 100644 --- a/apps/realtime/src/handlers/workflow.test.ts +++ b/apps/realtime/src/handlers/workflow.test.ts @@ -4,12 +4,21 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { IRoomManager } from '@/rooms' -const { mockGetWorkflowState, mockVerifyWorkflowAccess, mockResolveCurrentWorkflowRole } = - vi.hoisted(() => ({ - mockGetWorkflowState: vi.fn(), - mockVerifyWorkflowAccess: vi.fn(), - mockResolveCurrentWorkflowRole: vi.fn(), - })) +const { + mockGetWorkflowState, + mockVerifyWorkflowAccess, + mockResolveCurrentWorkflowRole, + mockResolveAvatarUrl, +} = vi.hoisted(() => ({ + mockGetWorkflowState: vi.fn(), + mockVerifyWorkflowAccess: vi.fn(), + mockResolveCurrentWorkflowRole: vi.fn(), + mockResolveAvatarUrl: vi.fn(), +})) + +vi.mock('@/handlers/avatar', () => ({ + resolveAvatarUrl: mockResolveAvatarUrl, +})) vi.mock('@sim/db', () => ({ db: { select: vi.fn() }, @@ -33,13 +42,14 @@ interface JoinWorkflowPayload { } function createSocket(overrides?: Partial>) { - const handlers: Record Promise | void> = {} + // leave-workflow takes no payload; join-workflow takes one — so the stored handler's arg is optional. + const handlers: Record Promise | void> = {} const socket = { id: 'socket-1', userId: 'user-1', userName: 'Test User', userImage: 'avatar.png', - on: vi.fn((event: string, handler: (payload: JoinWorkflowPayload) => Promise | void) => { + on: vi.fn((event: string, handler: (payload?: JoinWorkflowPayload) => Promise | void) => { handlers[event] = handler }), emit: vi.fn(), @@ -57,21 +67,21 @@ function createSocket(overrides?: Partial>) { function createRoomManager(overrides?: Partial): IRoomManager { return { isReady: vi.fn().mockReturnValue(true), - getWorkflowIdForSocket: vi.fn().mockResolvedValue(null), - removeUserFromRoom: vi.fn().mockResolvedValue(null), + getRoomForSocket: vi.fn().mockResolvedValue(null), + getRoomsForSocket: vi.fn().mockResolvedValue([]), + removeUserFromRoom: vi.fn().mockResolvedValue(false), + removeSocketFromAllRooms: vi.fn().mockResolvedValue([]), broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), - getWorkflowUsers: vi.fn().mockResolvedValue([]), - hasWorkflowRoom: vi.fn().mockResolvedValue(false), + getRoomUsers: vi.fn().mockResolvedValue([]), + hasRoom: vi.fn().mockResolvedValue(false), + deleteRoom: vi.fn().mockResolvedValue(undefined), addUserToRoom: vi.fn().mockResolvedValue(undefined), getUserSession: vi.fn().mockResolvedValue(null), updateUserActivity: vi.fn().mockResolvedValue(undefined), updateRoomLastModified: vi.fn().mockResolvedValue(undefined), - emitToWorkflow: vi.fn(), + emitToRoom: vi.fn(), getUniqueUserCount: vi.fn().mockResolvedValue(1), getTotalActiveConnections: vi.fn().mockResolvedValue(0), - handleWorkflowDeletion: vi.fn().mockResolvedValue(undefined), - handleWorkflowRevert: vi.fn().mockResolvedValue(undefined), - handleWorkflowUpdate: vi.fn().mockResolvedValue(undefined), shutdown: vi.fn().mockResolvedValue(undefined), initialize: vi.fn().mockResolvedValue(undefined), io: { @@ -90,6 +100,36 @@ describe('setupWorkflowHandlers', () => { mockGetWorkflowState.mockResolvedValue({ id: 'workflow-1', state: {} }) mockVerifyWorkflowAccess.mockResolvedValue({ hasAccess: true, role: 'admin' }) mockResolveCurrentWorkflowRole.mockResolvedValue('admin') + mockResolveAvatarUrl.mockResolvedValue('avatar.png') + }) + + it('resolves the avatar before joining so no await sits between socket.join and addUserToRoom', async () => { + const order: string[] = [] + mockResolveAvatarUrl.mockImplementation(async () => { + order.push('avatar') + return 'avatar.png' + }) + const { socket, handlers } = createSocket({ + join: vi.fn(() => { + order.push('join') + }), + }) + const roomManager = createRoomManager({ + addUserToRoom: vi.fn(async () => { + order.push('add') + }), + }) + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + + // The avatar await must complete before socket.join; reintroducing it between + // join and addUserToRoom reopens the revoke-race ghost-presence window. + expect(order).toEqual(['avatar', 'join', 'add']) }) it('includes workflowId when authentication is missing', async () => { @@ -193,7 +233,7 @@ describe('setupWorkflowHandlers', () => { expect(mockResolveCurrentWorkflowRole).toHaveBeenCalledWith('user-1', 'workflow-1', 'write') expect(socket.join).toHaveBeenCalledWith('workflow-1') expect(roomManager.addUserToRoom).toHaveBeenCalledWith( - 'workflow-1', + { type: 'workflow', id: 'workflow-1' }, 'socket-1', expect.objectContaining({ role: 'read' }) ) @@ -223,8 +263,8 @@ describe('setupWorkflowHandlers', () => { it('includes workflowId when an unexpected join failure occurs', async () => { const { socket, handlers } = createSocket() const roomManager = createRoomManager({ - getWorkflowIdForSocket: vi.fn().mockRejectedValue(new Error('boom')), - removeUserFromRoom: vi.fn().mockResolvedValue(null), + getRoomForSocket: vi.fn().mockRejectedValue(new Error('boom')), + removeUserFromRoom: vi.fn().mockResolvedValue(false), }) setupWorkflowHandlers( @@ -241,4 +281,174 @@ describe('setupWorkflowHandlers', () => { retryable: true, }) }) + + it('cancels a superseded queued join on a fast workflow switch', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + // Enqueue A without awaiting, then B: B bumps the generation synchronously, so A is superseded + // before its queued op runs and must never commit. + handlers['join-workflow']({ workflowId: 'workflow-a', tabSessionId: 'tab-1' }) + await handlers['join-workflow']({ workflowId: 'workflow-b', tabSessionId: 'tab-1' }) + + expect(socket.join).toHaveBeenCalledWith('workflow-b') + expect(socket.join).not.toHaveBeenCalledWith('workflow-a') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-b' }, + 'socket-1', + expect.anything() + ) + expect(roomManager.addUserToRoom).not.toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-a' }, + 'socket-1', + expect.anything() + ) + }) + + it('does not let a malformed join cancel a valid in-flight join', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + const validJoin = handlers['join-workflow']({ workflowId: 'workflow-a', tabSessionId: 'tab-1' }) + // A malformed join arrives mid-flight — it must be rejected WITHOUT advancing the generation, + // so it can't supersede the valid join already in flight. + handlers['join-workflow']({ workflowId: '', tabSessionId: 'tab-1' }) + await validJoin + + expect(socket.emit).toHaveBeenCalledWith( + 'join-workflow-error', + expect.objectContaining({ code: 'INVALID_PAYLOAD' }) + ) + // The valid join still committed — not superseded by the malformed one. + expect(socket.join).toHaveBeenCalledWith('workflow-a') + expect(roomManager.addUserToRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-a' }, + 'socket-1', + expect.anything() + ) + }) + + it('cancels an in-flight join when a leave is enqueued before it commits', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + await handlers['leave-workflow']() + + expect(socket.join).not.toHaveBeenCalled() + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + }) + + it('rolls back the workflow membership when addUserToRoom fails mid-commit', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + addUserToRoom: vi.fn().mockRejectedValue(new Error('redis down')), + }) + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + + expect(socket.leave).toHaveBeenCalledWith('workflow-1') + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-1' }, + 'socket-1' + ) + expect(socket.emit).toHaveBeenCalledWith( + 'join-workflow-error', + expect.objectContaining({ code: 'JOIN_WORKFLOW_FAILED' }) + ) + }) + + it('does not roll back a committed join when a post-success step fails', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + // Trailing broadcast (post-addUserToRoom, post-success-ack) fails on a Redis blip. + broadcastPresenceUpdate: vi.fn().mockRejectedValue(new Error('redis blip')), + }) + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + + // The user is genuinely joined and was acked; the trailing failure must NOT tear them out. + expect(socket.emit).toHaveBeenCalledWith( + 'join-workflow-success', + expect.objectContaining({ workflowId: 'workflow-1' }) + ) + expect(socket.leave).not.toHaveBeenCalled() + expect(roomManager.removeUserFromRoom).not.toHaveBeenCalled() + expect(socket.emit).not.toHaveBeenCalledWith('join-workflow-error', expect.anything()) + }) + + it('rolls back and surfaces a retryable error when a pre-success step fails after commit', async () => { + // getWorkflowState runs after addUserToRoom but before the success ack — its failure must roll + // back and emit a retryable error so the client retries, never hanging committed-but-unacked. + mockGetWorkflowState.mockRejectedValue(new Error('db blip')) + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['join-workflow']({ workflowId: 'workflow-1', tabSessionId: 'tab-1' }) + + expect(socket.emit).not.toHaveBeenCalledWith('join-workflow-success', expect.anything()) + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-1' }, + 'socket-1' + ) + expect(socket.emit).toHaveBeenCalledWith( + 'join-workflow-error', + expect.objectContaining({ code: 'JOIN_WORKFLOW_FAILED', retryable: true }) + ) + }) + + it('leaves the workflow room even when the session key has expired', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager({ + getRoomForSocket: vi.fn().mockResolvedValue({ type: 'workflow', id: 'workflow-1' }), + getUserSession: vi.fn().mockResolvedValue(null), + }) + + setupWorkflowHandlers( + socket as unknown as Parameters[0], + roomManager + ) + + await handlers['leave-workflow']() + + expect(socket.leave).toHaveBeenCalledWith('workflow-1') + expect(roomManager.removeUserFromRoom).toHaveBeenCalledWith( + { type: 'workflow', id: 'workflow-1' }, + 'socket-1' + ) + expect(roomManager.broadcastPresenceUpdate).toHaveBeenCalledWith({ + type: 'workflow', + id: 'workflow-1', + }) + }) }) diff --git a/apps/realtime/src/handlers/workflow.ts b/apps/realtime/src/handlers/workflow.ts index 2c00895b201..523c3747779 100644 --- a/apps/realtime/src/handlers/workflow.ts +++ b/apps/realtime/src/handlers/workflow.ts @@ -1,15 +1,58 @@ -import { db, user } from '@sim/db' import { createLogger } from '@sim/logger' -import { eq } from 'drizzle-orm' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { getWorkflowState } from '@/database/operations' +import { resolveAvatarUrl } from '@/handlers/avatar' import type { AuthenticatedSocket } from '@/middleware/auth' import { resolveCurrentWorkflowRole, verifyWorkflowAccess } from '@/middleware/permissions' -import type { IRoomManager, UserPresence } from '@/rooms' +import { type IRoomManager, type UserPresence, workflowRoom as wf } from '@/rooms' +import { filterVisiblePresence } from '@/rooms/presence-visibility' const logger = createLogger('WorkflowHandlers') export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { - socket.on('join-workflow', async ({ workflowId, tabSessionId }) => { + // Monotonic per-socket generation: each JOIN/LEAVE bumps it synchronously on arrival, and a + // queued or in-flight op that finds a newer generation aborts — a fast workflow switch A→B thus + // cancels A the instant B arrives. + let joinGeneration = 0 + // Serialize this socket's room mutations (JOIN + LEAVE) so their multi-step async Redis commits + // can never interleave: two concurrent joins would otherwise race on the single-valued + // socket→room map (a late addUserToRoom clobbering a newer join's entry, leaving the socket a + // ghost in the old room and receiving its operation broadcasts). This matches the sibling + // handlers (tables, file-doc, workspace-files). + let opChain: Promise = Promise.resolve() + + socket.on('join-workflow', ({ workflowId, tabSessionId }) => { + // Validate the id BEFORE claiming a generation, so a malformed join can't advance joinGeneration + // and cancel a legitimate in-flight switch (matches tables/workspace-files). + if (typeof workflowId !== 'string' || workflowId.length === 0) { + socket.emit('join-workflow-error', { + workflowId: typeof workflowId === 'string' ? workflowId : '', + error: 'Invalid workflow id', + code: 'INVALID_PAYLOAD', + retryable: false, + }) + return + } + const joinAttempt = (joinGeneration += 1) + opChain = opChain + .then(() => runJoin(workflowId, tabSessionId, joinAttempt)) + .catch((error) => logger.error('Error joining workflow:', error)) + // Returned so callers awaiting this op (e.g. tests) can await its completion; Socket.IO + // ignores a handler's return value. + return opChain + }) + + async function runJoin( + workflowId: string, + tabSessionId: string | undefined, + joinAttempt: number + ) { + // True once this JOIN has been superseded — a newer JOIN/LEAVE bumped joinGeneration, or the + // socket disconnected. Because ops are serialized, no other op mutates room state while this + // one runs, so only two checks are needed: skip a superseded queued op (here), and one final + // check right before the membership commit. + const superseded = () => joinGeneration !== joinAttempt || socket.disconnected + if (superseded()) return try { const userId = socket.userId const userName = socket.userName @@ -64,18 +107,19 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: return } - // Leave current room if in one - const currentWorkflowId = await roomManager.getWorkflowIdForSocket(socket.id) - if (currentWorkflowId) { - socket.leave(currentWorkflowId) - await roomManager.removeUserFromRoom(socket.id, currentWorkflowId) - await roomManager.broadcastPresenceUpdate(currentWorkflowId) + // Leave a previously-joined workflow room if switching workflows. Guard on a DIFFERENT id so a + // re-join of the SAME workflow doesn't leave→re-add and flicker presence for peers. + const currentRoom = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) + if (currentRoom && currentRoom.id !== workflowId) { + socket.leave(currentRoom.id) + await roomManager.removeUserFromRoom(currentRoom, socket.id) + await roomManager.broadcastPresenceUpdate(currentRoom) } // Keep this above Redis socket key TTL (1h) so a normal idle user is not evicted too aggressively. const STALE_THRESHOLD_MS = 75 * 60 * 1000 const now = Date.now() - const existingUsers = await roomManager.getWorkflowUsers(workflowId) + const existingUsers = await roomManager.getRoomUsers(wf(workflowId)) let liveSocketIds = new Set() let canCheckLiveness = false @@ -106,7 +150,7 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: logger.info( `Cleaning up socket ${existingUser.socketId} for user ${existingUser.userId} (same tab)` ) - await roomManager.removeUserFromRoom(existingUser.socketId, workflowId) + await roomManager.removeUserFromRoom(wf(workflowId), existingUser.socketId) await roomManager.io.in(existingUser.socketId).socketsLeave(workflowId) continue } @@ -124,21 +168,30 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: logger.info( `Cleaning up socket ${existingUser.socketId} for user ${existingUser.userId} (stale activity)` ) - await roomManager.removeUserFromRoom(existingUser.socketId, workflowId) + await roomManager.removeUserFromRoom(wf(workflowId), existingUser.socketId) await roomManager.io.in(existingUser.socketId).socketsLeave(workflowId) } catch (error) { logger.warn(`Best-effort cleanup failed for socket ${existingUser.socketId}`, error) } } + // Resolve the avatar before the critical section below. It is the only + // await that used to sit between socket.join and addUserToRoom, and a sweep + // eviction in that gap would socketsLeave the socket while its presence + // mapping did not yet exist — cleanupEvictedSocket would find nothing to + // remove, then this join would write presence for a socket already out of + // the room (a ghost collaborator until the stale sweep). Hoisting it keeps + // the whole re-auth -> socket.join -> addUserToRoom section await-free. + const avatarUrl = await resolveAvatarUrl(socket, userId) + // Re-authorize immediately before joining: the access-revalidation sweep // may have evicted this socket while the awaits above were in flight, and // its eviction is recorded in the shared role cache before it runs — so a // revoked user resolves to null here. The resolver is single-flighted per // (user, workflow), so this read cannot race the sweep's and overwrite a // recorded revocation with a stale role; and no awaits sit between this - // check and socket.join, so a sweep eviction cannot interleave after it - // and be reversed by this join. + // check and addUserToRoom (avatar resolution is hoisted above), so a sweep + // eviction cannot interleave inside the join and be reversed by it. const currentRole = await resolveCurrentWorkflowRole(userId, workflowId, userRole) if (currentRole === null) { logger.warn( @@ -154,29 +207,19 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: } userRole = currentRole + // Final re-check before the membership commit: a LEAVE or a newer JOIN enqueued during the + // awaits above bumped the generation, or the socket disconnected. Abort before registering. + // (This guards against a superseding op; the avatar hoist above guards against the off-chain + // access-revalidation sweep, which does not bump the generation.) + if (superseded()) return + // Join the new room socket.join(workflowId) - // Get avatar URL - let avatarUrl = socket.userImage || null - if (!avatarUrl) { - try { - const [userRecord] = await db - .select({ image: user.image }) - .from(user) - .where(eq(user.id, userId)) - .limit(1) - - avatarUrl = userRecord?.image ?? null - } catch (error) { - logger.warn('Failed to load user avatar for presence', { userId, error }) - } - } - // Create presence entry const userPresence: UserPresence = { userId, - workflowId, + room: wf(workflowId), userName, socketId: socket.id, tabSessionId, @@ -186,11 +229,16 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: avatarUrl, } - // Add user to room - await roomManager.addUserToRoom(workflowId, socket.id, userPresence) + // Add user to room — the membership commit. + await roomManager.addUserToRoom(wf(workflowId), socket.id, userPresence) - // Get current presence list for the join acknowledgment - const presenceUsers = await roomManager.getWorkflowUsers(workflowId) + // Get current presence list for the join acknowledgment, filtered to live members so a new + // joiner never sees a ghost from an entry the stale sweep hasn't reclaimed yet. + const presenceUsers = await filterVisiblePresence( + roomManager.io, + wf(workflowId), + await roomManager.getRoomUsers(wf(workflowId)) + ) // Get workflow state const workflowState = await getWorkflowState(workflowId) @@ -205,18 +253,34 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: // Send workflow state socket.emit('workflow-state', workflowState) - // Broadcast presence update to all users in the room - await roomManager.broadcastPresenceUpdate(workflowId) - - const uniqueUserCount = await roomManager.getUniqueUserCount(workflowId) - logger.info( - `User ${userId} (${userName}) joined workflow ${workflowId}. Room now has ${uniqueUserCount} unique users.` - ) + // Post-success, purely decorative: notify peers and log the count. The user is already joined + // and acked, so a Redis blip here must not surface as a join failure — swallow it (the next + // healthy presence broadcast reconciles peers). It must stay OUT of the rollback catch below, + // which is only for pre-success failures. + try { + await roomManager.broadcastPresenceUpdate(wf(workflowId)) + const uniqueUserCount = await roomManager.getUniqueUserCount(wf(workflowId)) + logger.info( + `User ${userId} (${userName}) joined workflow ${workflowId}. Room now has ${uniqueUserCount} unique users.` + ) + } catch (error) { + logger.warn(`Post-join presence broadcast failed for workflow ${workflowId}`, error) + } } catch (error) { logger.error('Error joining workflow:', error) - // Undo socket.join and room manager entry if any operation failed + // Roll back a partial join: cleanup keys off the socket→room map, so a `socket.join` that + // landed without a matching `addUserToRoom` (a throw in between) would otherwise strand the + // socket in the Socket.IO room, unreclaimable by any later op. A failure between the commit + // and the success ack rolls back too and surfaces a retryable error — so the client retries + // rather than hanging (never left committed-but-unacked). Safe even when superseded — + // serialization means the newer op hasn't committed yet, so this touches only this join's own + // room state, never the newer op's. socket.leave(workflowId) - await roomManager.removeUserFromRoom(socket.id, workflowId) + await roomManager.removeUserFromRoom(wf(workflowId), socket.id) + // Suppress the client-facing error when this join was already superseded: the client has moved + // to a newer workflow, and a retryable error naming the abandoned one could make it re-join and + // supersede the newer join (an A/B flicker). The rollback above still runs. + if (superseded()) return const isReady = roomManager.isReady() socket.emit('join-workflow-error', { workflowId, @@ -225,26 +289,33 @@ export function setupWorkflowHandlers(socket: AuthenticatedSocket, roomManager: retryable: true, }) } + } + + socket.on('leave-workflow', () => { + // A leave always cancels any in-flight/queued join for this socket (the client emits it with no + // payload — there is no partial-switch case as there is for tables). Bumped synchronously here, + // before the teardown is enqueued, so it cancels a running join at its next generation check. + joinGeneration += 1 + opChain = opChain + .then(() => runLeave()) + .catch((error) => logger.error('Error leaving workflow:', error)) + return opChain }) - socket.on('leave-workflow', async () => { + async function runLeave() { try { - if (!roomManager.isReady()) { - return - } - - const workflowId = await roomManager.getWorkflowIdForSocket(socket.id) - const session = await roomManager.getUserSession(socket.id) - - if (workflowId && session) { - socket.leave(workflowId) - await roomManager.removeUserFromRoom(socket.id, workflowId) - await roomManager.broadcastPresenceUpdate(workflowId) - - logger.info(`User ${session.userId} (${session.userName}) left workflow ${workflowId}`) - } + if (!roomManager.isReady()) return + const room = await roomManager.getRoomForSocket(socket.id, ROOM_TYPES.WORKFLOW) + // The room ref alone is sufficient to leave; no session lookup is gated in front of it, so an + // idle user whose 1h session key expired (while the 24h room mapping is still live) can still + // leave cleanly instead of being stranded as a ghost until disconnect. + if (!room) return + socket.leave(room.id) + await roomManager.removeUserFromRoom(room, socket.id) + await roomManager.broadcastPresenceUpdate(room) + logger.info(`User ${socket.userId} (${socket.userName}) left workflow ${room.id}`) } catch (error) { logger.error('Error leaving workflow:', error) } - }) + } } diff --git a/apps/realtime/src/handlers/workspace-invalidation-room.test.ts b/apps/realtime/src/handlers/workspace-invalidation-room.test.ts new file mode 100644 index 00000000000..643fca696f0 --- /dev/null +++ b/apps/realtime/src/handlers/workspace-invalidation-room.test.ts @@ -0,0 +1,223 @@ +/** + * @vitest-environment node + */ +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { IRoomManager } from '@/rooms' + +const { mockAuthorizeRoom } = vi.hoisted(() => ({ + mockAuthorizeRoom: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { select: vi.fn() }, + user: { image: 'image' }, +})) + +vi.mock('@sim/platform-authz/rooms', () => ({ + authorizeRoom: mockAuthorizeRoom, +})) + +import { setupWorkspaceInvalidationRoom } from '@/handlers/workspace-invalidation-room' + +type Payload = { workspaceId?: string } + +function createSocket(overrides?: Record) { + const handlers: Record Promise | void> = {} + // Live Set so the handler's native `socket.rooms` membership tracking works in tests. + const rooms = new Set() + const socket = { + id: 'socket-1', + userId: 'user-1', + userName: 'Test User', + userImage: 'avatar.png', + rooms, + on: vi.fn((event: string, handler: (payload?: Payload) => Promise | void) => { + handlers[event] = handler + }), + emit: vi.fn(), + join: vi.fn((room: string) => rooms.add(room)), + leave: vi.fn((room: string) => rooms.delete(room)), + to: vi.fn().mockReturnValue({ emit: vi.fn() }), + ...overrides, + } + return { handlers, socket, rooms } +} + +function createRoomManager(overrides?: Partial): IRoomManager { + return { + isReady: vi.fn().mockReturnValue(true), + getRoomForSocket: vi.fn().mockResolvedValue(null), + getRoomsForSocket: vi.fn().mockResolvedValue([]), + removeUserFromRoom: vi.fn().mockResolvedValue(false), + removeSocketFromAllRooms: vi.fn().mockResolvedValue([]), + broadcastPresenceUpdate: vi.fn().mockResolvedValue(undefined), + getRoomUsers: vi.fn().mockResolvedValue([]), + hasRoom: vi.fn().mockResolvedValue(false), + deleteRoom: vi.fn().mockResolvedValue(undefined), + addUserToRoom: vi.fn().mockResolvedValue(undefined), + getUserSession: vi.fn().mockResolvedValue(null), + updateUserActivity: vi.fn().mockResolvedValue(undefined), + updateRoomLastModified: vi.fn().mockResolvedValue(undefined), + emitToRoom: vi.fn(), + getUniqueUserCount: vi.fn().mockResolvedValue(1), + getTotalActiveConnections: vi.fn().mockResolvedValue(0), + shutdown: vi.fn().mockResolvedValue(undefined), + initialize: vi.fn().mockResolvedValue(undefined), + io: { + in: vi.fn().mockReturnValue({ socketsLeave: vi.fn().mockResolvedValue(undefined) }), + }, + ...overrides, + } as unknown as IRoomManager +} + +// The two presence-free live-list rooms share one implementation; run the whole suite against both +// so files and tables can never drift. Event names and room names derive from the room type. +describe.each([ROOM_TYPES.WORKSPACE_FILES, ROOM_TYPES.WORKSPACE_TABLES] as const)( + 'setupWorkspaceInvalidationRoom(%s)', + (roomType) => { + const joinEvent = `join-${roomType}` + const successEvent = `${joinEvent}-success` + const errorEvent = `${joinEvent}-error` + const leaveEvent = `leave-${roomType}` + const roomOf = (workspaceId: string) => `${roomType}:${workspaceId}` + + const setup = (socket: ReturnType['socket'], roomManager: IRoomManager) => + setupWorkspaceInvalidationRoom( + socket as unknown as Parameters[0], + roomManager, + roomType + ) + + beforeEach(() => { + vi.clearAllMocks() + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', + }) + }) + + it('rejects join when the socket is not authenticated', async () => { + const { socket, handlers } = createSocket({ userId: undefined, userName: undefined }) + setup(socket, createRoomManager()) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith(errorEvent, { + workspaceId: 'ws-1', + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + }) + + it('rejects join with a retryable error when realtime is unavailable', async () => { + const { socket, handlers } = createSocket() + setup(socket, createRoomManager({ isReady: vi.fn().mockReturnValue(false) })) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + errorEvent, + expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) + ) + }) + + it('rejects join when workspace access is denied', async () => { + mockAuthorizeRoom.mockResolvedValue({ + allowed: false, + status: 403, + workspaceId: 'ws-1', + workspacePermission: null, + }) + const { socket, handlers } = createSocket() + setup(socket, createRoomManager()) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + errorEvent, + expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) + ) + }) + + it('joins the room on success without any presence bookkeeping', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + setup(socket, roomManager) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.join).toHaveBeenCalledWith(roomOf('ws-1')) + expect(socket.emit).toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-1' }) + // The room is live-list-only: no room-manager presence is tracked or broadcast. + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + expect(roomManager.broadcastPresenceUpdate).not.toHaveBeenCalled() + }) + + it('leaves a previously-joined room when switching workspaces', async () => { + const { socket, handlers, rooms } = createSocket() + rooms.add(roomOf('ws-old')) + setup(socket, createRoomManager()) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.leave).toHaveBeenCalledWith(roomOf('ws-old')) + expect(socket.join).toHaveBeenCalledWith(roomOf('ws-1')) + }) + + it('leaves the scoped room on leave', () => { + const { socket, handlers, rooms } = createSocket() + rooms.add(roomOf('ws-1')) + setup(socket, createRoomManager()) + + handlers[leaveEvent]({ workspaceId: 'ws-1' }) + + expect(socket.leave).toHaveBeenCalledWith(roomOf('ws-1')) + }) + + it('cancels an in-flight join when the user leaves that workspace mid-authorize', async () => { + const { socket, handlers } = createSocket() + let resolveAuth: (value: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve + }) + ) + setup(socket, createRoomManager()) + + // Join ws-1 is awaiting authorization when the view unmounts and leaves ws-1. + const joinPromise = handlers[joinEvent]({ workspaceId: 'ws-1' }) + handlers[leaveEvent]({ workspaceId: 'ws-1' }) + resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + // The stale join must NOT join the room the client has since left (no stranded membership). + expect(socket.join).not.toHaveBeenCalled() + expect(socket.emit).not.toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-1' }) + }) + + it('does not cancel an in-flight join when a deferred leave targets a different workspace', async () => { + const { socket, handlers } = createSocket() + let resolveAuth: (value: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve + }) + ) + setup(socket, createRoomManager()) + + // The client has switched to ws-2 (join in-flight) when a stale leave for the prior ws-1 lands. + const joinPromise = handlers[joinEvent]({ workspaceId: 'ws-2' }) + handlers[leaveEvent]({ workspaceId: 'ws-1' }) + resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-2', workspacePermission: 'admin' }) + await joinPromise + + // The deferred leave for ws-1 must not abort the join the client actually wants (ws-2). + expect(socket.join).toHaveBeenCalledWith(roomOf('ws-2')) + expect(socket.emit).toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-2' }) + }) + } +) diff --git a/apps/realtime/src/handlers/workspace-invalidation-room.ts b/apps/realtime/src/handlers/workspace-invalidation-room.ts new file mode 100644 index 00000000000..aacb141e9ae --- /dev/null +++ b/apps/realtime/src/handlers/workspace-invalidation-room.ts @@ -0,0 +1,155 @@ +import { createLogger } from '@sim/logger' +import { type RoomRef, type RoomType, roomName } from '@sim/realtime-protocol/rooms' +import { resolveRoomJoinAuth } from '@/handlers/room-join-auth' +import type { AuthenticatedSocket } from '@/middleware/auth' +import type { IRoomManager } from '@/rooms' + +const logger = createLogger('WorkspaceInvalidationRoom') + +interface JoinPayload { + workspaceId: string +} + +/** + * Wires a workspace-scoped, presence-free "invalidation room" onto a socket: the client joins a + * room named after its workspace, and a `${roomType}-changed` event — fanned out by the server-side + * mutation path over HTTP — reaches every viewer so they refetch. This is the shared core behind the + * workspace-files and workspace-tables browsers; they differ only in `roomType` (which also derives + * the event names, since each room type's wire token IS its event stem: `join-${roomType}`, + * `leave-${roomType}`, `join-${roomType}-success/-error`, and the `${roomType}-changed` broadcast). + * + * These rooms carry NO presence — "who's in a resource" comes from the per-resource room (file-doc / + * table), and mutations go over HTTP. Membership is tracked natively by Socket.IO (`socket.rooms`), + * so a workspace switch just leaves the prior room — no room-manager presence bookkeeping to sync. + */ +export function setupWorkspaceInvalidationRoom( + socket: AuthenticatedSocket, + roomManager: IRoomManager, + roomType: RoomType +) { + const joinEvent = `join-${roomType}` + const leaveEvent = `leave-${roomType}` + const successEvent = `${joinEvent}-success` + const errorEvent = `${joinEvent}-error` + const roomPrefix = `${roomType}:` + const room = (workspaceId: string): RoomRef => ({ type: roomType, id: workspaceId }) + + // Monotonic per-socket join counter: each join captures its number and, after the async + // authorize, aborts if a newer intent has superseded it — a fast workspace switch A→B can + // otherwise let A's late completion leave B and strand the socket in A, missing B's + // `${roomType}-changed` invalidations. + let joinGeneration = 0 + // The workspace the socket currently intends to be in (set when a join starts). A leave that + // targets this workspace — or an unscoped "leave all" — advances joinGeneration so an in-flight + // join is cancelled instead of completing after the view has closed. A stale/deferred leave for + // a DIFFERENT workspace must NOT advance it, or it would abort the join the client has since + // switched to (the bug that bit the file-doc room in #5941). + let currentWorkspace: string | null = null + + socket.on(joinEvent, async ({ workspaceId }: JoinPayload) => { + // Validate synchronously BEFORE claiming a generation, so a rejected/malformed join can't + // advance joinGeneration and cancel a legitimate in-flight join for another workspace. + if (!socket.userId || !socket.userName) { + socket.emit(errorEvent, { + workspaceId, + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, + }) + return + } + + if (!roomManager.isReady()) { + socket.emit(errorEvent, { + workspaceId, + error: 'Realtime unavailable', + code: 'ROOM_MANAGER_UNAVAILABLE', + retryable: true, + }) + return + } + + // Validate the client-supplied id before it reaches the DB query (join payloads are + // otherwise raw client input) and before advancing the generation. + if (typeof workspaceId !== 'string' || workspaceId.length === 0) { + socket.emit(errorEvent, { + workspaceId: typeof workspaceId === 'string' ? workspaceId : '', + error: 'Invalid workspace id', + code: 'INVALID_PAYLOAD', + retryable: false, + }) + return + } + + const joinAttempt = (joinGeneration += 1) + currentWorkspace = workspaceId + try { + const ref = room(workspaceId) + + const authorized = await resolveRoomJoinAuth({ + userId: socket.userId, + room: ref, + action: 'read', + logger, + logLabel: `${roomType} room for ${socket.userId}`, + messages: { + verifyFailed: 'Failed to verify workspace access', + notFound: 'Workspace not found', + accessDenied: 'Access denied to workspace', + }, + emitError: ({ error, code, retryable }) => + socket.emit(errorEvent, { workspaceId, error, code, retryable }), + }) + if (!authorized) return + + // A newer join started on this socket during authorize (or it dropped): abort so a + // stale join can't leave the room the client has since switched to. + if (joinGeneration !== joinAttempt || socket.disconnected) return + + // Leave any previously-joined room of this type (workspace switch), read straight from the + // socket's native room membership so there's no presence store to keep in sync. + const target = roomName(ref) + for (const joined of socket.rooms) { + if (joined !== target && joined.startsWith(roomPrefix)) socket.leave(joined) + } + + socket.join(target) + socket.emit(successEvent, { workspaceId }) + } catch (error) { + logger.error(`Error joining ${roomType} room:`, error) + try { + socket.leave(roomName(room(workspaceId))) + } catch {} + // Suppress the client-facing error when this join was already superseded: the client has + // switched to a newer workspace, and a retryable error naming the abandoned one could make it + // re-join and cancel the newer join. The leave above still runs. + if (joinGeneration !== joinAttempt || socket.disconnected) return + socket.emit(errorEvent, { + workspaceId, + error: `Failed to join ${roomType}`, + code: 'JOIN_FAILED', + retryable: true, + }) + } + }) + + socket.on(leaveEvent, (payload?: { workspaceId?: string }) => { + // Cancel an in-flight join whose target the client is now leaving: a join awaiting + // authorization when the view unmounts would otherwise complete afterwards and strand the + // socket in a room it has left. Only when the leave targets the current join intent (or is + // unscoped) — a deferred leave for a different workspace must not abort the join the client + // has since switched to. + if (!payload?.workspaceId || payload.workspaceId === currentWorkspace) { + joinGeneration += 1 + currentWorkspace = null + } + // Scope the leave to a specific workspace when the client provides one: a deferred leave + // from a prior page must not evict a room the socket has since switched into. + const target = payload?.workspaceId ? roomName(room(payload.workspaceId)) : null + for (const joined of socket.rooms) { + if (!joined.startsWith(roomPrefix)) continue + if (target && joined !== target) continue + socket.leave(joined) + } + }) +} diff --git a/apps/realtime/src/index.test.ts b/apps/realtime/src/index.test.ts index 92ddc101c69..eb7ee030937 100644 --- a/apps/realtime/src/index.test.ts +++ b/apps/realtime/src/index.test.ts @@ -4,11 +4,12 @@ * @vitest-environment node */ import { createServer, request as httpRequest } from 'http' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { createMockLogger } from '@sim/testing' import { randomInt } from '@sim/utils/random' import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' import { createSocketIOServer } from '@/config/socket' -import { MemoryRoomManager } from '@/rooms' +import { MemoryRoomManager, workflowRoom } from '@/rooms' import { createHttpHandler } from '@/routes/http' vi.mock('@/auth', () => ({ @@ -230,9 +231,10 @@ describe('Socket Server Index Integration', () => { const workflowId = 'test-workflow-123' const socketId = 'test-socket-123' + const room = workflowRoom(workflowId) const presence = { userId: 'user-123', - workflowId, + room, userName: 'Test User', socketId, joinedAt: Date.now(), @@ -240,10 +242,10 @@ describe('Socket Server Index Integration', () => { role: 'admin', } - await roomManager.addUserToRoom(workflowId, socketId, presence) + await roomManager.addUserToRoom(room, socketId, presence) - expect(await roomManager.hasWorkflowRoom(workflowId)).toBe(true) - const users = await roomManager.getWorkflowUsers(workflowId) + expect(await roomManager.hasRoom(room)).toBe(true) + const users = await roomManager.getRoomUsers(room) expect(users).toHaveLength(1) expect(users[0].socketId).toBe(socketId) }) @@ -252,9 +254,10 @@ describe('Socket Server Index Integration', () => { const socketId = 'test-socket-123' const workflowId = 'test-workflow-456' + const room = workflowRoom(workflowId) const presence = { userId: 'user-123', - workflowId, + room, userName: 'Test User', socketId, joinedAt: Date.now(), @@ -262,9 +265,9 @@ describe('Socket Server Index Integration', () => { role: 'admin', } - await roomManager.addUserToRoom(workflowId, socketId, presence) + await roomManager.addUserToRoom(room, socketId, presence) - expect(await roomManager.getWorkflowIdForSocket(socketId)).toBe(workflowId) + expect(await roomManager.getRoomForSocket(socketId, ROOM_TYPES.WORKFLOW)).toEqual(room) const session = await roomManager.getUserSession(socketId) expect(session).toBeDefined() expect(session?.userId).toBe('user-123') @@ -274,9 +277,10 @@ describe('Socket Server Index Integration', () => { const workflowId = 'test-workflow-789' const socketId = 'test-socket-789' + const room = workflowRoom(workflowId) const presence = { userId: 'user-789', - workflowId, + room, userName: 'Test User', socketId, joinedAt: Date.now(), @@ -284,16 +288,16 @@ describe('Socket Server Index Integration', () => { role: 'admin', } - await roomManager.addUserToRoom(workflowId, socketId, presence) + await roomManager.addUserToRoom(room, socketId, presence) - expect(await roomManager.hasWorkflowRoom(workflowId)).toBe(true) + expect(await roomManager.hasRoom(room)).toBe(true) // Remove user - await roomManager.removeUserFromRoom(socketId) + await roomManager.removeUserFromRoom(room, socketId) // Room should be cleaned up since it's now empty - expect(await roomManager.hasWorkflowRoom(workflowId)).toBe(false) - expect(await roomManager.getWorkflowIdForSocket(socketId)).toBeNull() + expect(await roomManager.hasRoom(room)).toBe(false) + expect(await roomManager.getRoomForSocket(socketId, ROOM_TYPES.WORKFLOW)).toBeNull() }) }) @@ -324,7 +328,7 @@ describe('Socket Server Index Integration', () => { expect(typeof roomManager.addUserToRoom).toBe('function') expect(typeof roomManager.removeUserFromRoom).toBe('function') - expect(typeof roomManager.handleWorkflowDeletion).toBe('function') + expect(typeof roomManager.removeSocketFromAllRooms).toBe('function') expect(typeof roomManager.broadcastPresenceUpdate).toBe('function') }) }) diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index 3f7f7eb22d0..80663141334 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -6,6 +6,8 @@ import { createSocketIOServer, shutdownSocketIOAdapter } from '@/config/socket' import { assertSchemaCompatibility } from '@/database/preflight' import { env } from '@/env' import { setupAllHandlers } from '@/handlers' +import { flushAllFileDocRooms } from '@/handlers/file-doc' +import { getFileDocStore, initFileDocStore } from '@/handlers/file-doc-store' import { type AuthenticatedSocket, authenticateSocket } from '@/middleware/auth' import { type IRoomManager, MemoryRoomManager, RedisRoomManager } from '@/rooms' import { createHttpHandler } from '@/routes/http' @@ -55,6 +57,10 @@ async function main() { // Initialize room manager (Redis or in-memory based on config) const roomManager = await createRoomManager(io) + // Initialize the shared Yjs backend for collaborative file docs (Redis Streams). Enabled only when + // REDIS_URL is set; otherwise the relay runs its original single-replica in-memory doc path. + await initFileDocStore(env.REDIS_URL) + // Set up authentication middleware io.use(authenticateSocket) @@ -106,11 +112,26 @@ async function main() { logger.info(`Health check available at: http://localhost:${PORT}/health`) }) + let shuttingDown = false const shutdown = async () => { + // SIGINT and SIGTERM both bind this; a double signal (or SIGTERM then SIGINT during the drain) + // must not run the whole teardown twice — that means a second forced-exit timer and a second + // Redis quit (which throws "The client is closed"). + if (shuttingDown) return + shuttingDown = true logger.info('Shutting down Socket.IO server...') accessRevalidation.stop() + // Flush open collaborative docs to durable markdown BEFORE tearing down Redis/the store — the + // per-socket disconnect flush is fire-and-forget and would race process exit. + try { + await flushAllFileDocRooms() + logger.info('Flushed open collaborative documents') + } catch (error) { + logger.error('Error flushing collaborative documents on shutdown:', error) + } + try { await roomManager.shutdown() logger.info('RoomManager shutdown complete') @@ -124,6 +145,21 @@ async function main() { logger.error('Error during Socket.IO adapter shutdown:', error) } + try { + await getFileDocStore().shutdown() + } catch (error) { + logger.error('Error during FileDocStore shutdown:', error) + } + + // Close local client connections so `httpServer.close()` can complete its callback and exit + // gracefully — otherwise open websockets keep it hanging until the forced-exit timer below. + // Local-only: a rolling deploy must not disconnect clients pinned to other pods. + try { + io.local.disconnectSockets(true) + } catch (error) { + logger.error('Error disconnecting sockets on shutdown:', error) + } + httpServer.close(() => { logger.info('Socket.IO server closed') process.exit(0) diff --git a/apps/realtime/src/rooms/index.ts b/apps/realtime/src/rooms/index.ts index 8067fc1b215..29157aa4581 100644 --- a/apps/realtime/src/rooms/index.ts +++ b/apps/realtime/src/rooms/index.ts @@ -1,3 +1,4 @@ export { MemoryRoomManager } from '@/rooms/memory-manager' export { RedisRoomManager } from '@/rooms/redis-manager' -export type { IRoomManager, UserPresence, UserSession, WorkflowRoom } from '@/rooms/types' +export type { IRoomManager, RoomState, UserPresence, UserSession } from '@/rooms/types' +export { WorkflowRoomService, workflowRoom } from '@/rooms/workflow-room-service' diff --git a/apps/realtime/src/rooms/memory-manager.test.ts b/apps/realtime/src/rooms/memory-manager.test.ts new file mode 100644 index 00000000000..af98414f540 --- /dev/null +++ b/apps/realtime/src/rooms/memory-manager.test.ts @@ -0,0 +1,209 @@ +/** + * Multi-room semantics for the room manager. Exercises the invariants the + * single-room → multi-room migration must preserve: a socket in two rooms, + * refcounted session cleanup, presence isolation, and full-disconnect cleanup. + * + * @vitest-environment node + */ +import { ROOM_TYPES, type RoomRef } from '@sim/realtime-protocol/rooms' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { MemoryRoomManager } from '@/rooms/memory-manager' +import { sweepStalePresence } from '@/rooms/presence-visibility' +import type { UserPresence } from '@/rooms/types' + +function fakeIo(liveSocketIds: string[] = []) { + const emit = vi.fn() + return { + emit, + io: { + to: vi.fn().mockReturnValue({ emit }), + in: vi.fn().mockReturnValue({ + fetchSockets: vi.fn().mockResolvedValue(liveSocketIds.map((id) => ({ id }))), + }), + } as never, + } +} + +function presence(room: RoomRef, socketId: string, userId: string): UserPresence { + return { + userId, + room, + userName: `user-${userId}`, + socketId, + joinedAt: Date.now(), + lastActivity: Date.now(), + role: 'admin', + } +} + +const WORKFLOW: RoomRef = { type: ROOM_TYPES.WORKFLOW, id: 'wf-1' } +const FILES: RoomRef = { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-1' } + +describe('MemoryRoomManager multi-room', () => { + let manager: MemoryRoomManager + + beforeEach(async () => { + manager = new MemoryRoomManager(fakeIo().io) + await manager.initialize() + }) + + it('tracks a single socket in two rooms of different types', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + const rooms = await manager.getRoomsForSocket('socket-1') + expect(rooms).toHaveLength(2) + expect(rooms).toContainEqual(WORKFLOW) + expect(rooms).toContainEqual(FILES) + + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKFLOW)).toEqual(WORKFLOW) + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKSPACE_FILES)).toEqual(FILES) + }) + + it('keeps the shared session alive when leaving one of two rooms (refcount)', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + const removed = await manager.removeUserFromRoom(WORKFLOW, 'socket-1') + expect(removed).toBe(true) + + // The files room and the shared session must survive. + expect(await manager.hasRoom(WORKFLOW)).toBe(false) + expect(await manager.hasRoom(FILES)).toBe(true) + expect(await manager.getUserSession('socket-1')).not.toBeNull() + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKFLOW)).toBeNull() + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKSPACE_FILES)).toEqual(FILES) + }) + + it('drops the shared session only when the last room is left', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + await manager.removeUserFromRoom(WORKFLOW, 'socket-1') + expect(await manager.getUserSession('socket-1')).not.toBeNull() + + await manager.removeUserFromRoom(FILES, 'socket-1') + expect(await manager.getUserSession('socket-1')).toBeNull() + expect(await manager.getRoomsForSocket('socket-1')).toHaveLength(0) + }) + + it('isolates presence between rooms of different types', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + expect(await manager.getRoomUsers(WORKFLOW)).toHaveLength(1) + expect(await manager.getRoomUsers(FILES)).toHaveLength(2) + }) + + it('removes a socket from every room on disconnect and reports them', async () => { + await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + const removed = await manager.removeSocketFromAllRooms('socket-1') + expect(removed).toHaveLength(2) + expect(removed).toContainEqual(WORKFLOW) + expect(removed).toContainEqual(FILES) + + expect(await manager.hasRoom(WORKFLOW)).toBe(false) + // The files room still has socket-2. + expect(await manager.getRoomUsers(FILES)).toHaveLength(1) + expect(await manager.getUserSession('socket-1')).toBeNull() + }) + + it('does not clobber another type when two sockets share a room', async () => { + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + await manager.removeUserFromRoom(FILES, 'socket-1') + expect(await manager.hasRoom(FILES)).toBe(true) + expect(await manager.getUserSession('socket-2')).not.toBeNull() + }) + + it('sweepStalePresence reclaims not-live stale entries but keeps live and fresh ones', async () => { + const { io } = fakeIo(['socket-live']) + const m = new MemoryRoomManager(io) + await m.initialize() + + const staleMs = 76 * 60 * 1000 + await m.addUserToRoom(FILES, 'socket-live', presence(FILES, 'socket-live', 'u1')) + await m.addUserToRoom(FILES, 'socket-dead', { + ...presence(FILES, 'socket-dead', 'u2'), + joinedAt: Date.now() - staleMs, + lastActivity: Date.now() - staleMs, + }) + await m.addUserToRoom(FILES, 'socket-recent', presence(FILES, 'socket-recent', 'u3')) + + await sweepStalePresence(m, FILES) + + const remaining = (await m.getRoomUsers(FILES)).map((u) => u.socketId).sort() + // socket-dead: not live + stale → removed. socket-live: live → kept. + // socket-recent: not live but fresh (transient) → kept. + expect(remaining).toEqual(['socket-live', 'socket-recent']) + }) + + it('deleteRoom unconditionally drops all room state', async () => { + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + expect(await manager.hasRoom(FILES)).toBe(true) + + await manager.deleteRoom(FILES) + + expect(await manager.hasRoom(FILES)).toBe(false) + expect(await manager.getRoomUsers(FILES)).toHaveLength(0) + }) + + it('ignores removal of a room the socket is not in (id-guarded)', async () => { + await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + // Removing a workflow room the socket never joined must be a no-op — it must + // not wipe the files mapping or the shared session. + const removed = await manager.removeUserFromRoom(WORKFLOW, 'socket-1') + expect(removed).toBe(false) + expect(await manager.hasRoom(FILES)).toBe(true) + expect(await manager.getUserSession('socket-1')).not.toBeNull() + expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKSPACE_FILES)).toEqual(FILES) + }) + + it('broadcasts presence on the room-type-specific event name', async () => { + const { emit, io } = fakeIo() + const m = new MemoryRoomManager(io) + await m.initialize() + await m.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + + await m.broadcastPresenceUpdate(FILES) + expect(emit).toHaveBeenCalledWith('workspace-files:presence-update', expect.any(Array)) + + await m.broadcastPresenceUpdate(WORKFLOW) + expect(emit).toHaveBeenCalledWith('presence-update', expect.any(Array)) + }) + + it('omits an excluded socket from the presence broadcast (disconnect ghost guard)', async () => { + const { emit, io } = fakeIo() + const m = new MemoryRoomManager(io) + await m.initialize() + await m.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await m.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + // Broadcast as if socket-1 is disconnecting: even though its presence entry is + // still present, it must not appear in the emitted list. + await m.broadcastPresenceUpdate(FILES, 'socket-1') + const emitted = emit.mock.calls.at(-1)?.[1] as Array<{ socketId: string }> + expect(emitted.map((u) => u.socketId)).toEqual(['socket-2']) + }) + + it('never emits a presence entry whose socket is no longer live (ghost guard)', async () => { + // Only socket-2 is a live Socket.IO member; socket-1 is an orphaned entry that + // outlived a failed removal. + const { emit, io } = fakeIo(['socket-2']) + const m = new MemoryRoomManager(io) + await m.initialize() + await m.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) + await m.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) + + await m.broadcastPresenceUpdate(FILES) + const emitted = emit.mock.calls.at(-1)?.[1] as Array<{ socketId: string }> + expect(emitted.map((u) => u.socketId)).toEqual(['socket-2']) + }) +}) diff --git a/apps/realtime/src/rooms/memory-manager.ts b/apps/realtime/src/rooms/memory-manager.ts index a0aeb9d1fea..f7ebed862ac 100644 --- a/apps/realtime/src/rooms/memory-manager.ts +++ b/apps/realtime/src/rooms/memory-manager.ts @@ -1,16 +1,30 @@ import { createLogger } from '@sim/logger' +import { + presenceEventName, + type RoomRef, + type RoomType, + roomName, +} from '@sim/realtime-protocol/rooms' import type { Server } from 'socket.io' -import type { IRoomManager, UserPresence, UserSession, WorkflowRoom } from '@/rooms/types' +import { filterVisiblePresence } from '@/rooms/presence-visibility' +import type { IRoomManager, RoomState, UserPresence, UserSession } from '@/rooms/types' const logger = createLogger('MemoryRoomManager') +/** Stable string key for a room in the local maps (distinct from the Socket.IO room name). */ +function roomKey(room: RoomRef): string { + return `${room.type}:${room.id}` +} + /** - * In-memory room manager for single-pod deployments - * Used as fallback when REDIS_URL is not configured + * In-memory room manager for single-pod deployments. Used when REDIS_URL is not + * configured. Domain-neutral: keyed by {@link RoomRef}, supports a socket in + * multiple rooms (one per {@link RoomType}). */ export class MemoryRoomManager implements IRoomManager { - private workflowRooms = new Map() - private socketToWorkflow = new Map() + private rooms = new Map() + /** socketId -> (roomType -> roomId) */ + private socketRooms = new Map>() private userSessions = new Map() private _io: Server @@ -31,235 +45,156 @@ export class MemoryRoomManager implements IRoomManager { } async shutdown(): Promise { - this.workflowRooms.clear() - this.socketToWorkflow.clear() + this.rooms.clear() + this.socketRooms.clear() this.userSessions.clear() logger.info('MemoryRoomManager shutdown complete') } - async addUserToRoom(workflowId: string, socketId: string, presence: UserPresence): Promise { - // Create room if it doesn't exist - if (!this.workflowRooms.has(workflowId)) { - this.workflowRooms.set(workflowId, { - workflowId, - users: new Map(), - lastModified: Date.now(), - activeConnections: 0, - }) + async addUserToRoom(room: RoomRef, socketId: string, presence: UserPresence): Promise { + const key = roomKey(room) + let state = this.rooms.get(key) + if (!state) { + state = { room, users: new Map(), lastModified: Date.now(), activeConnections: 0 } + this.rooms.set(key, state) } - const room = this.workflowRooms.get(workflowId)! - room.users.set(socketId, presence) - room.activeConnections++ - room.lastModified = Date.now() + state.users.set(socketId, presence) + state.activeConnections++ + state.lastModified = Date.now() - // Map socket to workflow - this.socketToWorkflow.set(socketId, workflowId) + let socketRoomMap = this.socketRooms.get(socketId) + if (!socketRoomMap) { + socketRoomMap = new Map() + this.socketRooms.set(socketId, socketRoomMap) + } + socketRoomMap.set(room.type, room.id) - // Store session this.userSessions.set(socketId, { userId: presence.userId, userName: presence.userName, avatarUrl: presence.avatarUrl, }) - logger.debug(`Added user ${presence.userId} to workflow ${workflowId} (socket: ${socketId})`) + logger.debug(`Added user ${presence.userId} to room ${key} (socket: ${socketId})`) } - async removeUserFromRoom(socketId: string, workflowIdHint?: string): Promise { - const currentWorkflowId = this.socketToWorkflow.get(socketId) ?? null - const workflowId = workflowIdHint ?? currentWorkflowId + async removeUserFromRoom(room: RoomRef, socketId: string): Promise { + const key = roomKey(room) + const state = this.rooms.get(key) + let existed = false - if (!workflowId) { - return null - } - - const room = this.workflowRooms.get(workflowId) - if (room) { - if (room.users.delete(socketId)) { - room.activeConnections = Math.max(0, room.activeConnections - 1) + if (state?.users.has(socketId)) { + existed = true + state.users.delete(socketId) + state.activeConnections = Math.max(0, state.activeConnections - 1) + if (state.users.size === 0) { + this.rooms.delete(key) + logger.info(`Cleaned up empty room: ${key}`) } + } - // Clean up empty rooms - if (room.activeConnections === 0) { - this.workflowRooms.delete(workflowId) - logger.info(`Cleaned up empty workflow room: ${workflowId}`) + const socketRoomMap = this.socketRooms.get(socketId) + if (socketRoomMap && socketRoomMap.get(room.type) === room.id) { + socketRoomMap.delete(room.type) + // Drop the shared session only when the socket has left its last room. + if (socketRoomMap.size === 0) { + this.socketRooms.delete(socketId) + this.userSessions.delete(socketId) } } - // Only clear the socket's own mappings when it is not mapped to a different - // room — removing a stale room's entry must not destroy the mapping of a - // room the socket has since moved to. - if (currentWorkflowId === null || currentWorkflowId === workflowId) { - this.socketToWorkflow.delete(socketId) + return existed + } + + async removeSocketFromAllRooms(socketId: string): Promise { + const socketRoomMap = this.socketRooms.get(socketId) + if (!socketRoomMap || socketRoomMap.size === 0) { this.userSessions.delete(socketId) + return [] + } + + const rooms: RoomRef[] = Array.from(socketRoomMap.entries()).map(([type, id]) => ({ type, id })) + for (const room of rooms) { + await this.removeUserFromRoom(room, socketId) } + // Belt-and-suspenders: ensure session is gone even if the map drifted. + this.socketRooms.delete(socketId) + this.userSessions.delete(socketId) + return rooms + } - logger.debug(`Removed socket ${socketId} from workflow ${workflowId}`) - return workflowId + async getRoomsForSocket(socketId: string): Promise { + const socketRoomMap = this.socketRooms.get(socketId) + if (!socketRoomMap) return [] + return Array.from(socketRoomMap.entries()).map(([type, id]) => ({ type, id })) } - async getWorkflowIdForSocket(socketId: string): Promise { - return this.socketToWorkflow.get(socketId) ?? null + async getRoomForSocket(socketId: string, type: RoomType): Promise { + const id = this.socketRooms.get(socketId)?.get(type) + return id ? { type, id } : null } async getUserSession(socketId: string): Promise { return this.userSessions.get(socketId) ?? null } - async getWorkflowUsers(workflowId: string): Promise { - const room = this.workflowRooms.get(workflowId) - if (!room) return [] - return Array.from(room.users.values()) + async getRoomUsers(room: RoomRef): Promise { + const state = this.rooms.get(roomKey(room)) + if (!state) return [] + return Array.from(state.users.values()) } - async hasWorkflowRoom(workflowId: string): Promise { - return this.workflowRooms.has(workflowId) + async hasRoom(room: RoomRef): Promise { + return this.rooms.has(roomKey(room)) + } + + async deleteRoom(room: RoomRef): Promise { + this.rooms.delete(roomKey(room)) } async updateUserActivity( - workflowId: string, + room: RoomRef, socketId: string, - updates: Partial> + updates: Partial> ): Promise { - const room = this.workflowRooms.get(workflowId) - if (!room) return - - const presence = room.users.get(socketId) - if (presence) { - if (updates.cursor !== undefined) presence.cursor = updates.cursor - if (updates.selection !== undefined) presence.selection = updates.selection - presence.lastActivity = updates.lastActivity ?? Date.now() - } - } + const presence = this.rooms.get(roomKey(room))?.users.get(socketId) + if (!presence) return - async updateRoomLastModified(workflowId: string): Promise { - const room = this.workflowRooms.get(workflowId) - if (room) { - room.lastModified = Date.now() - } + if (updates.cursor !== undefined) presence.cursor = updates.cursor + if (updates.selection !== undefined) presence.selection = updates.selection + if (updates.cell !== undefined) presence.cell = updates.cell + presence.lastActivity = updates.lastActivity ?? Date.now() } - async broadcastPresenceUpdate(workflowId: string): Promise { - const users = await this.getWorkflowUsers(workflowId) - this._io.to(workflowId).emit('presence-update', users) + async updateRoomLastModified(room: RoomRef): Promise { + const state = this.rooms.get(roomKey(room)) + if (state) state.lastModified = Date.now() } - emitToWorkflow(workflowId: string, event: string, payload: T): void { - this._io.to(workflowId).emit(event, payload) + async broadcastPresenceUpdate(room: RoomRef, excludeSocketId?: string): Promise { + const users = await this.getRoomUsers(room) + const visible = await filterVisiblePresence(this._io, room, users, excludeSocketId) + this._io.to(roomName(room)).emit(presenceEventName(room.type), visible) } - async getUniqueUserCount(workflowId: string): Promise { - const room = this.workflowRooms.get(workflowId) - if (!room) return 0 + emitToRoom(room: RoomRef, event: string, payload: T): void { + this._io.to(roomName(room)).emit(event, payload) + } + async getUniqueUserCount(room: RoomRef): Promise { + const state = this.rooms.get(roomKey(room)) + if (!state) return 0 const uniqueUsers = new Set() - room.users.forEach((presence) => { - uniqueUsers.add(presence.userId) - }) - + state.users.forEach((presence) => uniqueUsers.add(presence.userId)) return uniqueUsers.size } async getTotalActiveConnections(): Promise { let total = 0 - for (const room of this.workflowRooms.values()) { - total += room.activeConnections + for (const state of this.rooms.values()) { + total += state.activeConnections } return total } - - async handleWorkflowDeletion(workflowId: string): Promise { - logger.info(`Handling workflow deletion notification for ${workflowId}`) - - const room = this.workflowRooms.get(workflowId) - if (!room) { - logger.debug(`No active room found for deleted workflow ${workflowId}`) - return - } - - this._io.to(workflowId).emit('workflow-deleted', { - workflowId, - message: 'This workflow has been deleted', - timestamp: Date.now(), - }) - - const socketsToDisconnect: string[] = [] - room.users.forEach((_presence, socketId) => { - socketsToDisconnect.push(socketId) - }) - - for (const socketId of socketsToDisconnect) { - const socket = this._io.sockets.sockets.get(socketId) - if (socket) { - socket.leave(workflowId) - logger.debug(`Disconnected socket ${socketId} from deleted workflow ${workflowId}`) - } - await this.removeUserFromRoom(socketId) - } - - this.workflowRooms.delete(workflowId) - logger.info( - `Cleaned up workflow room ${workflowId} after deletion (${socketsToDisconnect.length} users disconnected)` - ) - } - - async handleWorkflowRevert(workflowId: string, timestamp: number): Promise { - logger.info(`Handling workflow revert notification for ${workflowId}`) - - const room = this.workflowRooms.get(workflowId) - if (!room) { - logger.debug(`No active room found for reverted workflow ${workflowId}`) - return - } - - this._io.to(workflowId).emit('workflow-reverted', { - workflowId, - message: 'Workflow has been reverted to deployed state', - timestamp, - }) - - room.lastModified = timestamp - - logger.info(`Notified ${room.users.size} users about workflow revert: ${workflowId}`) - } - - async handleWorkflowUpdate(workflowId: string): Promise { - logger.info(`Handling workflow update notification for ${workflowId}`) - - const room = this.workflowRooms.get(workflowId) - if (!room) { - logger.debug(`No active room found for updated workflow ${workflowId}`) - return - } - - const timestamp = Date.now() - - this._io.to(workflowId).emit('workflow-updated', { - workflowId, - message: 'Workflow has been updated externally', - timestamp, - }) - - room.lastModified = timestamp - - logger.info(`Notified ${room.users.size} users about workflow update: ${workflowId}`) - } - - async handleWorkflowDeployed(workflowId: string): Promise { - logger.info(`Handling workflow deployed notification for ${workflowId}`) - - const room = this.workflowRooms.get(workflowId) - if (!room) { - logger.debug(`No active room found for deployed workflow ${workflowId}`) - return - } - - this._io.to(workflowId).emit('workflow-deployed', { - workflowId, - timestamp: Date.now(), - }) - - logger.info(`Notified ${room.users.size} users about workflow deployment change: ${workflowId}`) - } } diff --git a/apps/realtime/src/rooms/presence-visibility.ts b/apps/realtime/src/rooms/presence-visibility.ts new file mode 100644 index 00000000000..56c22a0e861 --- /dev/null +++ b/apps/realtime/src/rooms/presence-visibility.ts @@ -0,0 +1,84 @@ +import { type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import type { Server } from 'socket.io' +import type { IRoomManager, UserPresence } from '@/rooms/types' + +/** + * How stale a not-live presence entry must be before a join-time sweep reclaims + * it. The `liveIds` gate (not this threshold) is what protects an active + * collaborator; the threshold only bounds how long a genuinely-orphaned entry + * (e.g. a crashed pod that never fired `disconnecting`) lingers. Matches the + * workflow join sweep. + */ +const STALE_PRESENCE_THRESHOLD_MS = 75 * 60 * 1000 + +/** + * Filters a room's stored presence down to what should actually be broadcast: + * drops `excludeSocketId` (e.g. a socket mid-disconnect that is still connected), + * then reconciles against the live Socket.IO membership so an entry orphaned by a + * failed removal (the room hashes have no TTL) is never emitted as a ghost. + * + * Fail-safe: if the liveness lookup throws, or returns an empty set while we still + * hold presence entries (a cross-pod `fetchSockets` timeout, not a truly empty + * room), we emit the unfiltered list rather than hide everyone — a transient + * ghost self-corrects on the next broadcast, but hiding live collaborators would + * be a worse, visible failure. + */ +export async function filterVisiblePresence( + io: Server, + room: RoomRef, + users: T[], + excludeSocketId?: string +): Promise { + const candidates = excludeSocketId + ? users.filter((user) => user.socketId !== excludeSocketId) + : users + + try { + const liveSockets = await io.in(roomName(room)).fetchSockets() + if (liveSockets.length === 0) { + return candidates + } + const liveIds = new Set(liveSockets.map((socket) => socket.id)) + return candidates.filter((user) => liveIds.has(user.socketId)) + } catch { + return candidates + } +} + +/** + * Reclaims orphaned presence entries in a room: any stored socket that is no + * longer a live Socket.IO member AND has been idle past + * {@link STALE_PRESENCE_THRESHOLD_MS} is removed. This is how a room-users hash + * (which has no TTL) is bounded against ungraceful disconnects — a pod crash + * fires no `disconnecting` event, so its entries would otherwise persist forever. + * Run on join, like the workflow room does. No-op when the liveness lookup fails + * (so a transient adapter blip can't evict live collaborators). + */ +export async function sweepStalePresence( + manager: IRoomManager, + room: RoomRef +): Promise { + // Read the roster first so it is returned to the caller (for same-tab dedup) even when the + // liveness probe below fails — a fetchSockets outage must skip only the stale-removal, never + // the caller's dedup. + const users = await manager.getRoomUsers(room) + let liveIds: Set + try { + const liveSockets = await manager.io.in(roomName(room)).fetchSockets() + liveIds = new Set(liveSockets.map((socket) => socket.id)) + } catch { + return users + } + + const now = Date.now() + for (const user of users) { + if (liveIds.has(user.socketId)) continue + const lastSeen = user.lastActivity || user.joinedAt || 0 + if (now - lastSeen > STALE_PRESENCE_THRESHOLD_MS) { + await manager.removeUserFromRoom(room, user.socketId) + } + } + // Return the pre-removal roster so a caller can reuse it (e.g. same-tab dedup) instead + // of re-reading; re-removing an already-swept entry downstream is a harmless no-op. + return users +} diff --git a/apps/realtime/src/rooms/redis-manager.ts b/apps/realtime/src/rooms/redis-manager.ts index 4ed34d2dfa5..7282f4a6778 100644 --- a/apps/realtime/src/rooms/redis-manager.ts +++ b/apps/realtime/src/rooms/redis-manager.ts @@ -1,140 +1,154 @@ import { createLogger } from '@sim/logger' +import { + presenceEventName, + type RoomRef, + type RoomType, + roomName, +} from '@sim/realtime-protocol/rooms' import { createClient, type RedisClientType } from 'redis' import type { Server } from 'socket.io' +import { filterVisiblePresence } from '@/rooms/presence-visibility' import type { IRoomManager, UserPresence, UserSession } from '@/rooms/types' const logger = createLogger('RedisRoomManager') +/** + * Redis key scheme (all room-scoped keys are prefixed by room type): + * {type}:{id}:users HASH socketId -> UserPresence JSON (room membership) + * {type}:{id}:meta HASH room metadata (lastModified) + * socket:{sid}:rooms HASH roomType -> roomId (the socket's rooms, one per type) + * socket:{sid}:session HASH userId/userName/avatarUrl (shared across the socket's rooms) + * + * Workflow rooms keep their historical `workflow:{id}:users`/`:meta` keys (the + * type prefix IS `workflow`), so no presence-state migration is needed for them. + */ const KEYS = { - workflowUsers: (wfId: string) => `workflow:${wfId}:users`, - workflowMeta: (wfId: string) => `workflow:${wfId}:meta`, - socketWorkflow: (socketId: string) => `socket:${socketId}:workflow`, + roomUsers: (room: RoomRef) => `${room.type}:${room.id}:users`, + roomMeta: (room: RoomRef) => `${room.type}:${room.id}:meta`, + socketRooms: (socketId: string) => `socket:${socketId}:rooms`, socketSession: (socketId: string) => `socket:${socketId}:session`, - socketPresenceWorkflow: (socketId: string) => `socket:${socketId}:presence-workflow`, } as const -const SOCKET_KEY_TTL = 3600 -const SOCKET_PRESENCE_WORKFLOW_KEY_TTL = 24 * 60 * 60 +/** TTL for the socket's room-set. Long enough that an idle-but-connected socket is not evicted. */ +const SOCKET_ROOMS_TTL = 24 * 60 * 60 +/** + * The shared session key MUST share the room-set TTL. `getRoomForSocket` reads the room-set and the + * workflow handlers gate edits/presence on `room && session`, so a session that expired while the + * room-set is still alive would wedge an active-but-idle collaborator into "session expired" until a + * full reload — the activity update only `EXPIRE`s the session, which cannot resurrect an + * already-gone key (only `addUserToRoom` re-`HSET`s it). Both are refreshed together on every + * activity, so they expire together. + */ +const SESSION_TTL = SOCKET_ROOMS_TTL /** - * Lua script for atomic user removal from room. - * The hint, when provided, is the target room to remove membership from; the - * socket's current mapping is only the fallback. Socket-level keys are cleared - * only when the socket is not mapped to a different room, so removing a stale - * room cannot destroy the mapping of a room the socket has since moved to. - * Returns the target workflowId, or null when no target could be resolved. - * Handles room cleanup atomically to prevent race conditions. + * Atomic single-room removal. Removes a socket from one room's presence, drops + * the room from the socket's room-set, and — critically — deletes the SHARED + * session key only when the socket has left its LAST room (otherwise a leave from + * one room would break the socket's other rooms). Cleans up empty room state. + * + * KEYS: [socketRooms, socketSession, roomUsers, roomMeta] + * ARGV: [roomType, socketId, roomId] + * Returns 1 if the socket was a member of the room, else 0. */ -const REMOVE_USER_SCRIPT = ` -local socketWorkflowKey = KEYS[1] +const REMOVE_ROOM_SCRIPT = ` +local socketRoomsKey = KEYS[1] local socketSessionKey = KEYS[2] -local socketPresenceWorkflowKey = KEYS[3] -local workflowUsersPrefix = ARGV[1] -local workflowMetaPrefix = ARGV[2] -local socketId = ARGV[3] -local workflowIdHint = ARGV[4] - -local currentWorkflowId = redis.call('GET', socketWorkflowKey) -if not currentWorkflowId then - currentWorkflowId = redis.call('GET', socketPresenceWorkflowKey) +local roomUsersKey = KEYS[3] +local roomMetaKey = KEYS[4] +local roomType = ARGV[1] +local socketId = ARGV[2] +local roomId = ARGV[3] + +local removed = redis.call('HDEL', roomUsersKey, socketId) + +-- Only drop the socket's mapping for this type if it points at THIS room, so +-- removing a room the socket isn't in can't wipe a different room's mapping or +-- spuriously trigger the last-room session cleanup (mirrors the memory manager). +if redis.call('HGET', socketRoomsKey, roomType) == roomId then + redis.call('HDEL', socketRoomsKey, roomType) + if redis.call('HLEN', socketRoomsKey) == 0 then + redis.call('DEL', socketRoomsKey, socketSessionKey) + end end -local workflowId = currentWorkflowId -if workflowIdHint ~= '' then - workflowId = workflowIdHint +if redis.call('HLEN', roomUsersKey) == 0 then + redis.call('DEL', roomUsersKey, roomMetaKey) end -if not workflowId then - return nil -end - -local workflowUsersKey = workflowUsersPrefix .. workflowId .. ':users' -local workflowMetaKey = workflowMetaPrefix .. workflowId .. ':meta' - -redis.call('HDEL', workflowUsersKey, socketId) - -if (not currentWorkflowId) or currentWorkflowId == workflowId then - redis.call('DEL', socketWorkflowKey, socketSessionKey, socketPresenceWorkflowKey) -end - -local remaining = redis.call('HLEN', workflowUsersKey) -if remaining == 0 then - redis.call('DEL', workflowUsersKey, workflowMetaKey) -end - -return workflowId +return removed ` /** - * Lua script for atomic user activity update. - * Performs read-modify-write atomically to prevent lost updates. - * Also refreshes TTL on socket keys to prevent expiry during long sessions. + * Atomic presence-activity update (read-modify-write) that also refreshes the + * socket key TTLs to keep a long-lived session alive. + * + * KEYS: [roomUsers, socketRooms, socketSession] + * ARGV: [socketId, cursorJson, selectionJson, lastActivity, roomsTtl, sessionTtl, cellJson] + * Returns 1 if the socket had presence in the room, else 0. */ const UPDATE_ACTIVITY_SCRIPT = ` -local workflowUsersKey = KEYS[1] -local socketWorkflowKey = KEYS[2] +local roomUsersKey = KEYS[1] +local socketRoomsKey = KEYS[2] local socketSessionKey = KEYS[3] -local socketPresenceWorkflowKey = KEYS[4] local socketId = ARGV[1] local cursorJson = ARGV[2] local selectionJson = ARGV[3] local lastActivity = ARGV[4] -local ttl = tonumber(ARGV[5]) -local presenceWorkflowTtl = tonumber(ARGV[6]) +local roomsTtl = tonumber(ARGV[5]) +local sessionTtl = tonumber(ARGV[6]) +local cellJson = ARGV[7] -local existingJson = redis.call('HGET', workflowUsersKey, socketId) +local existingJson = redis.call('HGET', roomUsersKey, socketId) if not existingJson then return 0 end local existing = cjson.decode(existingJson) - if cursorJson ~= '' then existing.cursor = cjson.decode(cursorJson) end if selectionJson ~= '' then existing.selection = cjson.decode(selectionJson) end +if cellJson ~= '' then + existing.cell = cjson.decode(cellJson) +end existing.lastActivity = tonumber(lastActivity) -redis.call('HSET', workflowUsersKey, socketId, cjson.encode(existing)) -redis.call('EXPIRE', socketWorkflowKey, ttl) -redis.call('EXPIRE', socketSessionKey, ttl) -redis.call('EXPIRE', socketPresenceWorkflowKey, presenceWorkflowTtl) +redis.call('HSET', roomUsersKey, socketId, cjson.encode(existing)) +redis.call('EXPIRE', socketRoomsKey, roomsTtl) +redis.call('EXPIRE', socketSessionKey, sessionTtl) return 1 ` /** - * Redis-backed room manager for multi-pod deployments. - * Uses Lua scripts for atomic operations to prevent race conditions. + * Redis-backed room manager for multi-pod deployments. Domain-neutral: keyed by + * {@link RoomRef}, supports a socket in multiple rooms (one per {@link RoomType}). + * Uses Lua scripts for atomic multi-key operations. */ export class RedisRoomManager implements IRoomManager { private redis: RedisClientType private _io: Server private isConnected = false - private removeUserScriptSha: string | null = null + private removeRoomScriptSha: string | null = null private updateActivityScriptSha: string | null = null constructor(io: Server, redisUrl: string) { this._io = io - this.redis = createClient({ - url: redisUrl, - }) + this.redis = createClient({ url: redisUrl }) this.redis.on('error', (err) => { logger.error('Redis client error:', err) }) - this.redis.on('reconnecting', () => { logger.warn('Redis client reconnecting...') this.isConnected = false }) - this.redis.on('ready', () => { logger.info('Redis client ready') this.isConnected = true }) - this.redis.on('end', () => { logger.warn('Redis client connection closed') this.isConnected = false @@ -146,7 +160,14 @@ export class RedisRoomManager implements IRoomManager { } isReady(): boolean { - return this.isConnected + // Gate on the loaded script SHAs, not just the connection: the `ready` event flips + // `isConnected` true as soon as the socket connects — before `initialize()` loads the Lua + // scripts — so a bare `isConnected` check would report ready while `removeUserFromRoom` / + // `updateUserActivity` would silently no-op on a null SHA. Reporting not-ready here makes the + // POST endpoints return a retryable 503 during that startup window instead. + return ( + this.isConnected && this.removeRoomScriptSha !== null && this.updateActivityScriptSha !== null + ) } async initialize(): Promise { @@ -154,14 +175,18 @@ export class RedisRoomManager implements IRoomManager { try { await this.redis.connect() - this.isConnected = true - // Pre-load Lua scripts for better performance - this.removeUserScriptSha = await this.redis.scriptLoad(REMOVE_USER_SCRIPT) + this.removeRoomScriptSha = await this.redis.scriptLoad(REMOVE_ROOM_SCRIPT) this.updateActivityScriptSha = await this.redis.scriptLoad(UPDATE_ACTIVITY_SCRIPT) + // Mark ready only after the scripts load — isReady() gates removeUserFromRoom/updateUserActivity, + // which silently no-op without a script SHA. Setting the flag before scriptLoad would make + // isReady() lie if scriptLoad threw. + this.isConnected = true + logger.info('RedisRoomManager connected to Redis and scripts loaded') } catch (error) { + this.isConnected = false logger.error('Failed to connect to Redis:', error) throw error } @@ -169,7 +194,6 @@ export class RedisRoomManager implements IRoomManager { async shutdown(): Promise { if (!this.isConnected) return - try { await this.redis.quit() this.isConnected = false @@ -179,93 +203,102 @@ export class RedisRoomManager implements IRoomManager { } } - async addUserToRoom(workflowId: string, socketId: string, presence: UserPresence): Promise { + async addUserToRoom(room: RoomRef, socketId: string, presence: UserPresence): Promise { try { const pipeline = this.redis.multi() - pipeline.hSet(KEYS.workflowUsers(workflowId), socketId, JSON.stringify(presence)) - pipeline.hSet(KEYS.workflowMeta(workflowId), 'lastModified', Date.now().toString()) - pipeline.set(KEYS.socketWorkflow(socketId), workflowId) - pipeline.expire(KEYS.socketWorkflow(socketId), SOCKET_KEY_TTL) - pipeline.set(KEYS.socketPresenceWorkflow(socketId), workflowId) - pipeline.expire(KEYS.socketPresenceWorkflow(socketId), SOCKET_PRESENCE_WORKFLOW_KEY_TTL) + pipeline.hSet(KEYS.roomUsers(room), socketId, JSON.stringify(presence)) + pipeline.hSet(KEYS.roomMeta(room), 'lastModified', Date.now().toString()) + pipeline.hSet(KEYS.socketRooms(socketId), room.type, room.id) + pipeline.expire(KEYS.socketRooms(socketId), SOCKET_ROOMS_TTL) pipeline.hSet(KEYS.socketSession(socketId), { userId: presence.userId, userName: presence.userName, avatarUrl: presence.avatarUrl || '', }) - pipeline.expire(KEYS.socketSession(socketId), SOCKET_KEY_TTL) + pipeline.expire(KEYS.socketSession(socketId), SESSION_TTL) const results = await pipeline.exec() - // Check if any command failed const failed = results.some((result) => result instanceof Error) if (failed) { - logger.error(`Pipeline partially failed when adding user to room`, { workflowId, socketId }) + logger.error('Pipeline partially failed when adding user to room', { + room, + socketId, + }) throw new Error('Failed to store user session data in Redis') } - logger.debug(`Added user ${presence.userId} to workflow ${workflowId} (socket: ${socketId})`) + logger.debug(`Added user ${presence.userId} to room ${room.type}:${room.id} (${socketId})`) } catch (error) { - logger.error(`Failed to add user to room: ${socketId} -> ${workflowId}`, error) + logger.error(`Failed to add user to room: ${socketId} -> ${room.type}:${room.id}`, error) throw error } } - async removeUserFromRoom( - socketId: string, - workflowIdHint?: string, - retried = false - ): Promise { - if (!this.removeUserScriptSha) { + async removeUserFromRoom(room: RoomRef, socketId: string, retried = false): Promise { + if (!this.removeRoomScriptSha) { logger.error('removeUserFromRoom called before initialize()') - return null + return false } try { - const workflowId = await this.redis.evalSha(this.removeUserScriptSha, { + const removed = await this.redis.evalSha(this.removeRoomScriptSha, { keys: [ - KEYS.socketWorkflow(socketId), + KEYS.socketRooms(socketId), KEYS.socketSession(socketId), - KEYS.socketPresenceWorkflow(socketId), + KEYS.roomUsers(room), + KEYS.roomMeta(room), ], - arguments: ['workflow:', 'workflow:', socketId, workflowIdHint ?? ''], + arguments: [room.type, socketId, room.id], }) - - if (typeof workflowId === 'string' && workflowId.length > 0) { - logger.debug(`Removed socket ${socketId} from workflow ${workflowId}`) - return workflowId - } - - return null + return typeof removed === 'number' ? removed > 0 : Number(removed) > 0 } catch (error) { if ((error as Error).message?.includes('NOSCRIPT') && !retried) { logger.warn('Lua script not found, reloading...') - this.removeUserScriptSha = await this.redis.scriptLoad(REMOVE_USER_SCRIPT) - return this.removeUserFromRoom(socketId, workflowIdHint, true) + this.removeRoomScriptSha = await this.redis.scriptLoad(REMOVE_ROOM_SCRIPT) + return this.removeUserFromRoom(room, socketId, true) } - logger.error(`Failed to remove user from room: ${socketId}`, error) - return null + logger.error(`Failed to remove socket ${socketId} from room ${room.type}:${room.id}`, error) + return false } } - async getWorkflowIdForSocket(socketId: string): Promise { - const workflowId = await this.redis.get(KEYS.socketWorkflow(socketId)) - if (workflowId) { - return workflowId + async removeSocketFromAllRooms(socketId: string): Promise { + const rooms = await this.getRoomsForSocket(socketId) + if (rooms.length === 0) { + // Nothing tracked (already cleaned up or TTL-expired); ensure session is gone. + await this.redis.del(KEYS.socketSession(socketId)).catch(() => {}) + return [] } - return this.redis.get(KEYS.socketPresenceWorkflow(socketId)) + const removed: RoomRef[] = [] + for (const room of rooms) { + const wasMember = await this.removeUserFromRoom(room, socketId) + if (wasMember) removed.push(room) + } + return removed } - async getUserSession(socketId: string): Promise { + async getRoomsForSocket(socketId: string): Promise { try { - const session = await this.redis.hGetAll(KEYS.socketSession(socketId)) + const entries = await this.redis.hGetAll(KEYS.socketRooms(socketId)) + return Object.entries(entries).map(([type, id]) => ({ type: type as RoomType, id })) + } catch (error) { + logger.error(`Failed to get rooms for socket ${socketId}:`, error) + return [] + } + } - if (!session.userId) { - return null - } + async getRoomForSocket(socketId: string, type: RoomType): Promise { + const id = await this.redis.hGet(KEYS.socketRooms(socketId), type) + return id ? { type, id } : null + } + async getUserSession(socketId: string): Promise { + try { + const session = await this.redis.hGetAll(KEYS.socketSession(socketId)) + if (!session.userId) return null return { userId: session.userId, userName: session.userName, @@ -277,34 +310,54 @@ export class RedisRoomManager implements IRoomManager { } } - async getWorkflowUsers(workflowId: string): Promise { + /** + * Reads and parses the room roster. Throws on a transport error (so a caller can + * distinguish "genuinely empty" from "read failed"); a single corrupted entry is + * skipped, not fatal. + */ + private async readRoomUsers(room: RoomRef): Promise { + const users = await this.redis.hGetAll(KEYS.roomUsers(room)) + return Object.entries(users) + .map(([socketId, json]) => { + try { + return JSON.parse(json) as UserPresence + } catch { + logger.warn(`Corrupted user data for socket ${socketId}, skipping`) + return null + } + }) + .filter((u): u is UserPresence => u !== null) + } + + async getRoomUsers(room: RoomRef): Promise { try { - const users = await this.redis.hGetAll(KEYS.workflowUsers(workflowId)) - return Object.entries(users) - .map(([socketId, json]) => { - try { - return JSON.parse(json) as UserPresence - } catch { - logger.warn(`Corrupted user data for socket ${socketId}, skipping`) - return null - } - }) - .filter((u): u is UserPresence => u !== null) + return await this.readRoomUsers(room) } catch (error) { - logger.error(`Failed to get workflow users for ${workflowId}:`, error) + logger.error(`Failed to get room users for ${room.type}:${room.id}:`, error) return [] } } - async hasWorkflowRoom(workflowId: string): Promise { - const exists = await this.redis.exists(KEYS.workflowUsers(workflowId)) + async hasRoom(room: RoomRef): Promise { + const exists = await this.redis.exists(KEYS.roomUsers(room)) return exists > 0 } + async deleteRoom(room: RoomRef): Promise { + // Log AND rethrow (like addUserToRoom): a failed wipe must not be reported as a + // clean deletion by the caller — the request surfaces it (and can be retried). + try { + await this.redis.del([KEYS.roomUsers(room), KEYS.roomMeta(room)]) + } catch (error) { + logger.error(`Failed to delete room ${room.type}:${room.id}:`, error) + throw error + } + } + async updateUserActivity( - workflowId: string, + room: RoomRef, socketId: string, - updates: Partial>, + updates: Partial>, retried = false ): Promise { if (!this.updateActivityScriptSha) { @@ -314,155 +367,64 @@ export class RedisRoomManager implements IRoomManager { try { await this.redis.evalSha(this.updateActivityScriptSha, { - keys: [ - KEYS.workflowUsers(workflowId), - KEYS.socketWorkflow(socketId), - KEYS.socketSession(socketId), - KEYS.socketPresenceWorkflow(socketId), - ], + keys: [KEYS.roomUsers(room), KEYS.socketRooms(socketId), KEYS.socketSession(socketId)], arguments: [ socketId, updates.cursor !== undefined ? JSON.stringify(updates.cursor) : '', updates.selection !== undefined ? JSON.stringify(updates.selection) : '', (updates.lastActivity ?? Date.now()).toString(), - SOCKET_KEY_TTL.toString(), - SOCKET_PRESENCE_WORKFLOW_KEY_TTL.toString(), + SOCKET_ROOMS_TTL.toString(), + SESSION_TTL.toString(), + // Trailing arg (ARGV[7]) so existing indices stay stable. `null` (cleared + // selection) serializes to 'null'; `undefined` (no cell change) to '' (skip). + updates.cell !== undefined ? JSON.stringify(updates.cell) : '', ], }) } catch (error) { if ((error as Error).message?.includes('NOSCRIPT') && !retried) { logger.warn('Lua script not found, reloading...') this.updateActivityScriptSha = await this.redis.scriptLoad(UPDATE_ACTIVITY_SCRIPT) - return this.updateUserActivity(workflowId, socketId, updates, true) + return this.updateUserActivity(room, socketId, updates, true) } logger.error(`Failed to update user activity: ${socketId}`, error) } } - async updateRoomLastModified(workflowId: string): Promise { - await this.redis.hSet(KEYS.workflowMeta(workflowId), 'lastModified', Date.now().toString()) - } - - async broadcastPresenceUpdate(workflowId: string): Promise { - const users = await this.getWorkflowUsers(workflowId) - // io.to() with Redis adapter broadcasts to all pods - this._io.to(workflowId).emit('presence-update', users) - } - - emitToWorkflow(workflowId: string, event: string, payload: T): void { - this._io.to(workflowId).emit(event, payload) - } - - async getUniqueUserCount(workflowId: string): Promise { - const users = await this.getWorkflowUsers(workflowId) - const uniqueUserIds = new Set(users.map((u) => u.userId)) - return uniqueUserIds.size + async updateRoomLastModified(room: RoomRef): Promise { + await this.redis.hSet(KEYS.roomMeta(room), 'lastModified', Date.now().toString()) } - async getTotalActiveConnections(): Promise { - // This is more complex with Redis - we'd need to scan all workflow:*:users keys - // For now, just count sockets in this server instance - // The true count would require aggregating across all pods - return this._io.sockets.sockets.size - } - - async handleWorkflowDeletion(workflowId: string): Promise { - logger.info(`Handling workflow deletion notification for ${workflowId}`) - + async broadcastPresenceUpdate(room: RoomRef, excludeSocketId?: string): Promise { + let users: UserPresence[] try { - const users = await this.getWorkflowUsers(workflowId) - if (users.length === 0) { - logger.debug(`No active users found for deleted workflow ${workflowId}`) - return - } - - // Notify all clients across all pods via Redis adapter - this._io.to(workflowId).emit('workflow-deleted', { - workflowId, - message: 'This workflow has been deleted', - timestamp: Date.now(), - }) - - // Use Socket.IO's cross-pod socketsLeave() to remove all sockets from the room - // This works across all pods when using the Redis adapter - await this._io.in(workflowId).socketsLeave(workflowId) - logger.debug(`All sockets left workflow room ${workflowId} via socketsLeave()`) - - // Remove all users from Redis state - for (const user of users) { - await this.removeUserFromRoom(user.socketId, workflowId) - } - - // Clean up room data - await this.redis.del([KEYS.workflowUsers(workflowId), KEYS.workflowMeta(workflowId)]) - - logger.info( - `Cleaned up workflow room ${workflowId} after deletion (${users.length} users disconnected)` - ) + // Read via the throwing variant, NOT getRoomUsers: a transport error there returns `[]`, which + // would broadcast an empty roster and clear every remaining collaborator's presence until the + // next healthy update. Skip instead — the next successful join/activity broadcast (or the + // stale sweep) reconciles peers. + users = await this.readRoomUsers(room) } catch (error) { - logger.error(`Failed to handle workflow deletion for ${workflowId}:`, error) - } - } - - async handleWorkflowRevert(workflowId: string, timestamp: number): Promise { - logger.info(`Handling workflow revert notification for ${workflowId}`) - - const hasRoom = await this.hasWorkflowRoom(workflowId) - if (!hasRoom) { - logger.debug(`No active room found for reverted workflow ${workflowId}`) + logger.error( + `Skipping presence broadcast for ${room.type}:${room.id} (roster read failed):`, + error + ) return } - - this._io.to(workflowId).emit('workflow-reverted', { - workflowId, - message: 'Workflow has been reverted to deployed state', - timestamp, - }) - - await this.updateRoomLastModified(workflowId) - - const userCount = await this.getUniqueUserCount(workflowId) - logger.info(`Notified ${userCount} users about workflow revert: ${workflowId}`) + const visible = await filterVisiblePresence(this._io, room, users, excludeSocketId) + // io.to() with the Redis adapter broadcasts to all pods. + this._io.to(roomName(room)).emit(presenceEventName(room.type), visible) } - async handleWorkflowUpdate(workflowId: string): Promise { - logger.info(`Handling workflow update notification for ${workflowId}`) - - const hasRoom = await this.hasWorkflowRoom(workflowId) - if (!hasRoom) { - logger.debug(`No active room found for updated workflow ${workflowId}`) - return - } - - const timestamp = Date.now() - - this._io.to(workflowId).emit('workflow-updated', { - workflowId, - message: 'Workflow has been updated externally', - timestamp, - }) - - await this.updateRoomLastModified(workflowId) - - const userCount = await this.getUniqueUserCount(workflowId) - logger.info(`Notified ${userCount} users about workflow update: ${workflowId}`) + emitToRoom(room: RoomRef, event: string, payload: T): void { + this._io.to(roomName(room)).emit(event, payload) } - async handleWorkflowDeployed(workflowId: string): Promise { - logger.info(`Handling workflow deployed notification for ${workflowId}`) - - const hasRoom = await this.hasWorkflowRoom(workflowId) - if (!hasRoom) { - logger.debug(`No active room found for deployed workflow ${workflowId}`) - return - } - - this._io.to(workflowId).emit('workflow-deployed', { - workflowId, - timestamp: Date.now(), - }) + async getUniqueUserCount(room: RoomRef): Promise { + const users = await this.getRoomUsers(room) + return new Set(users.map((u) => u.userId)).size + } - const userCount = await this.getUniqueUserCount(workflowId) - logger.info(`Notified ${userCount} users about workflow deployment change: ${workflowId}`) + async getTotalActiveConnections(): Promise { + // Local instance only; the true cross-pod count would require aggregation. + return this._io.sockets.sockets.size } } diff --git a/apps/realtime/src/rooms/types.ts b/apps/realtime/src/rooms/types.ts index 0447516805e..f1a67d66abb 100644 --- a/apps/realtime/src/rooms/types.ts +++ b/apps/realtime/src/rooms/types.ts @@ -1,11 +1,16 @@ +import type { RoomRef, RoomType } from '@sim/realtime-protocol/rooms' +import type { TableCellSelection } from '@sim/realtime-protocol/table-presence' import type { Server } from 'socket.io' /** - * User presence data stored in room state + * User presence data stored in room state. + * + * `room` is the generic room address (see `@sim/realtime-protocol/rooms`). A + * socket may hold presence in more than one room, but only one room per type. */ export interface UserPresence { userId: string - workflowId: string + room: RoomRef userName: string socketId: string tabSessionId?: string @@ -14,11 +19,19 @@ export interface UserPresence { role: string cursor?: { x: number; y: number } selection?: { type: 'block' | 'edge' | 'none'; id?: string } + /** The viewer's current table cell selection, for table presence rooms. */ + cell?: TableCellSelection avatarUrl?: string | null + /** + * The subfolder the user is viewing, recorded at join for room types that track + * a per-viewer location (e.g. the workspace file browser). `null` is the root. + */ + folderId?: string | null } /** - * User session data (minimal info for quick lookups) + * User session data (minimal info for quick lookups). Shared across all rooms a + * socket is in — keyed by socket, not by room. */ export interface UserSession { userId: string @@ -27,125 +40,101 @@ export interface UserSession { } /** - * Workflow room state + * Room presence state. */ -export interface WorkflowRoom { - workflowId: string +export interface RoomState { + room: RoomRef users: Map lastModified: number activeConnections: number } /** - * Common interface for room managers (in-memory and Redis) - * All methods that access state are async to support Redis operations + * Common interface for room managers (in-memory and Redis). + * + * The manager is domain-neutral: it tracks room membership and presence keyed by + * {@link RoomRef}, and knows nothing about workflows, files, or any specific + * domain. Domain lifecycle concerns (e.g. workflow deletion/deploy broadcasts) + * live in domain services that compose a manager — see `WorkflowRoomService`. + * + * A socket may occupy multiple rooms, at most one per {@link RoomType}. The + * shared session key is dropped only when a socket leaves its last room. + * + * All state-accessing methods are async to support the Redis implementation. */ export interface IRoomManager { readonly io: Server - /** - * Initialize the room manager (connect to Redis, etc.) - */ + /** Initialize the manager (connect to Redis, load scripts, etc.). */ initialize(): Promise - /** - * Whether the room manager is ready to serve requests - */ + /** Whether the manager is ready to serve requests. */ isReady(): boolean - /** - * Clean shutdown - */ + /** Clean shutdown. */ shutdown(): Promise - /** - * Add a user to a workflow room - */ - addUserToRoom(workflowId: string, socketId: string, presence: UserPresence): Promise + /** Add a socket's presence to a room. */ + addUserToRoom(room: RoomRef, socketId: string, presence: UserPresence): Promise /** - * Remove a user's membership of a workflow room. - * When workflowIdHint is provided it is the target room; the socket's current - * mapping is only the fallback (and covers missing/expired mapping keys). - * Socket-level mappings are cleared only when the socket is not mapped to a - * different room, so removing a stale room cannot destroy the mapping of a - * room the socket has since moved to. - * Returns the target workflowId, or null when no target could be resolved - * (or, for the Redis manager, when the removal failed). + * Remove a socket from a single room. Returns `true` if it was a member. The + * shared session is dropped only if this was the socket's last room. */ - removeUserFromRoom(socketId: string, workflowIdHint?: string): Promise + removeUserFromRoom(room: RoomRef, socketId: string): Promise /** - * Get the workflow ID for a socket + * Remove a socket from every room it occupies (disconnect). Returns the rooms + * it was in, so the caller can rebroadcast presence per room. */ - getWorkflowIdForSocket(socketId: string): Promise + removeSocketFromAllRooms(socketId: string): Promise - /** - * Get user session data for a socket - */ + /** Every room the socket currently occupies. */ + getRoomsForSocket(socketId: string): Promise + + /** The socket's room of a given type (at most one per type), or `null`. */ + getRoomForSocket(socketId: string, type: RoomType): Promise + + /** Session data for a socket (shared across its rooms). */ getUserSession(socketId: string): Promise - /** - * Get all users in a workflow room - */ - getWorkflowUsers(workflowId: string): Promise + /** All users present in a room. */ + getRoomUsers(room: RoomRef): Promise - /** - * Check if a workflow room exists - */ - hasWorkflowRoom(workflowId: string): Promise + /** Whether a room currently has any presence. */ + hasRoom(room: RoomRef): Promise /** - * Update user activity (cursor, selection, lastActivity) + * Unconditionally drop all state for a room (presence + metadata). Used when a + * room's underlying resource is destroyed (e.g. a deleted workflow) to guarantee + * no state lingers even if per-socket removals failed or a socket joined mid-teardown. */ + deleteRoom(room: RoomRef): Promise + + /** Update a socket's activity (cursor, selection, cell, lastActivity) within a room. */ updateUserActivity( - workflowId: string, + room: RoomRef, socketId: string, - updates: Partial> + updates: Partial> ): Promise - /** - * Update room's lastModified timestamp - */ - updateRoomLastModified(workflowId: string): Promise + /** Bump a room's lastModified timestamp. */ + updateRoomLastModified(room: RoomRef): Promise /** - * Broadcast presence update to all clients in a workflow room + * Broadcast the room's presence list to all clients in the room. Pass + * `excludeSocketId` (e.g. a disconnecting socket) to omit that socket from the + * broadcast even if its presence entry outlived a failed removal — so it is + * never shown as a ghost collaborator. */ - broadcastPresenceUpdate(workflowId: string): Promise + broadcastPresenceUpdate(room: RoomRef, excludeSocketId?: string): Promise - /** - * Emit an event to all clients in a workflow room - */ - emitToWorkflow(workflowId: string, event: string, payload: T): void + /** Emit an event to all clients in a room. */ + emitToRoom(room: RoomRef, event: string, payload: T): void - /** - * Get the number of unique users in a workflow room - */ - getUniqueUserCount(workflowId: string): Promise + /** Number of unique users in a room. */ + getUniqueUserCount(room: RoomRef): Promise - /** - * Get total active connections across all rooms - */ + /** Total active connections tracked by this instance. */ getTotalActiveConnections(): Promise - - /** - * Handle workflow deletion - notify users and clean up room - */ - handleWorkflowDeletion(workflowId: string): Promise - - /** - * Handle workflow revert - notify users - */ - handleWorkflowRevert(workflowId: string, timestamp: number): Promise - - /** - * Handle workflow update - notify users - */ - handleWorkflowUpdate(workflowId: string): Promise - - /** - * Handle workflow deployment change - notify users to refresh deployment state - */ - handleWorkflowDeployed(workflowId: string): Promise } diff --git a/apps/realtime/src/rooms/workflow-room-service.ts b/apps/realtime/src/rooms/workflow-room-service.ts new file mode 100644 index 00000000000..165224b08c4 --- /dev/null +++ b/apps/realtime/src/rooms/workflow-room-service.ts @@ -0,0 +1,126 @@ +import { createLogger } from '@sim/logger' +import { ROOM_TYPES, type RoomRef, roomName } from '@sim/realtime-protocol/rooms' +import type { IRoomManager } from '@/rooms/types' + +const logger = createLogger('WorkflowRoomService') + +/** The workflow room ref for a workflow id. Its Socket.IO room name is the bare id. */ +export function workflowRoom(workflowId: string): RoomRef { + return { type: ROOM_TYPES.WORKFLOW, id: workflowId } +} + +/** + * Workflow-domain lifecycle broadcasts, composed over a domain-neutral + * {@link IRoomManager}. Keeps workflow-specific concerns (deletion, revert, + * update, deploy notifications) out of the generic manager, mirroring how the + * workflow socket handlers own workflow semantics. + */ +export class WorkflowRoomService { + constructor(private readonly manager: IRoomManager) {} + + async handleWorkflowDeletion(workflowId: string): Promise { + logger.info(`Handling workflow deletion notification for ${workflowId}`) + const room = workflowRoom(workflowId) + + const name = roomName(room) + + // Always notify — reach every socket still in the Socket.IO room so the client + // clears the deleted workflow, even if that socket's Redis presence was evicted + // (in which case it would be missing from getRoomUsers). Emitting to an empty + // room is a harmless no-op. + this.manager.emitToRoom(room, 'workflow-deleted', { + workflowId, + message: 'This workflow has been deleted', + timestamp: Date.now(), + }) + + // Clean per-socket state for every socket that is either a live Socket.IO member + // OR still has presence — so an evicted/late-joined socket's room mapping and + // session are dropped too, not just the presence-tracked ones. + const socketIds = new Set() + try { + const liveSockets = await this.manager.io.in(name).fetchSockets() + for (const s of liveSockets) socketIds.add(s.id) + } catch (error) { + logger.warn(`Could not enumerate sockets for deleted workflow ${workflowId}`, error) + } + for (const user of await this.manager.getRoomUsers(room)) socketIds.add(user.socketId) + + // Remove every socket from the Socket.IO room (cross-pod via the Redis adapter). + await this.manager.io.in(name).socketsLeave(name) + + // Independent per-socket removals — run concurrently. + await Promise.all( + Array.from(socketIds, (socketId) => this.manager.removeUserFromRoom(room, socketId)) + ) + + // Final unconditional wipe — the workflow is gone, so no room state may linger + // even if a per-socket removal failed (matches the pre-refactor managers, which + // ended deletion with an unconditional room drop). + await this.manager.deleteRoom(room) + + logger.info( + `Cleaned up workflow room ${workflowId} after deletion (${socketIds.size} sockets removed)` + ) + } + + async handleWorkflowRevert(workflowId: string, timestamp: number): Promise { + logger.info(`Handling workflow revert notification for ${workflowId}`) + const room = workflowRoom(workflowId) + + if (!(await this.manager.hasRoom(room))) { + logger.debug(`No active room found for reverted workflow ${workflowId}`) + return + } + + this.manager.emitToRoom(room, 'workflow-reverted', { + workflowId, + message: 'Workflow has been reverted to deployed state', + timestamp, + }) + + await this.manager.updateRoomLastModified(room) + + const userCount = await this.manager.getUniqueUserCount(room) + logger.info(`Notified ${userCount} users about workflow revert: ${workflowId}`) + } + + async handleWorkflowUpdate(workflowId: string): Promise { + logger.info(`Handling workflow update notification for ${workflowId}`) + const room = workflowRoom(workflowId) + + if (!(await this.manager.hasRoom(room))) { + logger.debug(`No active room found for updated workflow ${workflowId}`) + return + } + + this.manager.emitToRoom(room, 'workflow-updated', { + workflowId, + message: 'Workflow has been updated externally', + timestamp: Date.now(), + }) + + await this.manager.updateRoomLastModified(room) + + const userCount = await this.manager.getUniqueUserCount(room) + logger.info(`Notified ${userCount} users about workflow update: ${workflowId}`) + } + + async handleWorkflowDeployed(workflowId: string): Promise { + logger.info(`Handling workflow deployed notification for ${workflowId}`) + const room = workflowRoom(workflowId) + + if (!(await this.manager.hasRoom(room))) { + logger.debug(`No active room found for deployed workflow ${workflowId}`) + return + } + + this.manager.emitToRoom(room, 'workflow-deployed', { + workflowId, + timestamp: Date.now(), + }) + + const userCount = await this.manager.getUniqueUserCount(room) + logger.info(`Notified ${userCount} users about workflow deployment change: ${workflowId}`) + } +} diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 0f8ed73cc52..3b184eb9f15 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -1,7 +1,9 @@ import type { IncomingMessage, ServerResponse } from 'http' +import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { safeCompare } from '@sim/security/compare' import { env } from '@/env' -import type { IRoomManager } from '@/rooms' +import { applyMarkdownToLiveFileDoc } from '@/handlers/file-doc' +import { type IRoomManager, WorkflowRoomService } from '@/rooms' interface Logger { info: (message: string, ...args: unknown[]) => void @@ -41,6 +43,10 @@ function readRequestBody(req: IncomingMessage): Promise { }) } +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 +} + function sendSuccess(res: ServerResponse): void { res.writeHead(200, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ success: true })) @@ -58,6 +64,8 @@ function sendError(res: ServerResponse, message: string, status = 500): void { * @returns HTTP request handler function */ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { + const workflowRoomService = new WorkflowRoomService(roomManager) + return async (req: IncomingMessage, res: ServerResponse) => { // Health check doesn't require auth if (req.method === 'GET' && req.url === '/health') { @@ -99,7 +107,8 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId } = JSON.parse(body) - await roomManager.handleWorkflowDeletion(workflowId) + if (!isNonEmptyString(workflowId)) return sendError(res, 'Invalid workflowId', 400) + await workflowRoomService.handleWorkflowDeletion(workflowId) sendSuccess(res) } catch (error) { logger.error('Error handling workflow deletion notification:', error) @@ -113,7 +122,8 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId } = JSON.parse(body) - await roomManager.handleWorkflowUpdate(workflowId) + if (!isNonEmptyString(workflowId)) return sendError(res, 'Invalid workflowId', 400) + await workflowRoomService.handleWorkflowUpdate(workflowId) sendSuccess(res) } catch (error) { logger.error('Error handling workflow update notification:', error) @@ -127,7 +137,8 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId } = JSON.parse(body) - await roomManager.handleWorkflowDeployed(workflowId) + if (!isNonEmptyString(workflowId)) return sendError(res, 'Invalid workflowId', 400) + await workflowRoomService.handleWorkflowDeployed(workflowId) sendSuccess(res) } catch (error) { logger.error('Error handling workflow deployed notification:', error) @@ -141,7 +152,8 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { try { const body = await readRequestBody(req) const { workflowId, timestamp } = JSON.parse(body) - await roomManager.handleWorkflowRevert(workflowId, timestamp) + if (!isNonEmptyString(workflowId)) return sendError(res, 'Invalid workflowId', 400) + await workflowRoomService.handleWorkflowRevert(workflowId, timestamp) sendSuccess(res) } catch (error) { logger.error('Error handling workflow revert notification:', error) @@ -150,6 +162,67 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } + // Fan out a file-tree change to everyone viewing a workspace's files, so their + // browser refetches. File mutations happen over the HTTP API (not the socket); + // this is the lossy liveness signal — a missed one only means stale-until-refetch. + if (req.method === 'POST' && req.url === '/api/workspace-files-changed') { + try { + const body = await readRequestBody(req) + const { workspaceId } = JSON.parse(body) + if (!isNonEmptyString(workspaceId)) return sendError(res, 'Invalid workspaceId', 400) + roomManager.emitToRoom( + { type: ROOM_TYPES.WORKSPACE_FILES, id: workspaceId }, + 'workspace-files-changed', + { workspaceId, timestamp: Date.now() } + ) + sendSuccess(res) + } catch (error) { + logger.error('Error handling workspace files changed notification:', error) + sendError(res, 'Failed to process files change notification') + } + return + } + + // Fan out a table-list change to everyone viewing a workspace's tables, so their browser + // refetches. The list-level counterpart to workspace-files-changed; same lossy-signal contract. + if (req.method === 'POST' && req.url === '/api/workspace-tables-changed') { + try { + const body = await readRequestBody(req) + const { workspaceId } = JSON.parse(body) + if (!isNonEmptyString(workspaceId)) return sendError(res, 'Invalid workspaceId', 400) + roomManager.emitToRoom( + { type: ROOM_TYPES.WORKSPACE_TABLES, id: workspaceId }, + 'workspace-tables-changed', + { workspaceId, timestamp: Date.now() } + ) + sendSuccess(res) + } catch (error) { + logger.error('Error handling workspace tables changed notification:', error) + sendError(res, 'Failed to process tables change notification') + } + return + } + + // Merge a copilot edit into a file's LIVE collaborative document so it streams into open editors + // (Stage C). Returns `{ applied }`: when false, no seeded live room exists and the caller writes + // the file directly instead. Any live user edits are preserved — the app builds a minimal CRDT diff. + if (req.method === 'POST' && req.url === '/api/file-doc/apply-edit') { + try { + const body = await readRequestBody(req) + const { fileId, markdown } = JSON.parse(body) + if (!isNonEmptyString(fileId) || typeof markdown !== 'string') { + return sendError(res, 'Invalid fileId or markdown', 400) + } + const result = await applyMarkdownToLiveFileDoc(fileId, markdown) + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ applied: result === 'applied' })) + } catch (error) { + logger.error('Error applying copilot edit to live file-doc:', error) + sendError(res, 'Failed to apply edit to live document') + } + return + } + res.writeHead(404, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ error: 'Not found' })) } diff --git a/apps/sim/app/api/internal/file-doc/merge/route.ts b/apps/sim/app/api/internal/file-doc/merge/route.ts new file mode 100644 index 00000000000..40f7e5204c0 --- /dev/null +++ b/apps/sim/app/api/internal/file-doc/merge/route.ts @@ -0,0 +1,37 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { mergeFileDocContract } from '@/lib/api/contracts/file-doc' +import { parseRequest } from '@/lib/api/server' +import { buildFileDocMergeUpdate } from '@/lib/collab-doc/merge' +import { checkInternalApiKey, createUnauthorizedResponse } from '@/lib/copilot/request/http' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + +const logger = createLogger('FileDocMergeAPI') + +/** + * POST /api/internal/file-doc/merge — merge new markdown into a live collaborative document as a + * minimal Yjs diff (Stage C — copilot writing into an open doc). The realtime relay ships the current + * doc state; the app returns the diff to apply + relay. Internal only: gated on the shared + * `x-api-key: INTERNAL_API_SECRET` secret, matching the seed endpoint and the realtime relay. + */ +export const POST = withRouteHandler(async (request: NextRequest) => { + const auth = checkInternalApiKey(request) + if (!auth.success) return createUnauthorizedResponse() + + const parsed = await parseRequest(mergeFileDocContract, request, {}) + if (!parsed.success) return parsed.response + const { fileId, docState, markdown } = parsed.data.body + + try { + const update = buildFileDocMergeUpdate(Buffer.from(docState, 'base64'), markdown) + return NextResponse.json({ update: Buffer.from(update).toString('base64') }) + } catch (error) { + logger.error('Failed to merge markdown into file-doc', { fileId, error }) + return NextResponse.json( + { error: getErrorMessage(error, 'Failed to merge document') }, + { status: 500 } + ) + } +}) diff --git a/apps/sim/app/api/internal/file-doc/persist/route.ts b/apps/sim/app/api/internal/file-doc/persist/route.ts new file mode 100644 index 00000000000..9ef87415991 --- /dev/null +++ b/apps/sim/app/api/internal/file-doc/persist/route.ts @@ -0,0 +1,43 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { persistFileDocContract } from '@/lib/api/contracts/file-doc' +import { parseRequest } from '@/lib/api/server' +import { persistFileDoc } from '@/lib/collab-doc/persist' +import { checkInternalApiKey, createUnauthorizedResponse } from '@/lib/copilot/request/http' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + +const logger = createLogger('FileDocPersistAPI') + +/** + * POST /api/internal/file-doc/persist — project a live collaborative document back to durable markdown + * (Yjs → markdown, through the exact editor engine) and write it to the file. Internal only: gated on + * the shared `x-api-key: INTERNAL_API_SECRET` secret, matching the header the realtime relay sends. + * The relay owns the live doc but not the conversion engine or blob/DB access, so it ships the current + * doc state here — the server-authoritative durable path that replaces the editor's client autosave. + */ +export const POST = withRouteHandler(async (request: NextRequest) => { + const auth = checkInternalApiKey(request) + if (!auth.success) return createUnauthorizedResponse() + + const parsed = await parseRequest(persistFileDocContract, request, {}) + if (!parsed.success) return parsed.response + const { workspaceId, fileId, userId, docState } = parsed.data.body + + try { + const persisted = await persistFileDoc( + workspaceId, + fileId, + userId, + new Uint8Array(Buffer.from(docState, 'base64')) + ) + return NextResponse.json({ persisted }) + } catch (error) { + logger.error('Failed to persist file-doc', { workspaceId, fileId, error }) + return NextResponse.json( + { error: getErrorMessage(error, 'Failed to persist document') }, + { status: 500 } + ) + } +}) diff --git a/apps/sim/app/api/internal/file-doc/seed/route.test.ts b/apps/sim/app/api/internal/file-doc/seed/route.test.ts new file mode 100644 index 00000000000..f64504f9ded --- /dev/null +++ b/apps/sim/app/api/internal/file-doc/seed/route.test.ts @@ -0,0 +1,70 @@ +/** + * @vitest-environment node + */ +import { createMockRequest } from '@sim/testing' +import { NextResponse } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCheckInternalApiKey, mockBuildFileDocSeed } = vi.hoisted(() => ({ + mockCheckInternalApiKey: vi.fn(), + mockBuildFileDocSeed: vi.fn(), +})) + +vi.mock('@/lib/copilot/request/http', () => ({ + checkInternalApiKey: mockCheckInternalApiKey, + createUnauthorizedResponse: () => NextResponse.json({ error: 'Unauthorized' }, { status: 401 }), +})) + +vi.mock('@/lib/collab-doc/seed', () => ({ + buildFileDocSeed: mockBuildFileDocSeed, +})) + +import { POST } from './route' + +function seedRequest(body: unknown) { + return createMockRequest('POST', body, { 'x-api-key': 'internal' }) +} + +describe('POST /api/internal/file-doc/seed', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckInternalApiKey.mockReturnValue({ success: true }) + }) + + // Regression guard for the auth-helper choice: the realtime relay authenticates with + // `x-api-key: INTERNAL_API_SECRET`, so this route MUST gate on `checkInternalApiKey`. Wiring the + // Bearer-JWT-only `checkInternalAuth` (which forbids `x-api-key`) 401s every real seed fetch. + it('401s when the internal api key is rejected, without building a seed', async () => { + mockCheckInternalApiKey.mockReturnValue({ success: false }) + const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' })) + expect(res.status).toBe(401) + expect(mockBuildFileDocSeed).not.toHaveBeenCalled() + }) + + it('returns the seed as base64 for an authorized request', async () => { + mockBuildFileDocSeed.mockResolvedValue({ update: new Uint8Array([1, 2, 3, 4]) }) + const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' })) + expect(res.status).toBe(200) + expect((await res.json()).update).toBe(Buffer.from([1, 2, 3, 4]).toString('base64')) + expect(mockBuildFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1') + }) + + it('returns update:null for a genuinely absent file', async () => { + mockBuildFileDocSeed.mockResolvedValue(null) + const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'missing' })) + expect(res.status).toBe(200) + expect((await res.json()).update).toBeNull() + }) + + it('400s on a body missing required fields (contract validation, after auth)', async () => { + const res = await POST(seedRequest({ workspaceId: 'ws-1' })) + expect(res.status).toBe(400) + expect(mockBuildFileDocSeed).not.toHaveBeenCalled() + }) + + it('500s when the seed build throws (a read error the relay should retry)', async () => { + mockBuildFileDocSeed.mockRejectedValue(new Error('db down')) + const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' })) + expect(res.status).toBe(500) + }) +}) diff --git a/apps/sim/app/api/internal/file-doc/seed/route.ts b/apps/sim/app/api/internal/file-doc/seed/route.ts new file mode 100644 index 00000000000..058e7a0003a --- /dev/null +++ b/apps/sim/app/api/internal/file-doc/seed/route.ts @@ -0,0 +1,39 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { buildFileDocSeedContract } from '@/lib/api/contracts/file-doc' +import { parseRequest } from '@/lib/api/server' +import { buildFileDocSeed } from '@/lib/collab-doc/seed' +import { checkInternalApiKey, createUnauthorizedResponse } from '@/lib/copilot/request/http' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + +const logger = createLogger('FileDocSeedAPI') + +/** + * POST /api/internal/file-doc/seed — build a server-authoritative collaborative-document seed + * (markdown → Yjs) for the realtime relay to apply on room creation. Internal only: gated on the + * shared `x-api-key: INTERNAL_API_SECRET` secret, matching the header the realtime relay sends + * (`apps/realtime/src/handlers/file-doc-app.ts`) and the realtime server's own inbound validator. + */ +export const POST = withRouteHandler(async (request: NextRequest) => { + const auth = checkInternalApiKey(request) + if (!auth.success) return createUnauthorizedResponse() + + const parsed = await parseRequest(buildFileDocSeedContract, request, {}) + if (!parsed.success) return parsed.response + const { workspaceId, fileId } = parsed.data.body + + try { + const seed = await buildFileDocSeed(workspaceId, fileId) + return NextResponse.json({ + update: seed ? Buffer.from(seed.update).toString('base64') : null, + }) + } catch (error) { + logger.error('Failed to build file-doc seed', { workspaceId, fileId, error }) + return NextResponse.json( + { error: getErrorMessage(error, 'Failed to build seed') }, + { status: 500 } + ) + } +}) diff --git a/apps/sim/app/api/table/[tableId]/columns/route.ts b/apps/sim/app/api/table/[tableId]/columns/route.ts index 227c1422b04..b592e68f3b6 100644 --- a/apps/sim/app/api/table/[tableId]/columns/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/route.ts @@ -19,6 +19,7 @@ import { updateColumnType, } from '@/lib/table' import { columnMatchesRef } from '@/lib/table/column-keys' +import { signalTableSchemaChanged } from '@/lib/table/events' import { accessError, checkAccess, @@ -59,6 +60,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Colum } const updatedTable = await addTableColumn(tableId, validated.column, requestId) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, @@ -190,6 +192,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu if (!updatedTable) { return NextResponse.json({ error: 'No updates specified' }, { status: 400 }) } + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, @@ -257,6 +260,7 @@ export const DELETE = withRouteHandler( { tableId, columnName: validated.columnName }, requestId ) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/events/stream/route.ts b/apps/sim/app/api/table/[tableId]/events/stream/route.ts index 2e9cff2eb37..2142e4a1d93 100644 --- a/apps/sim/app/api/table/[tableId]/events/stream/route.ts +++ b/apps/sim/app/api/table/[tableId]/events/stream/route.ts @@ -1,26 +1,13 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' import { type NextRequest, NextResponse } from 'next/server' import { tableEventStreamContract } from '@/lib/api/contracts/tables' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' -import { SSE_HEADERS } from '@/lib/core/utils/sse' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - getLatestTableEventId, - readTableEventsSince, - type TableEventEntry, -} from '@/lib/table/events' +import { createEventStreamResponse } from '@/lib/realtime/event-stream-route' +import { getLatestTableEventId, readTableEventsSince } from '@/lib/table/events' import { accessError, checkAccess } from '@/app/api/table/utils' -const logger = createLogger('TableEventStreamAPI') - -const POLL_INTERVAL_MS = 500 -const HEARTBEAT_INTERVAL_MS = 15_000 -const MAX_STREAM_DURATION_MS = 4 * 60 * 60 * 1000 // 4 hours; client reconnects past this - export const runtime = 'nodejs' export const dynamic = 'force-dynamic' @@ -30,12 +17,9 @@ interface RouteContext { /** GET /api/table/[tableId]/events/stream?from= * - * SSE stream of cell-state transitions. Replay-on-reconnect via `from`; - * absent `from` tails from the latest event id (fresh mount — the client has - * just fetched current state, so replaying history would rewind it). - * Pruning (buffer cap exceeded or TTL expired) sends a `pruned` event and - * closes; the client responds with a full row-query refetch and reconnects - * tailing from latest. */ + * SSE stream of cell-state transitions over the shared durable event log. Auth + * and access are checked here; the replay/tail/poll/heartbeat/prune mechanics + * come from `createEventStreamResponse`. */ export const GET = withRouteHandler(async (req: NextRequest, context: RouteContext) => { const requestId = generateRequestId() const parsed = await parseRequest(tableEventStreamContract, req, context) @@ -51,123 +35,13 @@ export const GET = withRouteHandler(async (req: NextRequest, context: RouteConte const access = await checkAccess(tableId, auth.userId, 'read') if (!access.ok) return accessError(access, requestId, tableId) - logger.info(`[${requestId}] Table event stream opened`, { tableId, fromEventId }) - - const encoder = new TextEncoder() - let closed = false - - const stream = new ReadableStream({ - async start(controller) { - let lastEventId = fromEventId ?? 0 - const deadline = Date.now() + MAX_STREAM_DURATION_MS - let nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS - - const enqueue = (text: string) => { - if (closed) return - try { - controller.enqueue(encoder.encode(text)) - } catch { - closed = true - } - } - - const sendEvents = (events: TableEventEntry[]) => { - for (const entry of events) { - if (closed) return - enqueue(`data: ${JSON.stringify(entry)}\n\n`) - lastEventId = entry.eventId - } - } - - const sendPrunedAndClose = (earliestEventId: number | undefined) => { - enqueue( - `event: pruned\ndata: ${JSON.stringify({ earliestEventId: earliestEventId ?? null })}\n\n` - ) - if (!closed) { - closed = true - try { - controller.close() - } catch {} - } - } - - const sendHeartbeat = () => { - // SSE comment line — keeps proxies (ALB default 60s idle) from closing - // the connection during quiet periods. - enqueue(`: ping ${Date.now()}\n\n`) - } - - try { - // No replay cursor → tail from the latest event id. Resolved inside - // the try so a Redis failure errors the stream (client reconnects - // with backoff) rather than silently replaying the whole buffer. - if (fromEventId === undefined) { - lastEventId = await getLatestTableEventId(tableId) - } - // Initial replay from buffer. - const initial = await readTableEventsSince(tableId, lastEventId) - if (initial.status === 'pruned') { - sendPrunedAndClose(initial.earliestEventId) - return - } - if (initial.status === 'unavailable') { - throw new Error(`Table event buffer unavailable: ${initial.error}`) - } - sendEvents(initial.events) - - // Stream loop — poll the buffer and forward new events. Workflow - // execution stream uses the same shape; pub/sub wakeups are an - // optimization we can add later if 500ms polling becomes a problem. - while (!closed && Date.now() < deadline) { - await sleep(POLL_INTERVAL_MS) - if (closed) return - - const result = await readTableEventsSince(tableId, lastEventId) - if (result.status === 'pruned') { - sendPrunedAndClose(result.earliestEventId) - return - } - if (result.status === 'unavailable') { - throw new Error(`Table event buffer unavailable: ${result.error}`) - } - if (result.events.length > 0) { - sendEvents(result.events) - } - - if (Date.now() >= nextHeartbeatAt) { - sendHeartbeat() - nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS - } - } - - // Reached the defensive duration ceiling — close cleanly so the client - // reconnects with the latest lastEventId. - if (!closed) { - enqueue(`event: rotate\ndata: {}\n\n`) - closed = true - try { - controller.close() - } catch {} - } - } catch (error) { - logger.error(`[${requestId}] Table event stream error`, { - tableId, - error: toError(error).message, - }) - if (!closed) { - try { - controller.error(error) - } catch {} - } - } - }, - cancel() { - closed = true - logger.info(`[${requestId}] Client disconnected from table event stream`, { tableId }) - }, - }) - - return new NextResponse(stream, { - headers: { ...SSE_HEADERS, 'X-Table-Id': tableId }, + return createEventStreamResponse({ + requestId, + label: 'table', + streamId: tableId, + fromEventId, + getLatestEventId: getLatestTableEventId, + readEventsSince: readTableEventsSince, + extraHeaders: { 'X-Table-Id': tableId }, }) }) diff --git a/apps/sim/app/api/table/[tableId]/groups/route.ts b/apps/sim/app/api/table/[tableId]/groups/route.ts index 8b5de09bee0..c6f2e7c46b4 100644 --- a/apps/sim/app/api/table/[tableId]/groups/route.ts +++ b/apps/sim/app/api/table/[tableId]/groups/route.ts @@ -10,6 +10,7 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { signalTableSchemaChanged } from '@/lib/table/events' import { addWorkflowGroup, deleteWorkflowGroup, @@ -106,6 +107,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro }, requestId ) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, data: { @@ -168,6 +170,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, { params }: R }, requestId ) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, data: { @@ -201,6 +204,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, { params }: { tableId, groupId: validated.groupId }, requestId ) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, data: { diff --git a/apps/sim/app/api/table/[tableId]/import/route.ts b/apps/sim/app/api/table/[tableId]/import/route.ts index 7a9802442cc..8ee9ed8f170 100644 --- a/apps/sim/app/api/table/[tableId]/import/route.ts +++ b/apps/sim/app/api/table/[tableId]/import/route.ts @@ -37,6 +37,7 @@ import { wouldExceedRowLimit, } from '@/lib/table' import { sniffCsvDelimiterFromStream } from '@/lib/table/csv-delimiter-stream' +import { signalTableSchemaChanged } from '@/lib/table/events' import { importAppendRows, importReplaceRows } from '@/lib/table/import-data' import { getUserSettings } from '@/lib/users/queries' import { @@ -323,6 +324,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro mappedColumns: validation.mappedHeaders.length, skippedHeaders: validation.skippedHeaders.length, }) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, @@ -385,6 +387,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro createdColumns: additions.length, mappedColumns: validation.mappedHeaders.length, }) + signalTableSchemaChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/metadata/route.ts b/apps/sim/app/api/table/[tableId]/metadata/route.ts index f85df0af4bd..6883b56038c 100644 --- a/apps/sim/app/api/table/[tableId]/metadata/route.ts +++ b/apps/sim/app/api/table/[tableId]/metadata/route.ts @@ -7,6 +7,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { TableMetadata } from '@/lib/table' import { updateTableMetadata } from '@/lib/table' +import { signalTableMetadataChanged, signalTableSchemaChanged } from '@/lib/table/events' import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableMetadataAPI') @@ -43,11 +44,20 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: TableR return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } - const updated = await updateTableMetadata( + const { metadata: updated, schemaChanged } = await updateTableMetadata( tableId, validated.metadata, table.metadata as TableMetadata | null ) + // Signal collaborators to re-apply the new column layout (width/pin/order) live; the + // grid reconciles against its in-progress resize/drag so a peer's change never clobbers + // the local gesture. A reorder that scrubs a workflow-group's dependencies also mutated + // the schema — escalate to the schema signal so peers refresh run-state/enrichment too. + if (schemaChanged) { + signalTableSchemaChanged(tableId) + } else { + signalTableMetadataChanged(tableId) + } return NextResponse.json({ success: true, data: { metadata: updated } }) } catch (error) { diff --git a/apps/sim/app/api/table/[tableId]/route.ts b/apps/sim/app/api/table/[tableId]/route.ts index 03eaa4c7243..225e7556613 100644 --- a/apps/sim/app/api/table/[tableId]/route.ts +++ b/apps/sim/app/api/table/[tableId]/route.ts @@ -19,6 +19,7 @@ import { updateTableLocks, } from '@/lib/table' import { getWorkspaceTableLimits } from '@/lib/table/billing' +import { signalTableSchemaChanged } from '@/lib/table/events' import { TABLE_LOCK_FLAGS, TABLE_LOCK_KINDS } from '@/lib/table/types' import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' import { @@ -185,6 +186,8 @@ export const PATCH = withRouteHandler( if (validated.name !== undefined) { await renameTable(tableId, validated.name, requestId, authResult.userId) + // Live-collab: tell open viewers the definition changed so they refetch. + signalTableSchemaChanged(tableId) } if (validated.folderId !== undefined) { diff --git a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts index 82ec048ca84..6cb994bff85 100644 --- a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts @@ -15,6 +15,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { RowData, TableSchema } from '@/lib/table' import { deleteRow, updateRow } from '@/lib/table' +import { signalTableRowsChanged } from '@/lib/table/events' import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, @@ -147,6 +148,9 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: RowR // Only `null` when a `cancellationGuard` is supplied and the SQL guard // rejects the write — this route doesn't pass one, so reaching null is a bug. if (!updatedRow) throw new Error('updateRow returned null without a cancellationGuard') + // An edit that also triggers a dispatch already emits dispatch/cell events; the + // debounced rows refetch on the peer coalesces the two. + signalTableRowsChanged(tableId) // Auto-dispatch for user edits is handled inside `updateRow` (mode: 'new'). // Firing a second mode: 'incomplete' dispatch here would race with the // `mode: 'new'` one AND bulk-clear sibling-group outputs (the incomplete @@ -213,6 +217,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Row } await deleteRow(table, rowId, requestId) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 5c73dd8dff1..3c42f757374 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -25,6 +25,7 @@ import { validateRowData, validateRowSize, } from '@/lib/table' +import { signalTableRowsChanged } from '@/lib/table/events' import { queryRows } from '@/lib/table/rows/service' import { TableQueryValidationError } from '@/lib/table/sql' import { rowWireTranslators } from '@/app/api/table/row-wire' @@ -79,6 +80,7 @@ async function handleBatchInsert( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -164,6 +166,7 @@ export const POST = withRouteHandler( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -371,6 +374,7 @@ export const PUT = withRouteHandler( { status: 200 } ) } + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -437,6 +441,7 @@ export const DELETE = withRouteHandler( { tableId, rowIds: validated.rowIds, workspaceId: validated.workspaceId }, requestId ) + if (result.deletedCount > 0) signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -462,6 +467,7 @@ export const DELETE = withRouteHandler( }, requestId ) + if (result.affectedCount > 0) signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -535,6 +541,7 @@ export const PATCH = withRouteHandler( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts b/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts index bc97623ef9a..d6656c11ab4 100644 --- a/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts @@ -8,6 +8,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { RowData, TableSchema } from '@/lib/table' import { upsertRow } from '@/lib/table' +import { signalTableRowsChanged } from '@/lib/table/events' import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' @@ -54,6 +55,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Upser table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts b/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts index f1751ee2120..567834b973f 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts @@ -18,6 +18,7 @@ import { updateColumnType, } from '@/lib/table' import { columnMatchesRef } from '@/lib/table/column-keys' +import { signalTableSchemaChanged } from '@/lib/table/events' import { accessError, checkAccess, @@ -73,6 +74,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Colum } const updatedTable = await addTableColumn(tableId, validated.column, requestId) + signalTableSchemaChanged(tableId) recordAudit({ workspaceId: validated.workspaceId, @@ -224,6 +226,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu if (!updatedTable) { return NextResponse.json({ error: 'No updates specified' }, { status: 400 }) } + signalTableSchemaChanged(tableId) recordAudit({ workspaceId: validated.workspaceId, @@ -309,6 +312,7 @@ export const DELETE = withRouteHandler( { tableId, columnName: validated.columnName }, requestId ) + signalTableSchemaChanged(tableId) recordAudit({ workspaceId: validated.workspaceId, diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts index 5fee4f3d03b..fe2a5a022ab 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts @@ -16,6 +16,7 @@ import type { RowData, TableSchema } from '@/lib/table' import { deleteRow, updateRow } from '@/lib/table' import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, rowDataNameToId } from '@/lib/table/column-keys' +import { signalTableRowsChanged } from '@/lib/table/events' import { accessError, checkAccess, tableLockErrorResponse } from '@/app/api/table/utils' import { checkRateLimit, @@ -159,6 +160,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: RowR if (!updatedRow) { return NextResponse.json({ error: 'Row not found' }, { status: 404 }) } + signalTableRowsChanged(tableId) // Auto-dispatch for user edits is handled inside `updateRow` (mode: 'new'). // Firing a second mode: 'incomplete' dispatch here would race with it AND // bulk-clear sibling-group outputs. @@ -241,6 +243,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Row // Route through the service (not a raw `db.delete`) so the delete lock is // enforced — the raw path would return 200 on a locked table. await deleteRow(result.table, rowId, requestId) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts index cc56cc61183..bf9f8dac719 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts @@ -28,6 +28,7 @@ import { rowDataNameToId, sortNamesToIds, } from '@/lib/table/column-keys' +import { signalTableRowsChanged } from '@/lib/table/events' import { queryRows } from '@/lib/table/rows/service' import { resolveFilterSelectValues } from '@/lib/table/select-values' import { TableQueryValidationError } from '@/lib/table/sql' @@ -89,6 +90,7 @@ async function handleBatchInsert( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -278,6 +280,7 @@ export const POST = withRouteHandler( table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -363,6 +366,7 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: TableR }, requestId ) + if (result.affectedCount > 0) signalTableRowsChanged(tableId) if (result.affectedCount === 0) { return NextResponse.json({ @@ -435,6 +439,7 @@ export const DELETE = withRouteHandler( { tableId, rowIds: validated.rowIds, workspaceId: validated.workspaceId }, requestId ) + if (result.deletedCount > 0) signalTableRowsChanged(tableId) return NextResponse.json({ success: true, @@ -463,6 +468,7 @@ export const DELETE = withRouteHandler( }, requestId ) + if (result.affectedCount > 0) signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/upsert/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/upsert/route.ts index bf4a00df91b..f89bad3096f 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/upsert/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/upsert/route.ts @@ -9,6 +9,7 @@ import type { RowData, TableSchema } from '@/lib/table' import { upsertRow } from '@/lib/table' import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, rowDataNameToId } from '@/lib/table/column-keys' +import { signalTableRowsChanged } from '@/lib/table/events' import { accessError, checkAccess, tableLockErrorResponse } from '@/app/api/table/utils' import { checkRateLimit, @@ -75,6 +76,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Upser table, requestId ) + signalTableRowsChanged(tableId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/workspaces/[id]/files/register/route.ts b/apps/sim/app/api/workspaces/[id]/files/register/route.ts index fd29a6c9dfb..4ed0b90c285 100644 --- a/apps/sim/app/api/workspaces/[id]/files/register/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/register/route.ts @@ -7,6 +7,7 @@ import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' +import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { FileConflictError, parseWorkspaceFileKey, @@ -63,6 +64,8 @@ export const POST = withRouteHandler( if (created) { logger.info(`Registered direct upload ${name} -> ${key}`) + await notifyWorkspaceFilesChanged(workspaceId) + captureServerEvent( userId, 'file_uploaded', diff --git a/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx new file mode 100644 index 00000000000..a76ec009448 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/components/presence/presence-avatars.tsx @@ -0,0 +1,116 @@ +'use client' + +import { Avatar, AvatarFallback, AvatarImage, cn, Tooltip } from '@sim/emcn' +import { getUserColor } from '@/lib/workspaces/colors' + +/** Minimal presence shape the avatar stack renders — shared by workflow and files. */ +export interface PresenceAvatarUser { + /** Unique id per presence entry, used as the render key: a socket id where presence is + * per-connection (workflow), absent where entries are deduped per user (file docs). */ + socketId?: string + userId: string + userName?: string + avatarUrl?: string | null +} + +interface UserAvatarProps { + user: PresenceAvatarUser + index: number +} + +/** + * A single collaborator avatar: their image, falling back to a colored circle + * with their initial. Wrapped in a name tooltip when the name is known. + */ +function UserAvatar({ user, index }: UserAvatarProps) { + const color = getUserColor(user.userId) + const initials = user.userName ? user.userName.charAt(0).toUpperCase() : '?' + + const avatarElement = ( + + {user.avatarUrl && ( + + )} + + {initials} + + + ) + + if (user.userName) { + return ( + + {avatarElement} + + {user.userName} + + + ) + } + + return avatarElement +} + +interface PresenceAvatarsProps { + /** Collaborators to show — already filtered to exclude the current socket. */ + users: PresenceAvatarUser[] + /** Max avatars before collapsing the remainder into a "+N" chip. */ + maxVisible?: number + /** Layout-only classes for the outer stack (e.g. surrounding margin); chrome is owned here. */ + className?: string +} + +const DEFAULT_MAX_VISIBLE = 5 + +/** + * Overlapping stack of collaborator avatars for presence. Presentational only — + * the caller owns fetching/filtering presence (workflow sidebar item, files + * header, etc.), so the stack looks identical everywhere it appears. + */ +export function PresenceAvatars({ + users, + maxVisible = DEFAULT_MAX_VISIBLE, + className, +}: PresenceAvatarsProps) { + // Reverse so the rightmost avatar stays stable as new ones reveal on the left. + // slice() already returns a fresh array, so the in-place reverse is safe. + const visibleUsers = users.slice(0, maxVisible).reverse() + const overflowCount = Math.max(0, users.length - maxVisible) + + if (visibleUsers.length === 0) { + return null + } + + return ( +
+ {overflowCount > 0 && ( + + + + + +{overflowCount} + + + + + {overflowCount} more user{overflowCount > 1 ? 's' : ''} + + + )} + {visibleUsers.map((user, index) => ( + + ))} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index 9984e16b0df..bba37374240 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -110,6 +110,17 @@ interface FileViewerProps { streamIsIncremental?: boolean disableStreamingAutoScroll?: boolean previewContextKey?: string + /** + * Opt this surface into live collaborative editing (markdown files only). Set by the + * Files page; the agent/Chat surface leaves it off so collaboration and agent-streaming + * never target one editor. See {@link RichMarkdownEditorProps.collaborative}. + */ + collaborative?: boolean + /** + * Called (debounced) with the markdown document's leading-heading text while the file is still + * untitled, so the caller can name the file after it. Only wired for the editable markdown editor. + */ + onDeriveTitleFromHeading?: (headingText: string) => void } export function FileViewer(props: FileViewerProps) { @@ -141,6 +152,8 @@ function FileViewerContent({ streamIsIncremental, disableStreamingAutoScroll = false, previewContextKey, + collaborative, + onDeriveTitleFromHeading, }: FileViewerProps) { const category = resolveFileCategory(file.type, file.name) @@ -185,6 +198,8 @@ function FileViewerContent({ streamIsIncremental={streamIsIncremental} disableStreamingAutoScroll={disableStreamingAutoScroll} previewContextKey={previewContextKey} + collaborative={collaborative} + onDeriveTitleFromHeading={onDeriveTitleFromHeading} /> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block-schema.ts new file mode 100644 index 00000000000..5feaeaf85dc --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block-schema.ts @@ -0,0 +1,34 @@ +import type { JSONContent } from '@tiptap/core' +import { CodeBlock } from '@tiptap/extension-code-block' + +/** + * React-free schema half of the code-block node. Lives apart from {@link ./code-block} (its React + * node view) so the shared editor schema — `createMarkdownContentExtensions` in `./extensions` — can + * be imported by server code (the collab-doc seed converter) without pulling a client component + * (`useEffect`) into a Server Component module. The client editor injects the node-view variant + * ({@link CodeBlockWithLanguage}) via `nodeViews`. + */ + +function codeBlockText(node: JSONContent): string { + return (node.content ?? []).map((child) => child.text ?? '').join('') +} + +/** Fence sized to one backtick longer than the longest run inside the code (CommonMark rule). */ +function fenceFor(text: string): string { + const longestRun = Math.max(0, ...[...text.matchAll(/`+/g)].map((match) => match[0].length)) + return '`'.repeat(Math.max(3, longestRun + 1)) +} + +/** + * Code block whose markdown serializer sizes the fence to the interior backtick runs, so a code + * block that itself contains a ``` line round-trips instead of shattering. Shared by the test + * (plain) and live ({@link CodeBlockWithLanguage}) paths. + */ +export const MarkdownCodeBlock = CodeBlock.extend({ + renderMarkdown: (node: JSONContent) => { + const language = typeof node.attrs?.language === 'string' ? node.attrs.language : '' + const text = codeBlockText(node) + const fence = fenceFor(text) + return `${fence}${language}\n${text}\n${fence}` + }, +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block.tsx index 7db9b1481b9..2464ccaec44 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/code-block.tsx @@ -8,12 +8,11 @@ import { DropdownMenuTrigger, useCopyToClipboard, } from '@sim/emcn' -import type { JSONContent } from '@tiptap/core' -import { CodeBlock } from '@tiptap/extension-code-block' import type { ReactNodeViewProps } from '@tiptap/react' import { NodeViewContent, NodeViewWrapper, ReactNodeViewRenderer } from '@tiptap/react' import { Check, ChevronDown, Code, Copy, Eye, WrapText } from 'lucide-react' import { looksLikeMermaid, MermaidDiagram } from '../mermaid-diagram' +import { MarkdownCodeBlock } from './code-block-schema' import { detectLanguage } from './detect-language' import { useEditorEditable } from './use-editor-editable' @@ -228,30 +227,6 @@ function CodeBlockView({ node, updateAttributes, editor, getPos }: ReactNodeView ) } -function codeBlockText(node: JSONContent): string { - return (node.content ?? []).map((child) => child.text ?? '').join('') -} - -/** Fence sized to one backtick longer than the longest run inside the code (CommonMark rule). */ -function fenceFor(text: string): string { - const longestRun = Math.max(0, ...[...text.matchAll(/`+/g)].map((match) => match[0].length)) - return '`'.repeat(Math.max(3, longestRun + 1)) -} - -/** - * Code block whose markdown serializer sizes the fence to the interior backtick runs, so a code - * block that itself contains a ``` line round-trips instead of shattering. Shared by the test - * (plain) and live ({@link CodeBlockWithLanguage}) paths. - */ -export const MarkdownCodeBlock = CodeBlock.extend({ - renderMarkdown: (node: JSONContent) => { - const language = typeof node.attrs?.language === 'string' ? node.attrs.language : '' - const text = codeBlockText(node) - const fence = fenceFor(text) - return `${fence}${language}\n${text}\n${fence}` - }, -}) - /** * Code block with hover-revealed controls (language picker, line-wrap toggle, copy). The * `language` attribute drives {@link CodeBlockHighlight}'s Prism highlighting and serializes to diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.test.ts new file mode 100644 index 00000000000..523335ad8e5 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.test.ts @@ -0,0 +1,55 @@ +/** + * @vitest-environment jsdom + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { activateCaretLabel, CARET_LABEL_HOLD_MS, renderCaret } from './caret-presence' + +const ACTIVE = 'collaboration-carets__caret--active' +const FLIP = 'collaboration-carets__caret--flip' + +describe('caret-presence', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + it('builds a tagged caret with a name label, shown on appearance', () => { + const caret = renderCaret({ name: 'Ada', color: '#f783ac', clientId: 4242 }) + expect(caret.classList.contains('collaboration-carets__caret')).toBe(true) + expect(caret.dataset.caretClientId).toBe('4242') + expect(caret.style.getPropertyValue('--caret-color')).toBeTruthy() + const label = caret.querySelector('.collaboration-carets__label') + expect(label?.textContent).toBe('Ada') + expect(caret.classList.contains(ACTIVE)).toBe(true) + }) + + it('falls back to a default name for a bare user state', () => { + const caret = renderCaret({ clientId: 1 }) + expect(caret.querySelector('.collaboration-carets__label')?.textContent).toBe('Collaborator') + }) + + it('hides the label after the inactivity hold, and re-activation restarts it', () => { + const caret = renderCaret({ name: 'Ada', color: '#f783ac', clientId: 4242 }) + vi.advanceTimersByTime(CARET_LABEL_HOLD_MS - 1) + expect(caret.classList.contains(ACTIVE)).toBe(true) + vi.advanceTimersByTime(1) + expect(caret.classList.contains(ACTIVE)).toBe(false) + + activateCaretLabel(caret) + expect(caret.classList.contains(ACTIVE)).toBe(true) + vi.advanceTimersByTime(CARET_LABEL_HOLD_MS) + expect(caret.classList.contains(ACTIVE)).toBe(false) + }) + + it('flips the label left only when it would overflow the editor right edge', () => { + const caret = renderCaret({ name: 'Ada', color: '#f783ac', clientId: 4242 }) + const label = caret.querySelector('.collaboration-carets__label') + if (!label) throw new Error('label missing') + // double-cast-allowed: jsdom has no layout; stub the label's right edge for the measure + label.getBoundingClientRect = () => ({ right: 500 }) as unknown as DOMRect + + activateCaretLabel(caret, 600) // editor edge past the label → no flip + expect(caret.classList.contains(FLIP)).toBe(false) + + activateCaretLabel(caret, 400) // editor edge before the label's right → flip + expect(caret.classList.contains(FLIP)).toBe(true) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts new file mode 100644 index 00000000000..fdbfadce592 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/caret-presence.ts @@ -0,0 +1,156 @@ +import { Extension } from '@tiptap/core' +import { Plugin, PluginKey } from '@tiptap/pm/state' +import type { Awareness } from 'y-protocols/awareness' + +/** + * Remote-collaborator caret presence for the file editor: the name label's + * show-then-fade behavior and the edge-aware flip, plus the ProseMirror plugin + * that drives them off awareness activity. + * + * Matches Google Docs: the name flag shows only while the peer has typed in the last + * few seconds or on local hover, then hides after inactivity, leaving just the caret. + */ + +/** + * How long a peer's name label stays visible after their last activity (cursor + * move / edit) before it fades, leaving just the colored caret bar. + */ +export const CARET_LABEL_HOLD_MS = 2000 + +/** Fallback caret color when a peer's awareness carries no `color`. */ +export const DEFAULT_CARET_COLOR = '#000000' + +/** + * The active-state class {@link activateCaretLabel} toggles on the caret node to reveal the + * name label; CSS transitions it back to hidden when removed. (yCursorPlugin reuses the DOM + * node and never re-runs `render`, which is why this file drives the class itself.) + */ +const CARET_ACTIVE_CLASS = 'collaboration-carets__caret--active' + +/** The class that flips the label to the caret's left near the editor's right edge. */ +const CARET_FLIP_CLASS = 'collaboration-carets__caret--flip' + +/** Per-caret fade timers, keyed by the (reused) caret DOM node. */ +const caretFadeTimers = new WeakMap>() + +/** + * Show a peer's name label, (re)start its fade timer, and flip it to the caret's + * left when it would run off the editor's right edge. + * + * yCursorPlugin renders each caret as a keyed widget decoration, so ProseMirror + * REUSES the same DOM node as the caret moves (verified: the render function is + * not re-invoked on a position change) — a CSS animation therefore cannot restart + * on activity. Instead the activity signal is the awareness `change` event, which + * (unlike `update`) fires only on a real state change, not the 15s heartbeat, so + * an idle peer's label correctly stays hidden. + */ +export function activateCaretLabel(caret: HTMLElement, editorRight?: number) { + caret.classList.add(CARET_ACTIVE_CLASS) + const existing = caretFadeTimers.get(caret) + if (existing) clearTimeout(existing) + caretFadeTimers.set( + caret, + setTimeout(() => { + caret.classList.remove(CARET_ACTIVE_CLASS) + caretFadeTimers.delete(caret) + }, CARET_LABEL_HOLD_MS) + ) + + if (editorRight === undefined) return + const label = caret.querySelector('.collaboration-carets__label') + if (!label) return + // Measure the default (rightward) position, then flip left only if it overflows. + caret.classList.remove(CARET_FLIP_CLASS) + if (label.getBoundingClientRect().right > editorRight) { + caret.classList.add(CARET_FLIP_CLASS) + } +} + +/** + * Builds a remote peer's caret DOM: a colored bar plus a name label tagged with + * the peer's Yjs client id (so {@link createCaretActivityExtension} can find and + * re-activate the reused node on later awareness changes). Shown immediately on + * (re)appearance; the fade timer hides it after inactivity. Passed to + * CollaborationCaret as its `render` option, which only supplies `user` — so the + * client id rides along in the awareness `user` payload (each client stamps its own + * `doc.clientID`; see `use-file-doc-collaboration.ts`). + */ +export function renderCaret(user: Record): HTMLElement { + const color = typeof user.color === 'string' ? user.color : DEFAULT_CARET_COLOR + const name = typeof user.name === 'string' && user.name ? user.name : 'Collaborator' + const clientId = typeof user.clientId === 'number' ? user.clientId : undefined + const caret = document.createElement('span') + caret.className = 'collaboration-carets__caret' + // One inline var drives the caret bar, the dormant cap, and the name tag (all in CSS). + caret.style.setProperty('--caret-color', color) + if (clientId !== undefined) caret.dataset.caretClientId = String(clientId) + const label = document.createElement('div') + label.className = 'collaboration-carets__label' + label.textContent = name + caret.appendChild(label) + activateCaretLabel(caret) + return caret +} + +/** + * Drives the caret name-label show-then-fade off awareness activity. Because the + * caret DOM node is reused across moves (see {@link activateCaretLabel}), the + * `render` function alone can't reveal the label when a peer moves — this listens + * for awareness `change` events and re-activates the matching caret node. Deferred + * to the next frame so the node exists and is laid out (for the edge-flip measure). + */ +export function createCaretActivityExtension(awareness: Awareness): Extension { + return Extension.create({ + name: 'collaborationCaretActivity', + addProseMirrorPlugins() { + return [ + new Plugin({ + key: new PluginKey('collaborationCaretActivity'), + view: (editorView) => { + // Coalesce bursts of awareness changes into a single rAF per frame, however many + // peers moved: accumulate the changed client ids, then re-activate each matching + // (reused) caret node once. The shared editorRight is read once up front; each moving + // caret then does one getBoundingClientRect for its edge-flip measure (bounded by the + // small number of concurrently-moving peers, not the awareness event rate). + let raf = 0 + const pending = new Set() + const flush = () => { + raf = 0 + const editorRight = editorView.dom.getBoundingClientRect().right + for (const id of pending) { + const caret = editorView.dom.querySelector( + `.collaboration-carets__caret[data-caret-client-id="${id}"]` + ) + if (caret) activateCaretLabel(caret, editorRight) + } + pending.clear() + } + const onChange = ({ added, updated }: { added: number[]; updated: number[] }) => { + for (const id of added) pending.add(id) + for (const id of updated) pending.add(id) + if (pending.size > 0 && !raf) raf = requestAnimationFrame(flush) + } + awareness.on('change', onChange) + return { + destroy: () => { + awareness.off('change', onChange) + if (raf) cancelAnimationFrame(raf) + // Clear any pending fade timers for this editor's carets so they don't fire on + // detached nodes after unmount (harmless no-op, but a genuine leaked timer). + for (const caret of editorView.dom.querySelectorAll( + '.collaboration-carets__caret' + )) { + const timer = caretFadeTimers.get(caret) + if (timer) { + clearTimeout(timer) + caretFadeTimers.delete(caret) + } + } + }, + } + }, + }), + ] + }, + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx new file mode 100644 index 00000000000..1d093b21fa2 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars.tsx @@ -0,0 +1,14 @@ +'use client' + +import { PresenceAvatars } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' +import { useFileDocOthers } from './file-doc-room-context' + +/** + * Avatar stack of the collaborators currently in the open file — the `useOthers` avatar + * stack, reading the room roster from {@link useFileDocOthers}. Renders nothing until + * someone else joins. Must sit inside a `FileDocRoomProvider`. + */ +export function FileDocAvatars() { + const others = useFileDocOthers() + return +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts new file mode 100644 index 00000000000..29e31b53301 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -0,0 +1,372 @@ +/** + * @vitest-environment node + */ +import { + FILE_DOC_EVENTS, + FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SEED, +} from '@sim/realtime-protocol/file-doc' +import * as encoding from 'lib0/encoding' +import type { Socket } from 'socket.io-client' +import { describe, expect, it, vi } from 'vitest' +import * as awarenessProtocol from 'y-protocols/awareness' +import * as syncProtocol from 'y-protocols/sync' +import * as Y from 'yjs' +import { FileDocProvider } from './file-doc-provider' + +/** A minimal fake Socket.IO client whose server→client events can be fired in tests. */ +function createSocket(connected = true) { + const listeners = new Map void>>() + const emit = vi.fn() + const socket = { + connected, + emit, + on(event: string, cb: (...args: unknown[]) => void) { + let set = listeners.get(event) + if (!set) { + set = new Set() + listeners.set(event, set) + } + set.add(cb) + }, + off(event: string, cb: (...args: unknown[]) => void) { + listeners.get(event)?.delete(cb) + }, + } + const fire = (event: string, ...args: unknown[]) => { + for (const cb of listeners.get(event) ?? []) cb(...args) + } + return { socket: socket as unknown as Socket, emit, fire } +} + +function createProvider(connected = true) { + const { socket, emit, fire } = createSocket(connected) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness) + return { provider, doc, awareness, emit, fire } +} + +/** Messages emitted to the server, decoded to their `{ type, bytes }`. */ +function emittedMessages(emit: ReturnType) { + return emit.mock.calls + .filter( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + .map(([, payload]) => payload as Uint8Array) +} + +describe('FileDocProvider', () => { + it('joins immediately with its client id when the socket is already connected', () => { + const { doc, emit } = createProvider(true) + expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, { + fileId: 'file-1', + clientId: doc.clientID, + }) + }) + + it('waits for connect before joining when the socket is offline', () => { + const { emit, fire } = createProvider(false) + expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + fire('connect') + expect(emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN, + expect.objectContaining({ fileId: 'file-1' }) + ) + }) + + it('exchanges sync only after JOIN_SUCCESS', () => { + const { emit, fire } = createProvider(true) + emit.mockClear() + + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + + // A sync step 1 (type tag 0) is sent to exchange state with the server. + const messages = emittedMessages(emit) + expect(messages.length).toBeGreaterThan(0) + expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + }) + + it('ignores a join ack for a different file', () => { + const { emit, fire } = createProvider(true) + emit.mockClear() + + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'other-file' }) + + // No sync/awareness exchange starts for a file this provider does not own. + expect(emittedMessages(emit)).toHaveLength(0) + }) + + it('applies a server sync step 2 and becomes synced', () => { + const { provider, doc, fire } = createProvider(true) + const synced = vi.fn() + provider.on('synced', synced) + + const serverDoc = new Y.Doc() + serverDoc.getText('default').insert(0, 'hello world') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + expect(doc.getText('default').toString()).toBe('hello world') + expect(provider.synced).toBe(true) + expect(synced).toHaveBeenCalledWith(true) + }) + + it('sends local document edits to the server as sync updates', () => { + const { doc, emit } = createProvider(true) + emit.mockClear() + + doc.getText('default').insert(0, 'x') + + const messages = emittedMessages(emit) + expect(messages.length).toBe(1) + expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + }) + + it('does not echo updates it applied from the server', () => { + const { provider, emit, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + emit.mockClear() + + const serverDoc = new Y.Doc() + serverDoc.getText('default').insert(0, 'remote') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(serverDoc)) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + // The applied remote update must not be re-emitted back to the server. + expect(emittedMessages(emit)).toHaveLength(0) + expect(provider.doc.getText('default').toString()).toBe('remote') + }) + + it('sends local awareness (cursor/selection) changes', () => { + const { awareness, emit } = createProvider(true) + emit.mockClear() + + awareness.setLocalStateField('user', { name: 'Ada', color: '#f783ac' }) + + const messages = emittedMessages(emit) + expect(messages.some((m) => m[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS)).toBe(true) + }) + + it('reseeds a cleared awareness so a reused instance can publish again', () => { + const { socket, emit } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + // Simulate a prior provider teardown having cleared the local state — after + // this, y-protocols' setLocalStateField is a permanent no-op, so the caret + // extension could never publish the local user/cursor on a reused instance. + awarenessProtocol.removeAwarenessStates(awareness, [doc.clientID], 'prior-destroy') + expect(awareness.getLocalState()).toBeNull() + + // Constructing a provider on the reused, cleared awareness must restore it. + new FileDocProvider(socket, 'file-1', doc, awareness) + expect(awareness.getLocalState()).not.toBeNull() + + emit.mockClear() + // The caret extension setting the user field must now actually publish. + awareness.setLocalStateField('user', { name: 'Ada', color: '#f783ac' }) + expect(emittedMessages(emit).some((m) => m[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS)).toBe(true) + }) + + it('does not forward awareness it applied from the server', () => { + const { emit, fire } = createProvider(true) + emit.mockClear() + + const remoteDoc = new Y.Doc() + remoteDoc.clientID = 8888 + const remoteAwareness = new awarenessProtocol.Awareness(remoteDoc) + remoteAwareness.setLocalStateField('user', { name: 'Remote' }) + const update = awarenessProtocol.encodeAwarenessUpdate(remoteAwareness, [8888]) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array(encoder, update) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + // The remote peer's awareness (client 8888, not ours) must not be re-published. + expect(emittedMessages(emit).some((m) => m[0] === FILE_DOC_MESSAGE_TYPE.AWARENESS)).toBe(false) + }) + + it('stops attempting to join and latches joinError after a non-retryable error', () => { + const { provider, emit, fire } = createProvider(true) + emit.mockClear() + + const error = { + fileId: 'file-1', + error: 'Access denied', + code: 'ACCESS_DENIED', + retryable: false, + } + fire(FILE_DOC_EVENTS.JOIN_ERROR, error) + fire('connect') + + expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + // Latched so a consumer subscribing after the event can still detect the failure. + expect(provider.joinError).toEqual(error) + }) + + it('still rejoins on reconnect after a retryable error', () => { + const { emit, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_ERROR, { + fileId: 'file-1', + error: 'Realtime unavailable', + code: 'ROOM_MANAGER_UNAVAILABLE', + retryable: true, + }) + emit.mockClear() + + fire('connect') + + expect(emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN, + expect.objectContaining({ fileId: 'file-1' }) + ) + }) + + it('resets synced and rejoins on a reconnect', () => { + const { provider, emit, fire } = createProvider(true) + // Become synced. + const serverDoc = new Y.Doc() + serverDoc.getText('default').insert(0, 'hi') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + emit.mockClear() + + // A reconnect must drop synced and re-issue JOIN so the doc re-syncs. + fire('connect') + + expect(provider.synced).toBe(false) + expect(emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN, + expect.objectContaining({ fileId: 'file-1' }) + ) + }) + + it('leaves the room and detaches on destroy', () => { + const { provider, doc, emit } = createProvider(true) + emit.mockClear() + + provider.destroy() + + expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.LEAVE, { fileId: 'file-1' }) + // After destroy, local edits are no longer forwarded. + emit.mockClear() + doc.getText('default').insert(0, 'y') + expect(emittedMessages(emit)).toHaveLength(0) + }) + + it('gives up with a non-retryable join-error when the first sync never arrives (offline)', () => { + vi.useFakeTimers() + try { + const { provider, emit, fire } = createProvider(false) // socket never connects + const onError = vi.fn() + provider.on('join-error', onError) + + vi.advanceTimersByTime(12_000) + + // Surfaces the same non-retryable rejection the fatal path uses, so the editor falls back to + // showing the file read-only. + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + ) + expect(provider.joinError).toEqual( + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + ) + // Latched fatal: a later connect must not re-join (which could sync server state in and + // duplicate the locally-seeded content). + emit.mockClear() + fire('connect') + expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + } finally { + vi.useRealTimers() + } + }) + + it('does not fire the fallback once the doc is synced AND seeded', () => { + vi.useFakeTimers() + try { + const { provider, fire } = createProvider(true) + const onError = vi.fn() + provider.on('join-error', onError) + + // The initial sync brings BOTH content and the server seed flag before the deadline. + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + const remote = new Y.Doc() + remote.getText('default').insert(0, 'hi') + remote.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, remote) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + + vi.advanceTimersByTime(12_000) + expect(onError).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('fires the fallback when the doc synced but the server seed never landed', () => { + vi.useFakeTimers() + try { + const { provider, fire } = createProvider(true) + const onError = vi.fn() + provider.on('join-error', onError) + + // The socket syncs an empty doc, but the server-side seed never arrives (its build persistently + // failed) — `synced` is true yet `initialContentLoaded` is never set. + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + const remote = new Y.Doc() + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, remote) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + + vi.advanceTimersByTime(12_000) + + // The readiness deadline still fires → the editor falls back to the stored content read-only, + // and `synced` is dropped so the `synced && seeded` gate stays closed (read-only, not editable). + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + ) + expect(provider.synced).toBe(false) + } finally { + vi.useRealTimers() + } + }) + + it('ignores a late SyncStep2 that arrives after the readiness deadline (no merge, stays gated)', () => { + vi.useFakeTimers() + try { + const { provider, doc, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1' }) + + // Deadline lapses with no first sync → fatal fallback (editor falls back to a read-only seed). + vi.advanceTimersByTime(12_000) + expect(provider.joinError).toEqual(expect.objectContaining({ code: 'READINESS_TIMEOUT' })) + + // A delayed SyncStep2 finally arrives. Applying it would merge server content into the + // already-seeded doc (duplication) and flip synced→true (un-gating autosave), so it MUST be + // dropped once fatal. + const remote = new Y.Doc() + remote.getText('default').insert(0, 'server content') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, remote) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + expect(provider.synced).toBe(false) + expect(doc.getText('default').toString()).toBe('') + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts new file mode 100644 index 00000000000..6049616f2ba --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -0,0 +1,318 @@ +import { + FILE_DOC_EVENTS, + FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SEED, + FILE_DOC_TIMEOUTS, + type JoinFileDocError, + type JoinFileDocSuccess, + toFileDocBytes, +} from '@sim/realtime-protocol/file-doc' +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import { ObservableV2 } from 'lib0/observable' +import type { Socket } from 'socket.io-client' +import * as awarenessProtocol from 'y-protocols/awareness' +import * as syncProtocol from 'y-protocols/sync' +import type * as Y from 'yjs' + +/** + * Events emitted by {@link FileDocProvider}. + * - `synced`: the first full document sync with the server completed. + * - `join-error`: the server rejected the join (e.g. lost write access). + */ +interface FileDocProviderEvents { + synced: (synced: boolean) => void + 'join-error': (error: JoinFileDocError) => void +} + +/** + * How long to wait to reach a USABLE editor — connected, synced, AND seeded (`initialContentLoaded` + * set by the server seed) — before giving up. It guards two failure modes with one timer: + * - the realtime server is unreachable, so the first sync never arrives; and + * - the socket syncs an empty doc but the server-side seed never lands (its build persistently fails + * / exhausts its retries), which `synced` alone would wrongly treat as "connected, all good". + * + * On the deadline the provider latches fatal and surfaces a non-retryable `join-error` — the exact + * path a fatal rejection uses — so the editor falls back to showing the file's stored content + * read-only instead of a permanently blank pane. Generous enough to clear a slow connect + seed + * round-trip; a healthy cold open reaches readiness well within it. Shared with (and must exceed) the + * relay's seed-fetch timeout — see `FILE_DOC_TIMEOUTS` and its ordering test. + */ +const READINESS_DEADLINE_MS = FILE_DOC_TIMEOUTS.readinessDeadlineMs + +/** + * The client half of the collaborative file-document protocol: a Yjs provider + * that carries document sync + awareness over the shared, already-authenticated + * Socket.IO connection (the server relay lives in + * `apps/realtime/src/handlers/file-doc.ts`). It is the Socket.IO analogue of + * `y-websocket`'s `WebsocketProvider` — the same `y-protocols` message framing — + * so TipTap's `Collaboration` (bound to {@link doc}) and `CollaborationCaret` + * (bound to this provider's {@link awareness}) work unmodified. + * + * The document and awareness are owned by the caller (the hook) and are NOT + * destroyed here, so the provider can be torn down and rebuilt (e.g. on a socket + * reconnect) without discarding local edits. + */ +export class FileDocProvider extends ObservableV2 { + synced = false + /** + * The latched non-retryable join rejection, or `null`. The `join-error` event is + * transient and can fire before a consumer subscribes, + * so consumers read this on subscription to detect a fatal failure they missed. + */ + joinError: JoinFileDocError | null = null + + private disposed = false + /** Set on a non-retryable join rejection (e.g. lost write access) so the + * provider stops attempting to (re)join until the owner tears it down. */ + private fatal = false + /** Deadline for reaching readiness (synced + seeded); fires the fallback if it is never reached. */ + private readinessTimer: ReturnType | null = null + + constructor( + private readonly socket: Socket, + private readonly fileId: string, + readonly doc: Y.Doc, + readonly awareness: awarenessProtocol.Awareness + ) { + super() + + // Restore an empty local awareness state if it has been cleared. A fresh + // Awareness starts with `{}`, but a *reused* one whose local state was removed + // (a prior provider's `destroy()` clears it, and so does `Awareness.destroy()`) + // returns `null` here — and y-protocols' `setLocalStateField` is a no-op while + // the local state is `null`. The editor binds CollaborationCaret to this exact + // awareness for its whole life, so without this reseed a remount (e.g. React + // StrictMode's mount→unmount→mount, which re-runs the provider effect on the + // same instance) would leave the caret extension unable to ever publish the + // local user/cursor — remote peers would see no caret or selection, even though + // document sync (which does not depend on local awareness) keeps working. + if (awareness.getLocalState() === null) awareness.setLocalState({}) + + socket.on(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) + socket.on(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) + socket.on(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + socket.on('connect', this.handleConnect) + doc.on('update', this.handleDocUpdate) + awareness.on('update', this.handleAwarenessUpdate) + // Watch the seed flag so reaching "seeded" (server seed applied) can clear the readiness deadline. + doc.getMap(FILE_DOC_SEED.configMap).observe(this.handleConfigChange) + + if (socket.connected) this.join() + + // Arm the fallback: if we don't reach readiness (synced + seeded) before the deadline, give up. + this.readinessTimer = setTimeout(this.handleReadinessDeadline, READINESS_DEADLINE_MS) + } + + /** Whether the server seed has recorded the initial content on the doc. */ + private isSeeded(): boolean { + return this.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag) === true + } + + /** Clear the readiness deadline once the editor is usable (synced AND seeded). */ + private handleConfigChange = () => { + if (this.synced && this.isSeeded()) this.clearReadinessTimer() + } + + /** + * Readiness was never reached within {@link READINESS_DEADLINE_MS} — either the realtime server is + * unreachable (never synced) or it synced but the server-side seed never landed (synced yet + * unseeded). Reset `synced` (so the editor gates read-only), latch fatal (so a late reconnect or + * seed can't sync server state in and merge-duplicate the content the editor is about to render + * locally), and surface a synthetic non-retryable join-error — the exact path a fatal rejection + * uses — so the owner falls back to the read-only view of the file's stored content instead of a + * blank pane. No-op if we already reached readiness, already failed fatally, or were torn down. + */ + private handleReadinessDeadline = () => { + this.readinessTimer = null + if ((this.synced && this.isSeeded()) || this.fatal || this.disposed) return + const error: JoinFileDocError = { + fileId: this.fileId, + error: 'Realtime document was not ready in time', + code: 'READINESS_TIMEOUT', + retryable: false, + } + this.fatal = true + this.joinError = error + // Drop `synced` so the editor's `synced && seeded` gate stays closed → the fallback renders the + // stored content read-only rather than becoming editable on a doc the server never seeded. + this.setSynced(false) + this.emit('join-error', [error]) + } + + private clearReadinessTimer() { + if (this.readinessTimer !== null) { + clearTimeout(this.readinessTimer) + this.readinessTimer = null + } + } + + /** Join the room, binding our client id so the server only accepts awareness we own. */ + private join = () => { + if (this.fatal) return + this.socket.emit(FILE_DOC_EVENTS.JOIN, { fileId: this.fileId, clientId: this.doc.clientID }) + } + + /** + * Re-join after a (re)connect. The server re-registers the room before acking, + * so the sync/awareness exchange is deferred to {@link handleJoinSuccess}. + */ + private handleConnect = () => { + if (this.fatal) return + this.setSynced(false) + this.join() + } + + /** + * Handle the join ack. The server registers the room before acking, so an earlier + * send could be dropped — the initial sync + local awareness exchange begins here. + */ + private handleJoinSuccess = (data: JoinFileDocSuccess) => { + if (data.fileId !== this.fileId) return + this.sendSyncStep1() + this.sendLocalAwareness() + } + + /** + * Handle a join rejection. A non-retryable rejection (access denied, invalid) + * won't succeed on retry, so latch {@link fatal} to stop (re)joining and let the + * owner fall back to the non-collaborative view. + */ + private handleJoinError = (data: JoinFileDocError) => { + if (data.fileId !== this.fileId) return + if (data.retryable === false) { + this.fatal = true + this.joinError = data + this.clearReadinessTimer() + } + this.emit('join-error', [data]) + } + + private handleMessage = (data: unknown) => { + // Once we've given up (a non-retryable rejection, or the connect deadline lapsed and the editor + // fell back to a read-only local seed), ignore ALL inbound frames. A late SyncStep2 arriving + // after the deadline would otherwise merge the server's state into the already-seeded doc — + // duplicating content — and flip `synced` true, which un-gates autosave and would persist the + // duplicate back to the real file. `fatal` guarding (re)join alone is not enough; it must also + // stop applying sync here. + if (this.fatal) return + const bytes = toFileDocBytes(data) + if (!bytes) return + + const decoder = decoding.createDecoder(bytes) + const messageType = decoding.readVarUint(decoder) + + switch (messageType) { + case FILE_DOC_MESSAGE_TYPE.SYNC: { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + // `this` is the transaction origin, so our own `doc.on('update')` skips + // re-sending updates we just applied from the server. + const syncType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this) + if (encoding.length(encoder) > 1) { + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + if (syncType === syncProtocol.messageYjsSyncStep2 && !this.synced) this.setSynced(true) + break + } + case FILE_DOC_MESSAGE_TYPE.AWARENESS: { + awarenessProtocol.applyAwarenessUpdate( + this.awareness, + decoding.readVarUint8Array(decoder), + this + ) + break + } + } + } + + private handleDocUpdate = (update: Uint8Array, origin: unknown) => { + // Once fatal (a non-retryable rejection, or the readiness deadline lapsed), the editor may render + // the stored content into the doc locally as its read-only fallback. Never relay those local + // writes — the server never seeded this doc, so echoing them would push unseeded content to peers + // (and each fallen-back client would do so, union-duplicating). A fatal client is fully local. + if (this.fatal) return + // Updates we applied from the server carry `this` as origin — don't echo them. + if (origin === this) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, update) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + + private handleAwarenessUpdate = ( + { added, updated, removed }: { added: number[]; updated: number[]; removed: number[] }, + origin: unknown + ) => { + // Only ever publish OUR OWN awareness. Remote changes (origin === this) were + // applied from the server; and a local `Awareness` also emits 30s `timeout` + // removals for remote peers — forwarding either would be a frame for a client + // id we don't own, which the server (correctly) rejects. Filter to our own id + // so honest traffic never trips the ownership guard. + if (origin === this) return + const localId = this.doc.clientID + const changed = [...added, ...updated, ...removed].filter((id) => id === localId) + if (changed.length === 0) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(this.awareness, changed) + ) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + + private sendSyncStep1() { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep1(encoder, this.doc) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + + private sendLocalAwareness() { + if (this.awareness.getLocalState() === null) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.AWARENESS) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(this.awareness, [this.doc.clientID]) + ) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + } + + private setSynced(synced: boolean) { + if (this.synced === synced) return + this.synced = synced + // Readiness needs synced AND seeded; only clear the deadline when both hold (the seed may have + // arrived first, or may still be pending — `handleConfigChange` clears it if seeded arrives later). + if (synced && this.isSeeded()) this.clearReadinessTimer() + this.emit('synced', [synced]) + } + + /** + * Tear down the provider: leave the room, clear our awareness (so peers drop our + * caret immediately rather than after the server's 30s timeout), and detach all + * listeners. The document and awareness objects are the caller's and are left intact. + */ + destroy() { + if (this.disposed) { + super.destroy() + return + } + this.disposed = true + this.clearReadinessTimer() + + awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'provider-destroy') + + this.socket.emit(FILE_DOC_EVENTS.LEAVE, { fileId: this.fileId }) + this.socket.off(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) + this.socket.off(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) + this.socket.off(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + this.socket.off('connect', this.handleConnect) + this.doc.off('update', this.handleDocUpdate) + this.doc.getMap(FILE_DOC_SEED.configMap).unobserve(this.handleConfigChange) + this.awareness.off('update', this.handleAwarenessUpdate) + + super.destroy() + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context.tsx new file mode 100644 index 00000000000..2afc31ae7f1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context.tsx @@ -0,0 +1,43 @@ +'use client' + +import { createContext, type ReactNode, useContext, useState } from 'react' +import type { PresenceAvatarUser } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' + +const EMPTY_OTHERS: PresenceAvatarUser[] = [] +const noop = () => {} + +// Split into two contexts on purpose: the roster (`others`) changes on every join/leave, +// but the setter is stable. The editor (which owns the awareness) only ever *reports* the +// roster, so it subscribes to the setter context — which never changes identity — and never +// re-renders when the roster does; only the header avatar stack subscribes to `others`. +const FileDocOthersContext = createContext(EMPTY_OTHERS) +const FileDocSetOthersContext = createContext<(users: PresenceAvatarUser[]) => void>(noop) + +/** + * Scopes "who's in this file" presence to the open document — the `RoomProvider` + + * `useOthers` pattern (Liveblocks / y-presence) adapted to our component tree. The editor + * owns the Yjs awareness but sits *below* the file-detail header that renders the avatar + * stack, so it publishes the SERVER-AUTHENTICATED roster into this context + * ({@link useReportFileDocOthers}) and the header reads it ({@link useFileDocOthers}). + * Presence is ephemeral and room-scoped, so it lives in this provider, not a global store. + */ +export function FileDocRoomProvider({ children }: { children: ReactNode }) { + const [others, setOthers] = useState(EMPTY_OTHERS) + return ( + + {children} + + ) +} + +/** The roster of collaborators currently in the open file, for an avatar stack. Empty + * outside a {@link FileDocRoomProvider}. */ +export function useFileDocOthers(): PresenceAvatarUser[] { + return useContext(FileDocOthersContext) +} + +/** Publishes the server roster into the room context (editor side). Returns a stable no-op + * outside a {@link FileDocRoomProvider}. */ +export function useReportFileDocOthers(): (users: PresenceAvatarUser[]) => void { + return useContext(FileDocSetOthersContext) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts new file mode 100644 index 00000000000..5afd9470a21 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts @@ -0,0 +1,172 @@ +'use client' + +import { useEffect, useMemo, useRef, useState } from 'react' +import { FILE_DOC_EVENTS, type FileDocPresence } from '@sim/realtime-protocol/file-doc' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { getUserColor } from '@/lib/workspaces/colors' +import { useSocket } from '@/app/workspace/providers/socket-provider' +import { FileDocProvider } from './file-doc-provider' +import { useReportFileDocOthers } from './file-doc-room-context' + +/** The live collaboration binding the editor wires into TipTap's Collaboration + * (the {@link Y.Doc}) and CollaborationCaret (the awareness). */ +export interface FileDocCollaboration { + /** Bound to TipTap's Collaboration extension (created synchronously at mount). */ + doc: Y.Doc + /** Bound to CollaborationCaret (via `{ awareness }`); relayed by the provider. */ + awareness: Awareness + /** + * The realtime provider, or `null` until the socket is available. `doc` and + * `awareness` exist before it connects, so the editor can bind immediately; the + * provider is consumed for its readiness signal (`synced`) and fatal `join-error`. + */ + provider: FileDocProvider | null + /** + * The local caret identity published to awareness: `name`/`color` for CollaborationCaret, + * and `clientId` so the caret activity extension can tag each caret node (see + * caret-presence.ts). The avatar roster does NOT come from here — it's server-authenticated + * (see the PRESENCE subscription below) so a peer can't spoof identity via awareness. + */ + user: { name: string; color: string; clientId: number | undefined } +} + +interface UseFileDocCollaborationParams { + fileId: string + userId: string + userName: string + /** + * Whether to establish collaboration. Decided once at editor mount — only for a + * live, editable, non-streaming workspace document. When `false` the hook + * returns `null` and the editor stays fully local. + */ + enabled: boolean +} + +/** + * Owns the per-file Yjs document, awareness, and {@link FileDocProvider} for + * collaborative editing. The document + awareness are created once (this hook + * lives inside an editor that is keyed by file id, so one instance == one file) + * and are the stable objects TipTap binds to; the provider connects them to the + * realtime relay over the shared socket. Returns `null` while disabled. + */ +export function useFileDocCollaboration({ + fileId, + userId, + userName, + enabled, +}: UseFileDocCollaborationParams): FileDocCollaboration | null { + const { socket } = useSocket() + + // The Y.Doc + Awareness are the editor's authoritative binding — created once + // and stable for the hook's life (see sim-react-performance: lazy-init ref). + // Only allocated when collaboration is enabled, so read-only / streaming / + // round-trip-unsafe views never build a Yjs document they won't use. + const docRef = useRef(null) + const awarenessRef = useRef(null) + // Created ONCE and kept stable for the whole editor lifetime. The editor freezes these into + // its extension set at mount (`useEditor` fixes extensions at creation), so the instances the + // provider binds MUST stay byte-identical to what the editor holds — never destroyed-and- + // recreated mid-life. If a StrictMode dev remount destroyed them, the still-mounted editor + // would keep the dead doc while the provider synced a fresh one, and a joining peer would see a + // blank document. Teardown is therefore deferred to a REAL unmount only (see below). + if (enabled && docRef.current === null) { + docRef.current = new Y.Doc() + awarenessRef.current = new Awareness(docRef.current) + } + + // Destroy the doc + awareness on a REAL unmount only. `Awareness` runs a setInterval to expire + // stale peers, so it MUST be destroyed or that timer leaks for every file ever opened. But a + // StrictMode dev remount fires this cleanup and then re-runs the setup synchronously after — so + // we SCHEDULE the teardown and the remount cancels it, keeping the frozen instances alive. A + // genuine unmount has no remount, so the scheduled teardown runs on the next tick. + const pendingTeardownRef = useRef | null>(null) + useEffect(() => { + if (pendingTeardownRef.current !== null) { + clearTimeout(pendingTeardownRef.current) + pendingTeardownRef.current = null + } + return () => { + pendingTeardownRef.current = setTimeout(() => { + awarenessRef.current?.destroy() + docRef.current?.destroy() + }, 0) + } + }, []) + + const [provider, setProvider] = useState(null) + + useEffect(() => { + if (!enabled || !socket) return + // Non-null: both refs are set during render before any effect runs, and are never destroyed + // (see above), so this always binds the same doc/awareness the editor froze at mount. + const doc = docRef.current as Y.Doc + const awareness = awarenessRef.current as Awareness + const fileProvider = new FileDocProvider(socket, fileId, doc, awareness) + setProvider(fileProvider) + return () => { + fileProvider.destroy() + setProvider(null) + } + }, [enabled, socket, fileId]) + + const reportOthers = useReportFileDocOthers() + const reportOthersRef = useRef(reportOthers) + reportOthersRef.current = reportOthers + + // "Who's in this file" roster (the useOthers side of the pattern). The server broadcasts a + // roster of SERVER-AUTHENTICATED identities (see FILE_DOC_EVENTS.PRESENCE) — trusted, + // unlike the client-set awareness `user` field a peer could spoof. Publish it (minus self) + // to the room context for the file-detail avatar stack; cleared on unmount so a file switch + // never shows the previous file's occupants. + useEffect(() => { + if (!enabled || !socket) return + const handlePresence = (data: FileDocPresence) => { + if (data.fileId !== fileId) return + // Exclude only our OWN socket (this session), NOT every session that shares our userId — + // so a second tab of the same account still counts as present, matching the canvas avatars + // (avatars.tsx filters by socketId). Then dedupe per user for the display stack, so + // multiple tabs of one person collapse to a single avatar. + const byUser = new Map< + string, + { userId: string; userName: string; avatarUrl: string | null } + >() + for (const peer of data.users) { + if (peer.socketId === socket.id || byUser.has(peer.userId)) continue + byUser.set(peer.userId, { + userId: peer.userId, + userName: peer.userName, + avatarUrl: peer.avatarUrl, + }) + } + reportOthersRef.current([...byUser.values()]) + } + socket.on(FILE_DOC_EVENTS.PRESENCE, handlePresence) + return () => { + socket.off(FILE_DOC_EVENTS.PRESENCE, handlePresence) + reportOthersRef.current([]) + } + }, [enabled, socket, fileId]) + + // The client id rides in the awareness `user` payload so the caret `render` (which only + // receives `user`) can tag each caret node for the activity-driven name label (see + // caret-presence.ts). `doc.clientID` is stable for the doc's life, so reading it from the + // ref needs no memo dep. + const user = useMemo( + () => ({ name: userName, color: getUserColor(userId), clientId: docRef.current?.clientID }), + [userName, userId] + ) + + return useMemo( + () => + enabled + ? { + doc: docRef.current as Y.Doc, + awareness: awarenessRef.current as Awareness, + provider, + user, + } + : null, + [enabled, provider, user] + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts index 601d702c07c..0e75964c427 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts @@ -1,8 +1,18 @@ import type { Extensions } from '@tiptap/core' +import Collaboration from '@tiptap/extension-collaboration' +import CollaborationCaret from '@tiptap/extension-collaboration-caret' import Placeholder from '@tiptap/extension-placeholder' +import type { Awareness } from 'y-protocols/awareness' +import type * as Y from 'yjs' +import { withAlpha } from '@/lib/workspaces/colors' import { BlockMover } from './block-mover' import { CodeBlockWithLanguage } from './code-block' import { CodeBlockHighlight } from './code-highlight' +import { + createCaretActivityExtension, + DEFAULT_CARET_COLOR, + renderCaret, +} from './collaboration/caret-presence' import { LinkEmbed } from './embed/link-embed' import { createMarkdownContentExtensions } from './extensions' import { ResizableImage } from './image' @@ -13,10 +23,20 @@ import { MentionChip } from './mention/mention-chip' import { FootnoteDefWithView, RawHtmlBlockWithView } from './raw-markdown-snippet' import { SlashCommand } from './slash-command/slash-command' +/** Live collaboration binding for the editor. When present, the editor's history + * is Yjs-backed and remote carets/selection render via CollaborationCaret. */ +export interface EditorCollaboration { + doc: Y.Doc + awareness: Awareness + user: { name: string; color: string } +} + interface MarkdownEditorExtensionOptions { placeholder: string /** Renders supported media links as live players beneath a standalone link. Off by default. */ embeds?: boolean + /** When set, wires TipTap Collaboration + CollaborationCaret onto the shared document. */ + collaboration?: EditorCollaboration } /** @@ -32,15 +52,41 @@ interface MarkdownEditorExtensionOptions { export function createMarkdownEditorExtensions({ placeholder, embeds = false, + collaboration, }: MarkdownEditorExtensionOptions): Extensions { return [ - ...createMarkdownContentExtensions({ - codeBlock: CodeBlockWithLanguage, - image: ResizableImage, - mention: MentionChip, - rawHtmlBlock: RawHtmlBlockWithView, - footnoteDef: FootnoteDefWithView, - }), + ...createMarkdownContentExtensions( + { + codeBlock: CodeBlockWithLanguage, + image: ResizableImage, + mention: MentionChip, + rawHtmlBlock: RawHtmlBlockWithView, + footnoteDef: FootnoteDefWithView, + }, + { disableHistory: Boolean(collaboration) } + ), + ...(collaboration + ? [ + Collaboration.configure({ document: collaboration.doc }), + // CollaborationCaret reads only `provider.awareness` (created synchronously, + // relayed by the socket provider once connected). `render` tags each caret + // with the peer's client id and shows its name label; the selection tint is + // a translucent fill of the peer's identity color. + CollaborationCaret.configure({ + provider: { awareness: collaboration.awareness }, + user: collaboration.user, + render: renderCaret, + selectionRender: (user) => { + const hex = typeof user.color === 'string' ? user.color : DEFAULT_CARET_COLOR + return { + class: 'collaboration-carets__selection', + style: `background-color: ${withAlpha(hex, 0.2)};`, + } + }, + }), + createCaretActivityExtension(collaboration.awareness), + ] + : []), CodeBlockHighlight, SlashCommand, Mention, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts index 52e4731f27f..8c7d42342f9 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions.ts @@ -11,13 +11,18 @@ import { } from '@tiptap/extension-table' import { Markdown } from '@tiptap/markdown' import StarterKit from '@tiptap/starter-kit' -import { MarkdownCodeBlock } from './code-block' +import { MarkdownCodeBlock } from './code-block-schema' import { Highlight } from './highlight' -import { MarkdownImage } from './image' +import { MarkdownImage } from './image-schema' import { MarkdownLinkInputRule } from './link-input-rule' import { MarkdownMention } from './mention/mention-node' import { SIM_LINK_SCHEME } from './mention/sim-link' -import { FootnoteDef, FootnoteRef, RawHtmlBlock, RawInlineHtml } from './raw-markdown-snippet' +import { + FootnoteDef, + FootnoteRef, + RawHtmlBlock, + RawInlineHtml, +} from './raw-markdown-snippet-schema' /** * The `@`-mention link scheme, registered on the Link mark — without it the schema strips the @@ -117,7 +122,10 @@ export interface ContentNodeViews { * registry. The live editor passes the node-view nodes via {@link createMarkdownEditorExtensions}; the * schema and markdown output are identical either way. */ -export function createMarkdownContentExtensions(nodeViews: ContentNodeViews = {}): Extensions { +export function createMarkdownContentExtensions( + nodeViews: ContentNodeViews = {}, + options: { disableHistory?: boolean } = {} +): Extensions { const codeBlock = (nodeViews.codeBlock ?? MarkdownCodeBlock).configure({ HTMLAttributes: { class: 'code-editor-theme' }, }) @@ -128,6 +136,9 @@ export function createMarkdownContentExtensions(nodeViews: ContentNodeViews = {} codeBlock: false, code: false, paragraph: false, + // Collaboration provides its own (Yjs-backed) undo/redo — disabling the + // built-in history avoids the two fighting over the shared document. + ...(options.disableHistory ? { undoRedo: false as const } : {}), }), BlockSafeParagraph, InlineCode, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts new file mode 100644 index 00000000000..dccc926d6b4 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts @@ -0,0 +1,161 @@ +import type { JSONContent } from '@tiptap/core' +import { Image } from '@tiptap/extension-image' + +/** + * React-free schema half of the image node. Lives apart from {@link ./image} (its React resize node + * view) so the shared editor schema — `createMarkdownContentExtensions` in `./extensions` — can be + * imported by server code (the collab-doc seed converter) without pulling a client component + * (`useEffect`) into a Server Component module. The client editor injects the node-view variant + * ({@link ResizableImage}) via `nodeViews`. + */ + +/** + * A markdown linked image `[![alt](src "t")](href "t2")` — an image wrapped in a link, the canonical + * form of a README badge. `@tiptap/markdown` parses this as a link mark over an image node, but an + * image node can't carry inline marks, so the wrapping link is silently dropped. We instead tokenize + * the whole construct ourselves and hang the link target on the image node's `href` attribute, so it + * round-trips losslessly (and the file stays editable rather than opening read-only). + */ +const LINKED_IMAGE_RE = + /^\[!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)/ + +/** Escape a value for safe interpolation into a double-quoted HTML attribute. */ +function escapeAttr(value: string): string { + return value + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(//g, '>') +} + +/** + * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when + * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to + * preserve its dimensions. Unsized images stay clean `![alt](src)`. An image with an `href` is + * wrapped in a markdown link so a linked badge round-trips as `[![alt](src)](href)`. + * + * A *sized **and** linked* image is the one case markdown can't represent: the linked-image tokenizer + * only recognizes `[![alt](src)](href)`, so emitting `[](href)` would silently drop the link on + * reparse (and the round-trip-safety probe wouldn't catch it). We keep the link and fall back to the + * unsized `[![alt](src)](href)` form — the link matters more than the exact dimensions for a badge. + */ +function imageMarkdown(node: JSONContent): string { + const attrs = node.attrs ?? {} + const src = typeof attrs.src === 'string' ? attrs.src : '' + const alt = typeof attrs.alt === 'string' ? attrs.alt : '' + const title = typeof attrs.title === 'string' ? attrs.title : '' + const href = typeof attrs.href === 'string' ? attrs.href : '' + const hrefTitle = typeof attrs.hrefTitle === 'string' ? attrs.hrefTitle : '' + const width = attrs.width + const height = attrs.height + let image: string + if ((width || height) && !href) { + const parts = [`src="${escapeAttr(src)}"`] + if (alt) parts.push(`alt="${escapeAttr(alt)}"`) + if (title) parts.push(`title="${escapeAttr(title)}"`) + if (width) parts.push(`width="${escapeAttr(String(width))}"`) + if (height) parts.push(`height="${escapeAttr(String(height))}"`) + image = `` + } else { + // Escape so an alt with `]`/`[` or a title with `"` can't break out of the `![…](… "…")` syntax + // and corrupt the round-trip; a src with spaces/parens goes in angle brackets (CommonMark). + const titlePart = title ? ` "${title.replace(/["\\]/g, '\\$&')}"` : '' + const safeSrc = /[\s()]/.test(src) ? `<${src}>` : src + image = `![${alt.replace(/[\\[\]]/g, '\\$&')}](${safeSrc}${titlePart})` + } + if (!href) return image + // Escape `"`/`\` so an href title can't break out of the `[…](href "title")` syntax (mirrors the + // image title escaping above). + const hrefTitlePart = hrefTitle ? ` "${hrefTitle.replace(/["\\]/g, '\\$&')}"` : '' + return `[${image}](${href}${hrefTitlePart})` +} + +interface MarkdownImageToken { + /** Set only by our linked-image tokenizer; absent on the built-in `![](src)` token. */ + src?: string + alt?: string + title?: string | null + /** Built-in image token holds the source URL here; our linked token holds the link target. */ + href?: string + hrefTitle?: string | null + /** Built-in image token holds the alt text here. */ + text?: string +} + +/** Map both the built-in image token and our linked-image token onto the image node's attributes. */ +function parseImageToken(token: MarkdownImageToken): JSONContent { + const isLinked = typeof token.src === 'string' + return { + type: 'image', + attrs: isLinked + ? { + src: token.src, + alt: token.alt ?? '', + title: token.title ?? null, + href: token.href ?? null, + hrefTitle: token.hrefTitle ?? null, + } + : { + src: token.href ?? '', + alt: token.text ?? '', + title: token.title ?? null, + href: null, + hrefTitle: null, + }, + } +} + +const widthAttr = { + default: null, + parseHTML: (element: HTMLElement) => element.getAttribute('width'), + renderHTML: (attributes: Record) => + attributes.width ? { width: String(attributes.width) } : {}, +} + +const heightAttr = { + default: null, + parseHTML: (element: HTMLElement) => element.getAttribute('height'), + renderHTML: (attributes: Record) => + attributes.height ? { height: String(attributes.height) } : {}, +} + +/** Link target of a linked image — markdown-only state, never emitted as an HTML `` attribute. */ +const hrefAttr = { default: null, rendered: false } +const hrefTitleAttr = { default: null, rendered: false } + +/** + * Image node that carries optional `width`/`height` (serialized as an HTML `` tag) and an + * optional `href`/`hrefTitle` (a wrapping markdown link, for badges). Shared by the headless + * round-trip path (no node view) and the live {@link ResizableImage}. + */ +export const MarkdownImage = Image.extend({ + addAttributes() { + return { + ...this.parent?.(), + width: widthAttr, + height: heightAttr, + href: hrefAttr, + hrefTitle: hrefTitleAttr, + } + }, + markdownTokenizer: { + name: 'image', + level: 'inline', + start: (src: string) => src.indexOf('[!['), + tokenize: (src: string): (MarkdownImageToken & { type: string; raw: string }) | undefined => { + const match = LINKED_IMAGE_RE.exec(src) + if (!match) return undefined + return { + type: 'image', + raw: match[0], + alt: match[1] ?? '', + src: match[2], + title: match[3] ?? null, + href: match[4], + hrefTitle: match[5] ?? null, + } + }, + }, + parseMarkdown: parseImageToken, + renderMarkdown: imageMarkdown, +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx index fa198e8658e..fc9ce30fc1b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx @@ -1,167 +1,15 @@ import { useEffect, useRef, useState } from 'react' import { cn } from '@sim/emcn' -import type { JSONContent } from '@tiptap/core' -import { Image } from '@tiptap/extension-image' import { NodeSelection, Plugin } from '@tiptap/pm/state' import type { ReactNodeViewProps } from '@tiptap/react' import { NodeViewWrapper, ReactNodeViewRenderer } from '@tiptap/react' import { useFileContentSource } from '@/hooks/use-file-content-source' +import { MarkdownImage } from './image-schema' import { normalizeLinkHref } from './markdown-fidelity' import { useEditorEditable } from './use-editor-editable' const MIN_WIDTH = 64 -/** - * A markdown linked image `[![alt](src "t")](href "t2")` — an image wrapped in a link, the canonical - * form of a README badge. `@tiptap/markdown` parses this as a link mark over an image node, but an - * image node can't carry inline marks, so the wrapping link is silently dropped. We instead tokenize - * the whole construct ourselves and hang the link target on the image node's `href` attribute, so it - * round-trips losslessly (and the file stays editable rather than opening read-only). - */ -const LINKED_IMAGE_RE = - /^\[!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)/ - -/** Escape a value for safe interpolation into a double-quoted HTML attribute. */ -function escapeAttr(value: string): string { - return value - .replace(/&/g, '&') - .replace(/"/g, '"') - .replace(//g, '>') -} - -/** - * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when - * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to - * preserve its dimensions. Unsized images stay clean `![alt](src)`. An image with an `href` is - * wrapped in a markdown link so a linked badge round-trips as `[![alt](src)](href)`. - * - * A *sized **and** linked* image is the one case markdown can't represent: the linked-image tokenizer - * only recognizes `[![alt](src)](href)`, so emitting `[](href)` would silently drop the link on - * reparse (and the round-trip-safety probe wouldn't catch it). We keep the link and fall back to the - * unsized `[![alt](src)](href)` form — the link matters more than the exact dimensions for a badge. - */ -function imageMarkdown(node: JSONContent): string { - const attrs = node.attrs ?? {} - const src = typeof attrs.src === 'string' ? attrs.src : '' - const alt = typeof attrs.alt === 'string' ? attrs.alt : '' - const title = typeof attrs.title === 'string' ? attrs.title : '' - const href = typeof attrs.href === 'string' ? attrs.href : '' - const hrefTitle = typeof attrs.hrefTitle === 'string' ? attrs.hrefTitle : '' - const width = attrs.width - const height = attrs.height - let image: string - if ((width || height) && !href) { - const parts = [`src="${escapeAttr(src)}"`] - if (alt) parts.push(`alt="${escapeAttr(alt)}"`) - if (title) parts.push(`title="${escapeAttr(title)}"`) - if (width) parts.push(`width="${escapeAttr(String(width))}"`) - if (height) parts.push(`height="${escapeAttr(String(height))}"`) - image = `` - } else { - // Escape so an alt with `]`/`[` or a title with `"` can't break out of the `![…](… "…")` syntax - // and corrupt the round-trip; a src with spaces/parens goes in angle brackets (CommonMark). - const titlePart = title ? ` "${title.replace(/["\\]/g, '\\$&')}"` : '' - const safeSrc = /[\s()]/.test(src) ? `<${src}>` : src - image = `![${alt.replace(/[\\[\]]/g, '\\$&')}](${safeSrc}${titlePart})` - } - if (!href) return image - // Escape `"`/`\` so an href title can't break out of the `[…](href "title")` syntax (mirrors the - // image title escaping above). - const hrefTitlePart = hrefTitle ? ` "${hrefTitle.replace(/["\\]/g, '\\$&')}"` : '' - return `[${image}](${href}${hrefTitlePart})` -} - -interface MarkdownImageToken { - /** Set only by our linked-image tokenizer; absent on the built-in `![](src)` token. */ - src?: string - alt?: string - title?: string | null - /** Built-in image token holds the source URL here; our linked token holds the link target. */ - href?: string - hrefTitle?: string | null - /** Built-in image token holds the alt text here. */ - text?: string -} - -/** Map both the built-in image token and our linked-image token onto the image node's attributes. */ -function parseImageToken(token: MarkdownImageToken): JSONContent { - const isLinked = typeof token.src === 'string' - return { - type: 'image', - attrs: isLinked - ? { - src: token.src, - alt: token.alt ?? '', - title: token.title ?? null, - href: token.href ?? null, - hrefTitle: token.hrefTitle ?? null, - } - : { - src: token.href ?? '', - alt: token.text ?? '', - title: token.title ?? null, - href: null, - hrefTitle: null, - }, - } -} - -const widthAttr = { - default: null, - parseHTML: (element: HTMLElement) => element.getAttribute('width'), - renderHTML: (attributes: Record) => - attributes.width ? { width: String(attributes.width) } : {}, -} - -const heightAttr = { - default: null, - parseHTML: (element: HTMLElement) => element.getAttribute('height'), - renderHTML: (attributes: Record) => - attributes.height ? { height: String(attributes.height) } : {}, -} - -/** Link target of a linked image — markdown-only state, never emitted as an HTML `` attribute. */ -const hrefAttr = { default: null, rendered: false } -const hrefTitleAttr = { default: null, rendered: false } - -/** - * Image node that carries optional `width`/`height` (serialized as an HTML `` tag) and an - * optional `href`/`hrefTitle` (a wrapping markdown link, for badges). Shared by the headless - * round-trip path (no node view) and the live {@link ResizableImage}. - */ -export const MarkdownImage = Image.extend({ - addAttributes() { - return { - ...this.parent?.(), - width: widthAttr, - height: heightAttr, - href: hrefAttr, - hrefTitle: hrefTitleAttr, - } - }, - markdownTokenizer: { - name: 'image', - level: 'inline', - start: (src: string) => src.indexOf('[!['), - tokenize: (src: string): (MarkdownImageToken & { type: string; raw: string }) | undefined => { - const match = LINKED_IMAGE_RE.exec(src) - if (!match) return undefined - return { - type: 'image', - raw: match[0], - alt: match[1] ?? '', - src: match[2], - title: match[3] ?? null, - href: match[4], - hrefTitle: match[5] ?? null, - } - }, - }, - parseMarkdown: parseImageToken, - renderMarkdown: imageMarkdown, -}) - /** * Drag-to-resize image node view (handle at the bottom-right, revealed on selection). Dragging * commits the new pixel width to the `width` attribute, which serializes to ``. diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.test.ts index 586f2bc2ff2..9c2ba8f3b70 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.test.ts @@ -11,6 +11,7 @@ import { GapCursor } from '@tiptap/pm/gapcursor' import { AllSelection, NodeSelection } from '@tiptap/pm/state' import { beforeEach, describe, expect, it, vi } from 'vitest' import { createMarkdownEditorExtensions } from './editor-extensions' +import { postProcessSerializedMarkdown } from './markdown-fidelity' import { MENTION_PLUGIN_KEY } from './mention' import { SLASH_COMMAND_PLUGIN_KEY } from './slash-command/slash-command' @@ -192,13 +193,11 @@ describe('empty wrapped-block Backspace', () => { it.each([ ['bullet middle', '- one\n- two\n- three', 'two', '- one\n- three'], ['bullet first', '- one\n- two\n- three', 'one', '- two\n- three'], - ['bullet last', '- one\n- two', 'two', '- one'], ['ordered middle', '1. one\n2. two\n3. three', 'two', '1. one\n2. three'], ['task middle', '- [ ] one\n- [ ] two\n- [ ] three', 'two', '- [ ] one\n- [ ] three'], ['blockquote middle', '> one\n>\n> two\n>\n> three', 'two', '> one\n>\n> three'], - ['nested item', '- one\n - two\n- three', 'two', '- one\n- three'], ])( - 'removes the emptied %s cleanly — one container, no stray paragraph, round-trips', + 'removes an emptied non-trailing %s cleanly — one container, no stray paragraph, round-trips', (_label, markdown, word, expected) => { const editor = editorWith('') editor.commands.setContent(markdown, { contentType: 'markdown' }) @@ -213,10 +212,10 @@ describe('empty wrapped-block Backspace', () => { } ) - it('leaves a gap cursor — never a NodeSelection — when the removed bullet was followed by an image at doc start', () => { - // Regression: `Selection.near` after the delete silently NodeSelected the following image, so a - // second Backspace while "clearing the bullet" deleted the image (and typing would have replaced - // it). The selection left behind must never make the next keystroke destructive. + it('clears a lone empty bullet before an image to a paragraph and never destroys the image', () => { + // Regression: an earlier delete-and-jump path silently NodeSelected the following image, so a + // second Backspace while "clearing the bullet" deleted it. The lone empty bullet now lifts into an + // empty paragraph in place, the image is untouched, and no repeated Backspace destroys it. const editor = editorWith({ type: 'doc', content: [ @@ -227,11 +226,11 @@ describe('empty wrapped-block Backspace', () => { editor.commands.setTextSelection(3) pressBackspace(editor) - expect(blockShape(editor)).toEqual(['image', 'paragraph']) + expect(blockShape(editor)).toEqual(['paragraph', 'image', 'paragraph']) expect(editor.state.selection).not.toBeInstanceOf(NodeSelection) pressBackspace(editor) - expect(blockShape(editor)).toEqual(['image', 'paragraph']) + expect(blockShape(editor)).toContain('image') editor.destroy() }) @@ -252,11 +251,10 @@ describe('empty wrapped-block Backspace', () => { editor.destroy() }) - it('never NodeSelects a leaf BEFORE the removed bullet either (findFrom textOnly skips atoms)', () => { - // `Selection.findFrom($gap, -1, true)` cannot return a NodeSelection: with textOnly, - // prosemirror-state's findSelectionIn skips atoms entirely (`!text && isSelectable`). With an - // image directly before the emptied bullet and no textblock behind it, the backward search - // returns null and the gap-cursor branch takes over — the image is never silently selected. + it('clears a lone empty bullet after an image to a paragraph without selecting the image', () => { + // With an image directly before the emptied bullet, clearing the bullet must not silently select + // (and so endanger) the image. The bullet lifts into an empty paragraph in place; the image is + // untouched and no repeated Backspace destroys it. const editor = editorWith({ type: 'doc', content: [ @@ -268,11 +266,11 @@ describe('empty wrapped-block Backspace', () => { expect(editor.state.selection.$from.parent.type.name).toBe('paragraph') pressBackspace(editor) - expect(blockShape(editor)).toEqual(['image', 'paragraph']) + expect(blockShape(editor)).toEqual(['image', 'paragraph', 'paragraph']) expect(editor.state.selection).not.toBeInstanceOf(NodeSelection) pressBackspace(editor) - expect(blockShape(editor)).toEqual(['image', 'paragraph']) + expect(blockShape(editor)).toContain('image') editor.destroy() }) @@ -291,7 +289,7 @@ describe('empty wrapped-block Backspace', () => { editor.destroy() }) - it('still prefers the previous textblock caret when one exists (image after the bullet untouched)', () => { + it('clears a lone empty bullet between a paragraph and an image to a paragraph, leaving the image', () => { const editor = editorWith({ type: 'doc', content: [ @@ -303,10 +301,204 @@ describe('empty wrapped-block Backspace', () => { editor.commands.setTextSelection(10) pressBackspace(editor) - expect(blockShape(editor)).toEqual(['paragraph', 'image', 'paragraph']) + expect(blockShape(editor)).toEqual(['paragraph', 'paragraph', 'image', 'paragraph']) expect(editor.state.selection.empty).toBe(true) expect(editor.state.selection).not.toBeInstanceOf(NodeSelection) - expect(editor.state.selection.$from.parent.textContent).toBe('hello') + // The cleared bullet is now an empty paragraph, and 'hello' is untouched above it. + expect(editor.state.doc.firstChild?.textContent).toBe('hello') + editor.destroy() + }) +}) + +describe('list Backspace (clear / outdent)', () => { + beforeEach(() => { + Element.prototype.scrollIntoView = vi.fn() + }) + + /** Puts the caret at the very start of the item text `word`. */ + function caretAtStartOf(editor: Editor, word: string): void { + editor.state.doc.descendants((node, pos) => { + if (node.isText && node.text === word) editor.commands.setTextSelection(pos) + }) + } + + it('lifts a top-level bullet WITH TEXT into a paragraph, keeping the text (round-trips)', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n- two', { contentType: 'markdown' }) + editor.commands.focus() + caretAtStartOf(editor, 'two') + pressBackspace(editor) + + // A bulletList (just 'one') followed by the lifted 'two' paragraph (+ TipTap's trailing filler). + expect(blockShape(editor).slice(0, 2)).toEqual(['bulletList', 'paragraph']) + const { md, reparsed } = markdownRoundTrip(editor) + expect(md.trim()).toBe('- one\n\ntwo') + expect(reparsed).toBe(md) + editor.destroy() + }) + + it('clears an empty TRAILING bullet to a paragraph in place — no delete, caret stays on the line', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n- two', { contentType: 'markdown' }) + editor.commands.focus() + emptyItem(editor, 'two') + pressBackspace(editor) + + // The bullet becomes a paragraph; the list keeps only 'one'. The caret sits in the new empty + // paragraph rather than jumping back into the 'one' bullet. + const list = editor.getJSON().content?.find((node) => node.type === 'bulletList') + expect(list?.content).toHaveLength(1) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.$from.parent.type.name).toBe('paragraph') + expect(editor.state.selection.$from.parent.textContent).toBe('') + expect(editor.getMarkdown().trim()).toBe('- one') + editor.destroy() + }) + + it('clears a lone empty bullet to an empty paragraph (whole doc)', () => { + const editor = editorWith('') + editor.commands.setContent('- one', { contentType: 'markdown' }) + editor.commands.focus() + emptyItem(editor, 'one') + pressBackspace(editor) + + expect(editor.getJSON().content?.some((n) => n.type === 'bulletList')).toBe(false) + expect(editor.state.selection.$from.parent.type.name).toBe('paragraph') + editor.destroy() + }) + + it('outdents a nested bullet WITH TEXT one level instead of merging it (round-trips)', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n - two', { contentType: 'markdown' }) + editor.commands.focus() + caretAtStartOf(editor, 'two') + pressBackspace(editor) + + const { md, reparsed } = markdownRoundTrip(editor) + expect(md.trim()).toBe('- one\n- two') + expect(reparsed).toBe(md) + editor.destroy() + }) + + it('outdents an empty nested bullet one level (round-trips)', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n - two\n- three', { contentType: 'markdown' }) + editor.commands.focus() + emptyItem(editor, 'two') + pressBackspace(editor) + + const { md, reparsed } = markdownRoundTrip(editor) + expect(md.trim()).toBe('- one\n- \n- three') + expect(reparsed).toBe(md) + editor.destroy() + }) + + it('clears a checklist item the same way (task item → paragraph)', () => { + const editor = editorWith('') + editor.commands.setContent('- [ ] one\n- [ ] two', { contentType: 'markdown' }) + editor.commands.focus() + caretAtStartOf(editor, 'two') + pressBackspace(editor) + + expect(blockShape(editor).slice(0, 2)).toEqual(['taskList', 'paragraph']) + expect(editor.getMarkdown().trim()).toBe('- [ ] one\n\ntwo') + editor.destroy() + }) + + it('does not delete a non-trailing item whose block holds only a non-text atom', () => { + // Emptiness is the caret block's content.size, not its text: a bullet holding only an inline atom + // (image/mention — here a hardBreak stand-in) is NOT block-empty, so Backspace clears it to a + // paragraph (content preserved) instead of removeEmptyWrappedBlock deleting the whole row. + const editor = editorWith('') + editor.commands.setContent({ + type: 'doc', + content: [ + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [{ type: 'paragraph', content: [{ type: 'hardBreak' }] }], + }, + { + type: 'listItem', + content: [{ type: 'paragraph', content: [{ type: 'text', text: 'two' }] }], + }, + ], + }, + ], + }) + const atomPos = firstPosOf(editor, 'hardBreak') + editor.commands.setTextSelection(atomPos) + pressBackspace(editor) + + // The atom survives (not deleted) and 'two' is untouched. + expect(firstPosOf(editor, 'hardBreak')).toBeGreaterThanOrEqual(0) + expect(editor.state.doc.textContent).toContain('two') + editor.destroy() + }) + + it('removes only the empty first block of a multi-block item, not the whole item', () => { + // An empty first block whose item has sibling blocks must not lift the whole item out of the list; + // only that empty block is removed, the rest of the item (and the list) stays intact. + const editor = editorWith('') + editor.commands.setContent({ + type: 'doc', + content: [ + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [ + { type: 'paragraph' }, + { type: 'paragraph', content: [{ type: 'text', text: 'more' }] }, + ], + }, + ], + }, + ], + }) + // Caret at the start of the empty first paragraph (position 3: doc>bulletList>listItem>paragraph). + editor.commands.setTextSelection(3) + pressBackspace(editor) + + // Still a list (item was NOT lifted out to a top-level paragraph), and 'more' survives. + expect(blockShape(editor)[0]).toBe('bulletList') + expect(editor.state.doc.textContent).toBe('more') + const list = editor.getJSON().content?.find((n) => n.type === 'bulletList') + expect(list?.content).toHaveLength(1) + editor.destroy() + }) +}) + +describe('empty nested bullet does not corrupt its parent (Enter → Tab)', () => { + beforeEach(() => { + Element.prototype.scrollIntoView = vi.fn() + }) + + it('serializes a stranded empty sub-bullet away instead of turning the parent into a heading', () => { + // Repro: type a bullet, Enter for a new bullet, Tab to indent it into an empty sub-bullet, then + // leave it. The serialized `- one\n - ` would re-parse as `- ## one` (Setext underline). The + // serialize step must strip the empty sub-bullet so the parent stays a bullet and round-trips. + const editor = editorWith('') + editor.commands.setContent('- one', { contentType: 'markdown' }) + editor.commands.focus() + let end = -1 + editor.state.doc.descendants((node, pos) => { + if (node.isText && node.text === 'one') end = pos + 3 + }) + editor.commands.setTextSelection(end) + pressKey(editor, 'Enter') + pressKey(editor, 'Tab') + + const saved = postProcessSerializedMarkdown(editor.getMarkdown()) + expect(saved).toBe('- one\n') + + // Reloading the saved markdown keeps a bullet — never a heading. + editor.commands.setContent(saved, { contentType: 'markdown' }) + expect(blockShape(editor)).not.toContain('heading') + expect(blockShape(editor)).toContain('bulletList') editor.destroy() }) }) @@ -341,6 +533,52 @@ describe('empty list-item Enter', () => { expect(editor.getJSON().content?.some((node) => node.type === 'paragraph')).toBe(true) editor.destroy() }) + + it('outdents an empty NESTED item one level instead of removing it', () => { + const editor = editorWith('') + editor.commands.setContent('- one\n - two', { contentType: 'markdown' }) + editor.commands.focus() + emptyItem(editor, 'two') + pressKey(editor, 'Enter') + + // The emptied nested item outdents to a second top-level bullet rather than being deleted. + const list = editor.getJSON().content?.find((node) => node.type === 'bulletList') + expect(list?.content).toHaveLength(2) + expect(list?.content?.every((item) => item.type === 'listItem')).toBe(true) + editor.destroy() + }) + + it('removes only the empty first block of a multi-block item, matching Backspace (keeps the list)', () => { + // Symmetry with the Backspace multi-block case: an empty first block whose item has sibling blocks + // is removed in place — the continuation and the list stay intact — rather than exiting the list. + const editor = editorWith('') + editor.commands.setContent({ + type: 'doc', + content: [ + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [ + { type: 'paragraph' }, + { type: 'paragraph', content: [{ type: 'text', text: 'more' }] }, + ], + }, + ], + }, + ], + }) + // Caret at the start of the empty first paragraph (doc>bulletList>listItem>paragraph). + editor.commands.setTextSelection(3) + pressKey(editor, 'Enter') + + expect(blockShape(editor)[0]).toBe('bulletList') + expect(editor.state.doc.textContent).toBe('more') + const list = editor.getJSON().content?.find((n) => n.type === 'bulletList') + expect(list?.content).toHaveLength(1) + editor.destroy() + }) }) describe('verbatim block boundary (isolating)', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.ts index bdbac57bfc6..524e35eee21 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/keymap.ts @@ -20,6 +20,50 @@ const WRAPPER_TYPES = new Set(['listItem', 'taskItem', 'blockquote']) /** Item node types a list is built from, used to detect an empty item's position within its list. */ const LIST_ITEM_TYPES = new Set(['listItem', 'taskItem']) +/** The enclosing list/task item at a caret position, with the facts the boundary keys branch on. */ +interface ListItemContext { + /** `'listItem'` or `'taskItem'` — the type name `liftListItem` must be called with. */ + itemType: string + /** The item's list is itself inside another list item, i.e. the item is indented. */ + isNested: boolean + /** + * The caret's own block (the row the boundary key acts on) has no content. Uses `content.size`, so an + * inline image or mention atom counts as content — a bullet holding only an image is NOT block-empty. + */ + blockEmpty: boolean + /** The item has more than one child block (continuation paragraph, nested list, block image, …). */ + hasSiblingBlocks: boolean + /** The item is the last child of its immediate list. */ + isTrailing: boolean + /** The caret sits in the item's first child block (the row a boundary key should act on). */ + isFirstBlock: boolean +} + +/** + * Resolves the nearest enclosing list/task item at `$from` and the facts the Backspace/Enter handlers + * branch on (nesting, emptiness, trailing position, whether the caret is in the item's first block), or + * null when the caret is not inside a list item. Walking up from `$from` finds the item regardless of + * how deeply the caret's block is nested inside it. + */ +function getListItemContext($from: ResolvedPos): ListItemContext | null { + for (let depth = $from.depth; depth >= 1; depth--) { + const item = $from.node(depth) + if (!LIST_ITEM_TYPES.has(item.type.name)) continue + const listDepth = depth - 1 + const isNested = listDepth >= 1 && LIST_ITEM_TYPES.has($from.node(listDepth - 1).type.name) + const list = $from.node(listDepth) + return { + itemType: item.type.name, + isNested, + blockEmpty: $from.parent.content.size === 0, + hasSiblingBlocks: item.childCount > 1, + isTrailing: $from.index(listDepth) === list.childCount - 1, + isFirstBlock: $from.index(depth) === 0, + } + } + return null +} + const RICH_LEAF_SELECTION_FOCUS_KEY = new PluginKey('richLeafSelectionFocus') /** True when the resolved position sits anywhere inside a {@link WRAPPER_TYPES} ancestor. */ @@ -136,21 +180,26 @@ function selectAdjacentSelectedLeaf(editor: Editor, direction: 'up' | 'down'): b * Editor-specific keyboard behavior layered on top of StarterKit's defaults: * * - **Backspace** at the start of a heading reverts it to a paragraph (ProseMirror's default joins or - * no-ops, stranding the heading style; a second Backspace then merges as usual). At the start of an - * *empty block inside a list item, task item, or blockquote* it removes that whole emptied wrapper via - * {@link removeEmptyWrappedBlock} instead of ProseMirror's default lift — lifting an empty item out of - * the middle of a list/quote splits the container in two and strands an empty paragraph (a visible gap - * that also re-parses to a different markdown document), while the default `joinBackward` alternately - * no-ops on nested items (leaving them stuck) or merges an empty continuation paragraph into the - * previous item. At the start of a block whose previous sibling is a divider or image, where - * ProseMirror's `joinBackward` can't cross the leaf and no-ops: an *empty* block is deleted (clearing - * the blank line between/below dividers without touching the divider itself), while a *non-empty* - * block selects the leaf — so a first Backspace highlights what a second deletes, the same - * highlight-before-delete affordance as clicking it and parity with the arrow-key leaf selection. - * - **Enter** on an *empty, non-trailing list/task item* removes the empty item ({@link - * removeEmptyWrappedBlock}) rather than letting the default split the list into two around a stranded - * empty paragraph (which does not round-trip). A *trailing* empty item still falls through to the - * default, which exits the list — the standard "press Enter on a blank bullet to leave the list". + * no-ops, stranding the heading style; a second Backspace then merges as usual). At the start of a + * *list or task item* it outdents or clears in place via {@link getListItemContext}: a nested item outdents one + * level, a top-level item with text lifts out of the list into a paragraph (keeping the text), and a + * top-level *empty trailing* (or sole) item lifts into an empty paragraph in place — so the blank + * bullet made by pressing Enter can be cleared back to normal text on the same line instead of being + * deleted with the caret jumping to the previous block. The one case lift can't take is a top-level + * *empty, non-trailing* item: lifting it strands an empty paragraph between the two list halves, which + * re-parses to a different markdown document (an empty line between list items is a loose list, not a + * break); that item is removed via {@link removeEmptyWrappedBlock} instead, keeping the list whole. An + * empty block inside a *blockquote* is likewise removed via {@link removeEmptyWrappedBlock}. At the + * start of a block whose previous sibling is a divider or image, where ProseMirror's `joinBackward` + * can't cross the leaf and no-ops: an *empty* block is deleted (clearing the blank line between/below + * dividers without touching the divider itself), while a *non-empty* block selects the leaf — so a + * first Backspace highlights what a second deletes, the same highlight-before-delete affordance as + * clicking it and parity with the arrow-key leaf selection. + * - **Enter** on an empty *nested* list/task item outdents it one level, on an empty + * *non-trailing top-level* item removes it ({@link removeEmptyWrappedBlock}) rather than splitting the + * list around a stranded empty paragraph (which does not round-trip), and on an empty *trailing* item + * falls through to the default, which exits the list — the standard "press Enter on a blank bullet to + * leave the list". * - **Mod-A** inside a code block selects only that block's contents; pressing it again (when the * block is already fully selected) falls through to the default whole-document select-all, the * same scoped behavior as a code editor. @@ -183,6 +232,28 @@ export const RichMarkdownKeymap = Extension.create({ if ($from.parent.type.name === 'heading') { return editor.commands.setParagraph() } + const listCtx = getListItemContext($from) + if (listCtx?.isFirstBlock) { + const { itemType, isNested, blockEmpty, hasSiblingBlocks, isTrailing } = listCtx + // Backspace at the start of a bullet outdents or clears it in place rather than + // deleting the row and jumping the caret to the previous block. + // - Nested item → outdent one level (empty or not). + // - Top-level item whose first line has content (text OR an inline image/mention) → lift out of + // the list into a paragraph, keeping that content. + // - Top-level item whose empty first block has *sibling* blocks (a continuation paragraph, a + // block image, a nested list) → remove only that empty first block via {@link + // removeEmptyWrappedBlock}, leaving the rest of the item intact (never lift the whole item). + // - Top-level empty single-block item that is trailing (or the sole item) → lift into an empty + // paragraph in place, so a fresh bullet made with Enter can be cleared to normal text in place. + // A top-level *empty, non-trailing* single-block item is the one case lift can't take: it strands + // an empty paragraph between the two list halves, which re-parses to a different markdown document + // (an empty line between list items is a loose list, not a break). That case removes the row via + // {@link removeEmptyWrappedBlock} instead, which keeps the list whole and round-trips. + if (isNested || !blockEmpty) return editor.commands.liftListItem(itemType) + if (hasSiblingBlocks) return removeEmptyWrappedBlock(editor, $from) + if (isTrailing) return editor.commands.liftListItem(itemType) + return removeEmptyWrappedBlock(editor, $from) + } if ($from.parent.content.size === 0 && isInsideWrapper($from)) { return removeEmptyWrappedBlock(editor, $from) } @@ -207,11 +278,17 @@ export const RichMarkdownKeymap = Extension.create({ if (!selection.empty || selection.$from.parentOffset !== 0) return false const { $from } = selection if ($from.parent.content.size !== 0) return false - const itemDepth = $from.depth - 1 - if (itemDepth < 1 || !LIST_ITEM_TYPES.has($from.node(itemDepth).type.name)) return false - const listDepth = itemDepth - 1 - const isTrailingItem = $from.index(listDepth) === $from.node(listDepth).childCount - 1 - if (isTrailingItem) return false + const listCtx = getListItemContext($from) + if (!listCtx?.isFirstBlock) return false + // Enter on an empty item, mirroring the Backspace cases above: a nested item outdents one level; + // an empty first block that has *sibling* blocks (continuation paragraph, block image, nested + // list) removes only that empty block in place, keeping the rest of the item — never exiting the + // list or splitting it; a trailing single-block item falls through to the default (exits the + // list); and a non-trailing single-block item is removed rather than splitting the list around a + // stranded empty paragraph (which does not round-trip). + if (listCtx.isNested) return editor.commands.liftListItem(listCtx.itemType) + if (listCtx.hasSiblingBlocks) return removeEmptyWrappedBlock(editor, $from) + if (listCtx.isTrailing) return false return removeEmptyWrappedBlock(editor, $from) }, 'Mod-a': ({ editor }) => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.test.ts new file mode 100644 index 00000000000..721b91dc305 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { postProcessSerializedMarkdown } from './markdown-fidelity' + +describe('postProcessSerializedMarkdown — empty list-item stripping', () => { + it('drops a nested empty bullet that would re-parse as a Setext heading', () => { + // `- one\n - ` re-parses as `- ## one` (the ` - ` acts as a Setext underline). Stripping the + // empty bullet on serialize keeps the parent a bullet and makes the round-trip stable. + expect(postProcessSerializedMarkdown('- one\n - \n\n')).toBe('- one\n') + }) + + it('drops a nested empty ordered item', () => { + expect(postProcessSerializedMarkdown('1. one\n 2. \n')).toBe('1. one\n') + }) + + it('preserves a top-level empty bullet (placeholder / imported blank — round-trips faithfully)', () => { + // A top-level empty item is not Setext-hazardous and round-trips as an empty item, so it must be + // kept: it may be a placeholder row the user is about to fill, or an intentionally-blank imported item. + expect(postProcessSerializedMarkdown('- one\n- \n')).toBe('- one\n- \n') + expect(postProcessSerializedMarkdown('- one\n- \n- three\n')).toBe('- one\n- \n- three\n') + expect(postProcessSerializedMarkdown('1. one\n2. \n')).toBe('1. one\n2. \n') + }) + + it('keeps bullets that have content', () => { + expect(postProcessSerializedMarkdown('- a\n- b\n')).toBe('- a\n- b\n') + expect(postProcessSerializedMarkdown('- a\n - b\n')).toBe('- a\n - b\n') + }) + + it('keeps a nested empty parent whose next line is an indented child (no orphaning)', () => { + expect(postProcessSerializedMarkdown('- top\n - \n - child\n')).toBe( + '- top\n - \n - child\n' + ) + }) + + it('keeps a nested empty item that follows a same-indent sibling (real placeholder, no Setext hazard)', () => { + // ` - ` after ` - two` (same indent) is a real empty item the parser keeps — it does NOT underline + // a shallower parent's text, so it must not be stripped. (Only ` - ` directly under `- one` does.) + expect(postProcessSerializedMarkdown('- one\n - two\n - \n - three\n')).toBe( + '- one\n - two\n - \n - three\n' + ) + // The hazard case — empty item directly under the shallower parent — is still stripped. + expect(postProcessSerializedMarkdown('- one\n - \n - three\n')).toBe('- one\n - three\n') + }) + + it('keeps a thematic break and empty checklist items (not Setext-hazardous)', () => { + expect(postProcessSerializedMarkdown('text\n\n---\n\nmore\n')).toBe('text\n\n---\n\nmore\n') + expect(postProcessSerializedMarkdown('- [ ] a\n- [ ] \n')).toBe('- [ ] a\n- [ ] \n') + }) + + it('leaves marker-only lines inside a fenced code block untouched', () => { + const code = '```\n- \n-\n1. \n```\n' + expect(postProcessSerializedMarkdown(code)).toBe(code) + }) + + it('leaves marker-only lines inside a tilde (~~~) fence untouched', () => { + const code = '~~~\n- \n1. \n~~~\n' + expect(postProcessSerializedMarkdown(code)).toBe(code) + }) + + it('does not strip inside an unterminated fence (fence stays open to EOF)', () => { + // A fence with no closing delimiter must keep every interior line, including marker-only ones. + const code = '```\n- \n-\n' + expect(postProcessSerializedMarkdown(code)).toBe(code) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts index f416482d491..94511d5ebad 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts @@ -106,16 +106,78 @@ export function normalizeLinkHref(href: string): string { return `https://${trimmed}` } +/** A line that is a bullet/ordered list marker with no content (`-`, ` - `, `1. `). Task items (`- [ ]`) don't match. */ +const EMPTY_LIST_ITEM_LINE = /^([ \t]*)(?:[-*+]|\d+[.)])[ \t]*$/ +/** A fenced code-block delimiter (``` or ~~~), used to leave code interiors untouched. */ +const FENCE_DELIMITER = /^[ \t]*(`{3,}|~{3,})/ +/** Leading indentation of a line, used to detect whether an empty list item has indented children. */ +const LEADING_INDENT = /^[ \t]*/ + +/** + * Removes only the *nested* empty list-item marker lines that re-parse as a Setext heading underline: + * a nested empty bullet (` - `) sitting DIRECTLY under a shallower parent line silently turns that + * parent's text into an `## heading` and drops the bullet on the next load (a data-corrupting + * round-trip). The strip is therefore scoped by three conditions, all required: + * - *indented* (`indent > 0`): a top-level empty bullet (`- ` / `1. `) round-trips faithfully as an + * empty item, never a heading, so a placeholder/blank imported row is preserved. + * - the immediately-preceding line is *shallower* (the parent whose text the underline would consume): + * an empty item after a *same-indent sibling* (` - two` then ` - `) does NOT corrupt — the parser + * keeps it as a real empty item — so it is preserved. A blank line above also breaks the hazard. + * - no more-indented children on the next non-blank line, so its children are never orphaned. + * + * Operates only on the editor's own serialized output, which uses fenced (never 4-space-indented) code + * blocks and `\n` newlines — so tracking fences is sufficient and a bare `-` inside an indented code + * block or a `-\r` line is not a case that can occur here. + */ +function stripEmptyListItemLines(markdown: string): string { + const lines = markdown.split('\n') + const kept: string[] = [] + let fence: string | null = null + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + const delimiter = line.match(FENCE_DELIMITER)?.[1] + if (fence) { + kept.push(line) + if (delimiter && delimiter[0] === fence[0] && delimiter.length >= fence.length) fence = null + continue + } + if (delimiter) { + fence = delimiter + kept.push(line) + continue + } + const empty = line.match(EMPTY_LIST_ITEM_LINE) + if (empty) { + const indent = empty[1].length + let next = i + 1 + while (next < lines.length && lines[next].trim() === '') next++ + const hasChildren = + next < lines.length && (lines[next].match(LEADING_INDENT)?.[0].length ?? 0) > indent + // The Setext-underline hazard exists only when the empty item follows a SHALLOWER parent line + // (whose text the underline would consume). An empty item after a same/deeper-indent sibling + // (` - two` then ` - `) is a real empty item the parser keeps — a nested placeholder between + // siblings must not be lost. Uses the preceding non-blank line's indent; a lone empty item with + // nothing above it (`prevIndent = -1`) has no parent text to corrupt but stays stripped as before. + let prevIdx = i - 1 + while (prevIdx >= 0 && lines[prevIdx].trim() === '') prevIdx-- + const prevIndent = prevIdx >= 0 ? (lines[prevIdx].match(LEADING_INDENT)?.[0].length ?? 0) : -1 + if (indent > 0 && !hasChildren && prevIndent < indent) continue + } + kept.push(line) + } + return kept.join('\n') +} + /** - * Cleans up serializer output: restores callout markers the serializer backslash-escapes - * (`> \[!NOTE\]` → `> [!NOTE]`) and collapses trailing blank lines to a single newline. The - * table serializer's spurious surrounding blank lines are trimmed at the source (PipeSafeTable), - * so no global leading-newline strip is needed here — avoiding clobbering content that legitimately - * begins with whitespace. + * Cleans up serializer output: drops empty list-item marker lines that would otherwise corrupt on + * round-trip ({@link stripEmptyListItemLines}), restores callout markers the serializer + * backslash-escapes (`> \[!NOTE\]` → `> [!NOTE]`), and collapses trailing blank lines to a single + * newline. The table serializer's spurious surrounding blank lines are trimmed at the source + * (PipeSafeTable), so no global leading-newline strip is needed here — avoiding clobbering content + * that legitimately begins with whitespace. */ export function postProcessSerializedMarkdown(markdown: string): string { - return collapseAutolinkedUrls(markdown.replace(ESCAPED_CALLOUT_REGEX, '$1[!$2]')).replace( - /\n+$/, - '\n' - ) + return collapseAutolinkedUrls( + stripEmptyListItemLines(markdown).replace(ESCAPED_CALLOUT_REGEX, '$1[!$2]') + ).replace(/\n+$/, '\n') } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.test.ts index 48cb5718ab8..d59ef6afe6e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.test.ts @@ -59,6 +59,12 @@ const CASES: Array<[string, string]> = [ '1. First\n - sub bullet\n - another\n 1. deep ordered\n 2. item\n2. Second', ], ['heading-separated sections', '# A\n\nalpha\n\n## B\n\nbeta\n\n## C\n\ngamma'], + // Blank-line spacing: `@tiptap/markdown` reconstructs empty paragraphs from runs of blank lines, so + // the chunker must reinsert them or a saved blank line vanishes on reload. See the dedicated + // "empty paragraphs" suite below for the exact whole-document-parser parity. + ['one empty paragraph between paragraphs', 'first\n\n\n\nsecond'], + ['two empty paragraphs between paragraphs', 'first\n\n\n\n\n\nsecond'], + ['empty paragraphs between headings and text', '# A\n\n\n\nalpha\n\n\n\n## B'], ] describe('parseMarkdownToDoc (chunked)', () => { @@ -87,6 +93,66 @@ describe('parseMarkdownToDoc (chunked)', () => { expect(splitMarkdownBlocks('\n\n \n')).toEqual([]) }) + // The chunker used to drop empty paragraphs (visual blank lines between blocks) that the whole-document + // parser preserves, so a saved blank line silently vanished on the next load. These assert the chunked + // parse reconstructs the SAME empty-paragraph structure the whole-document parser does — at document + // edges and between blocks, for one or many blank lines, and around lists. + describe('empty paragraphs (blank-line spacing) match the whole-document parser', () => { + /** Block-type shape of a doc, `∅` for an empty paragraph, normalized through the editor. */ + function shapeOf(md: string, parse: 'chunked' | 'whole'): string { + editor = new Editor({ extensions: createMarkdownContentExtensions() }) + if (parse === 'whole') editor.commands.setContent(md, { contentType: 'markdown' }) + else editor.commands.setContent(parseMarkdownToDoc(md), { contentType: 'json' }) + const shape = (editor.getJSON().content ?? []) + .map((n) => (n.type === 'paragraph' && !n.content?.length ? '∅' : n.type)) + .join(',') + editor.destroy() + editor = null + return shape + } + + it.each([ + ['one empty between paragraphs', 'a\n\n\n\nb'], + ['two empties between paragraphs', 'a\n\n\n\n\n\nb'], + ['three empties between paragraphs', 'a\n\n\n\n\n\n\n\nb'], + ['even blank-line gap (rounds down)', 'a\n\n\n\n\nb'], + ['leading empties', '\n\n\n\na'], + ['leading + between', '\n\n\na\n\n\n\nb'], + ['empties between a heading and text', '# H\n\n\n\ntext'], + ['empties after a tight list', '- a\n- b\n\n\n\ntext'], + ['empties before a tight list', 'text\n\n\n\n- a\n- b'], + // Line-ending variants: the whole-vs-chunked routing must normalize first, or a `\r`-only body + // skips the empty-paragraph guard and is chunked (dropping the empties this fix restores). + ['CRLF between empties', 'a\r\n\r\n\r\n\r\nb'], + ['CR-only (classic Mac) between empties', 'a\r\r\r\rb'], + ])('chunked matches whole-doc: %s', (_label, md) => { + expect(shapeOf(md, 'chunked')).toBe(shapeOf(md, 'whole')) + }) + }) + + // Regression: a file ending in a blank line (a trailing empty paragraph) must stay EDITABLE. Such an + // empty paragraph can't be serialized stably (postProcess collapses trailing newlines), so the parser + // strips it — keeping the doc round-trip-safe/idempotent instead of flipping the file read-only. + describe('trailing blank lines stay editable (regression)', () => { + it.each([ + ['plain paragraph', 'abc\n\n'], + ['heading + text', '# Title\n\nSome text\n\n'], + ['three trailing newlines', 'hello\n\n\n'], + ['two paragraphs', 'para one\n\npara two\n\n'], + ['interior empties + trailing', 'a\n\n\n\nb\n\n'], + ])('a file ending in a blank line is round-trip-safe: %s', (_label, md) => { + expect(isRoundTripSafe(md)).toBe(true) + }) + + it('strips the trailing empty paragraph but keeps interior ones', () => { + const trailing = parseMarkdownToDoc('abc\n\n').content ?? [] + expect(trailing.at(-1)?.type).toBe('paragraph') + expect(trailing.at(-1)?.content?.length ?? 0).toBeGreaterThan(0) + const interior = parseMarkdownToDoc('a\n\n\n\nb').content ?? [] + expect(interior.some((n) => n.type === 'paragraph' && !n.content?.length)).toBe(true) + }) + }) + it('parses reference-style links whole (non-chunkable) without dropping the definition', () => { const body = 'See [the docs][ref] for details.\n\n[ref]: https://example.com/docs' expect(serializeMarkdownBody(body)).toBe(oneShot(body)) @@ -195,13 +261,18 @@ function buildFuzzDoc(seed: number): string { describe('chunked parse — property test over randomized documents', () => { it('chunked === one-shot for every document, and idempotent for every editable one', () => { const failures: Array<{ seed: number; kind: string }> = [] + // Compare modulo trailing whitespace: `parseMarkdownToDoc` strips trailing empty paragraphs (they + // can't be serialized stably — postProcess collapses trailing newlines — so keeping them would flip + // the file read-only), whereas the raw one-shot parse keeps them. That trailing-only divergence is + // intended and invisible after save; interior/leading fidelity is still compared exactly. + const trimEnd = (md: string) => md.replace(/\n+$/, '') for (let seed = 1; seed <= 400; seed++) { const body = buildFuzzDoc(seed) const chunked = serializeMarkdownBody(body) // Fidelity is the load-bearing invariant — chunked must never diverge from the whole-document // parse, for ANY input; idempotency only needs to hold where the doc is editable (raw HTML is // non-idempotent in the underlying editor regardless of chunking, which is why it opens read-only). - if (chunked !== oneShot(body)) failures.push({ seed, kind: 'fidelity' }) + if (trimEnd(chunked) !== trimEnd(oneShot(body))) failures.push({ seed, kind: 'fidelity' }) else if (isRoundTripSafe(body) && serializeMarkdownBody(chunked) !== chunked) { failures.push({ seed, kind: 'idempotency' }) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts index 5417c0ee047..d8255eef7e6 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse.ts @@ -47,6 +47,20 @@ const FENCE_CLOSE = /^ {0,3}(`{3,}|~{3,})[ \t]*$/ const LIST_MARKER = /^[ ]{0,3}(?:[-*+]|\d+[.)])\s/ const BLOCKQUOTE = /^[ ]{0,3}>/ +/** + * Blank-line spacing that `@tiptap/markdown` reconstructs as *interior* or *leading* empty paragraphs — + * a run of two or more blank lines somewhere, or blank line(s) at the document's leading edge. `[^\S\n]` + * matches horizontal whitespace, so a "blank" line may carry spaces/tabs. This is only ever tested + * against the `\r`-normalized body ({@link parseMarkdownToDoc}), so no CRLF handling is needed here. + * + * A *single* trailing blank line is deliberately not matched — purely to avoid routing an otherwise-plain + * file to the slower whole-document parser. Correctness does not depend on it: {@link parseMarkdownToDoc} + * strips trailing empty paragraphs on *both* parse paths ({@link stripTrailingEmptyParagraphs}), so + * serialize→parse stays idempotent regardless of which parser ran. (A trailing run of two or more blanks + * still matches the interior alternative — harmless, since the strip cleans it either way.) + */ +const EMPTY_PARAGRAPH_SPACING = /\n[^\S\n]*\n[^\S\n]*\n|^[^\S\n]*\n[^\S\n]*\n/ + /** * Split a markdown body into top-level blocks that can each be parsed independently and reassembled * without changing meaning. Blank lines separate candidate groups (fenced code blocks stay atomic), @@ -120,20 +134,57 @@ export function splitMarkdownBlocks(body: string): string[] { * vs ~1270ms at 61KB — and byte-identical, because each block is parsed with the same tokenizers. * Documents whose constructs span blocks ({@link NON_CHUNKABLE}) parse whole, and any failure falls * back to a single whole-document parse, so correctness never depends on the splitter. + * + * Blank-line spacing ({@link EMPTY_PARAGRAPH_SPACING}) also parses whole: the chunker parses each block + * stripped of the blank lines between them, so it drops the empty paragraphs `@tiptap/markdown` builds + * from runs of blank lines — a saved visual blank line would silently vanish on reload. Whether a gap + * yields an empty paragraph is a global, block-type-dependent decision (kept between two paragraphs, + * dropped after a heading), so it can't be reconstructed block-locally; these documents parse whole for + * exact fidelity. Ordinary single-blank-line separation still takes the fast chunked path. */ export function parseMarkdownToDoc(body: string): JSONContent { const manager = markdownManager() - if (NON_CHUNKABLE.test(body)) return manager.parse(body) - try { - const content: JSONContent[] = [] - for (const block of splitMarkdownBlocks(body)) { - // `MarkdownManager.parse` always returns a doc node with a `content` array; spread its blocks. - content.push(...(manager.parse(block).content ?? [])) + // Normalize line endings up front so the routing guards see the same `\n` the chunker and parser + // do — the guards' `\n`-anchored tests would otherwise miss a classic `\r`-only body (its blank + // lines are `\r`), routing it to the chunker that then drops its empty paragraphs. + const normalized = body.replace(/\r\n?/g, '\n') + let doc: JSONContent + if (NON_CHUNKABLE.test(normalized) || EMPTY_PARAGRAPH_SPACING.test(normalized)) { + doc = manager.parse(normalized) + } else { + try { + const content: JSONContent[] = [] + for (const block of splitMarkdownBlocks(normalized)) { + // `MarkdownManager.parse` always returns a doc node with a `content` array; spread its blocks. + content.push(...(manager.parse(block).content ?? [])) + } + doc = { type: 'doc', content } + } catch { + doc = manager.parse(normalized) } - return { type: 'doc', content } - } catch { - return manager.parse(body) } + return stripTrailingEmptyParagraphs(doc) +} + +/** An empty paragraph node — the shape a blank line reconstructs to (no content, or `content: []`). */ +function isEmptyParagraph(node: JSONContent): boolean { + return node.type === 'paragraph' && !node.content?.length +} + +/** + * Drop trailing empty paragraphs from a parsed doc. {@link postProcessSerializedMarkdown} collapses + * trailing blank lines to a single newline, so a trailing empty paragraph can never round-trip — the + * whole-document parser reconstructs one from a file ending in a blank line, but keeping it makes + * serialize→parse non-idempotent, which flips the file read-only via the round-trip-safety probe. + * Leading/interior empty paragraphs are untouched (postProcess never strips those). TipTap re-adds its + * own trailing filler paragraph on `setContent`, so the editor still has a place to type. + */ +function stripTrailingEmptyParagraphs(doc: JSONContent): JSONContent { + const content = doc.content + if (!content || content.length === 0) return doc + let end = content.length + while (end > 0 && isEmptyParagraph(content[end - 1])) end-- + return end === content.length ? doc : { ...doc, content: content.slice(0, end) } } /** @@ -142,8 +193,19 @@ export function parseMarkdownToDoc(body: string): JSONContent { * normalization the live editor applies, keeping the output identical to `editor.getMarkdown()`. */ export function serializeMarkdownBody(body: string): string { + return serializeDocToMarkdown(parseMarkdownToDoc(body)) +} + +/** + * Serialize a ProseMirror document (as TipTap {@link JSONContent}) to the editor's canonical + * markdown. Loaded via `setContent` so it passes through the same schema normalization the live + * editor applies — output identical to `editor.getMarkdown()`. The server-side collab-doc converter + * uses this to project a Yjs doc back to markdown through the exact client engine (parity by + * construction), so it must stay the single serialize path (do not inline `getMarkdown` elsewhere). + */ +export function serializeDocToMarkdown(doc: JSONContent): string { const editor = parserEditor() - editor.commands.setContent(parseMarkdownToDoc(body), { contentType: 'json' }) + editor.commands.setContent(doc, { contentType: 'json' }) return editor.getMarkdown() } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet-schema.ts new file mode 100644 index 00000000000..abe01ce42e2 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet-schema.ts @@ -0,0 +1,481 @@ +/** + * React-free schema half of the raw-HTML/footnote nodes. Lives apart from {@link ./raw-markdown-snippet} + * (their React node views) so the shared editor schema — `createMarkdownContentExtensions` in + * `./extensions` — can be imported by server code (the collab-doc seed converter) without pulling a + * client component (`useEffect`) into a Server Component module. The client editor injects the + * node-view variants ({@link RawHtmlBlockWithView}, {@link FootnoteDefWithView}) via `nodeViews`. + */ +import type { JSONContent, MarkdownToken } from '@tiptap/core' +import { mergeAttributes, Node } from '@tiptap/core' + +/** + * Constructs the schema has no node/mark for: raw HTML blocks (`
`, `
`, …), HTML + * comments, and footnotes. Before this file, all four made the *entire* document open read-only + * (see {@link isRoundTripSafe in ./round-trip-safety}) because the stock pipeline silently drops + * or mangles them. Each node below instead holds the exact source text as its content and + * re-emits it byte-for-byte on serialize — the same "hold raw source, re-render specially" shape + * `MarkdownCodeBlock` uses for Mermaid (see `./code-block.tsx`), just without the diagram render. + * + * Inline tags already covered by a real mark/node — `em`/`i`, `strong`/`b`, `s`/`del`/`strike`, + * `code`, `a`, `br`, `img` — are deliberately excluded from {@link RawInlineHtml} so they keep + * parsing into their proper mark (e.g. `x` → italic) instead of freezing as raw source. + */ +const HANDLED_INLINE_TAGS = new Set([ + 'br', + 'img', + 'em', + 'i', + 'strong', + 'b', + 's', + 'del', + 'strike', + 'code', + 'a', +]) + +const VOID_TAGS = new Set([ + 'area', + 'base', + 'br', + 'col', + 'embed', + 'hr', + 'img', + 'input', + 'link', + 'meta', + 'param', + 'source', + 'track', + 'wbr', +]) + +function verbatimText(node: JSONContent): string { + return (node.content ?? []).map((child) => child.text ?? '').join('') +} + +const RAW_HTML_COMMENT_RE = /^/ + +/** + * One HTML attribute: `name` or `name="value"`/`name='value'`/`name=bare`. The quoted-value + * alternatives are what matter — `[^"]*`/`[^']*` consume a literal `>` inside the quotes as part of + * the value, so an attribute like `data-example="a > b"` is treated as one unit instead of ending + * the tag match at the internal `>`. + */ +const ATTRS_RE_SOURCE = String.raw`(?:\s+[^\s"'=<>\`]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>\`]+))?)*` + +/** Matches one opening HTML tag, attributes included (see {@link ATTRS_RE_SOURCE}). Group 1 is the + * tag name, group 2 is the self-closing `/` if present — shared by inline and block tokenizing. */ +const OPEN_TAG_RE = new RegExp(`^<([a-z][\\w-]*)\\b${ATTRS_RE_SOURCE}\\s*(/)?>`, 'i') + +/** A fenced block's opening/closing marker may sit inside a blockquote (each line prefixed with + * up to 3 spaces then one or more `>` markers, each optionally followed by a space) and/or be + * independently indented up to 3 spaces with no blockquote at all (CommonMark's own fence-indent + * tolerance — matches `FENCE_OPEN`/`FENCE_CLOSE` in `./markdown-parse.ts`) — matched on both the + * open and close fence line so `> \`\`\`` and ` \`\`\`` both mask correctly. */ +const FENCE_PREFIX_SOURCE = '(?:[ ]{0,3}>[ ]?)*[ ]{0,3}' + +/** Same as {@link RAW_HTML_COMMENT_RE} but not anchored to the start of the string — used by + * {@link maskCodeRegions} to find a comment anywhere in the scanned text, not just at position 0. */ +const HTML_COMMENT_ANYWHERE_RE = //g + +/** + * Mask fenced code blocks, inline code spans, and HTML comments with same-length filler (newlines + * kept, everything else replaced with a space) so a tag-like mention *inside one of these* — + * `` `
` ``, a fenced example showing HTML syntax, or a comment documenting the tag + * (``) — is never mistaken for a real balancing tag while scanning. Mirrors + * the fenced/inline patterns `stripCode` in `./round-trip-safety.ts` matches (extended to also + * tolerate an indented and/or blockquoted fence marker via {@link FENCE_PREFIX_SOURCE}, since a raw + * HTML block can itself be indented or quoted), but preserves length/position (masks in place) + * instead of deleting, so match indices still map onto the original, unmodified `src` the caller + * slices from. + */ +function maskCodeRegions(src: string): string { + const fenceRe = new RegExp( + `^${FENCE_PREFIX_SOURCE}([\`~]{3,})[^\\n]*\\n[\\s\\S]*?^${FENCE_PREFIX_SOURCE}\\1[\`~]*[ \\t]*$`, + 'gm' + ) + return src + .replace(fenceRe, (m) => m.replace(/[^\n]/g, ' ')) + .replace(/`+[^`\n]*`+/g, (m) => ' '.repeat(m.length)) + .replace(HTML_COMMENT_ANYWHERE_RE, (m) => m.replace(/[^\n]/g, ' ')) +} + +/** + * Find the end of the close tag that balances the open tag of `tag` ending at `src[0, fromIndex)`, + * tracking nesting depth from `fromIndex` onward so `outer inner` consumes + * both levels instead of stopping at the first (inner) ``. Returns -1 if unterminated. A + * nested self-closing same-name tag (``) is skipped — it neither opens nor closes a level. + * Shared by the inline tokenizer (single line) and the block tokenizer (spans blank lines). + * + * Scans a {@link maskCodeRegions}-masked copy of `src` so a tag name mentioned inside code doesn't + * count as real markup — this narrows, but can't eliminate, the inherent ambiguity of regex-based + * (non-DOM) tag matching: a *bare, unescaped* mention of the same tag name in plain prose (not in + * code) is indistinguishable from a real closing tag here, exactly as it would be to a real HTML + * parser given the same ambiguous input (there is no valid way to "escape" a literal `` inside + * real HTML content other than an entity or code region). Verified this can't lose data even in that + * case — the result still reaches a stable fixpoint on save, just restructured — matching this file's + * "reject on doubt, but never require doubt-free input" gate (`isRoundTripSafe`). + */ +function findBalancedCloseEnd(src: string, tag: string, fromIndex: number): number { + const masked = maskCodeRegions(src) + const tagRe = new RegExp(`<(/?)${tag}\\b${ATTRS_RE_SOURCE}\\s*(/)?>`, 'gi') + tagRe.lastIndex = fromIndex + let depth = 1 + for (let match = tagRe.exec(masked); match; match = tagRe.exec(masked)) { + const isClose = match[1] === '/' + const isSelfClosing = Boolean(match[2]) + if (isSelfClosing) continue + if (isClose) { + depth -= 1 + if (depth === 0) return match.index + match[0].length + } else { + depth += 1 + } + } + return -1 +} + +/** + * Marked's own block tokenizer greedily consumes the blank-line run *after* an HTML block/comment + * or a def line as part of that token's own `raw` (the same behavior `PipeSafeTable` in + * `./extensions.ts` works around for tables) — storing it verbatim would double it up with the + * block joiner's own separator, growing by two newlines on every save. Block-level callers trim it; + * inline callers never carry one (inline tokens can't span a blank line), so trimming is a no-op there. + */ +function verbatimParse(raw: string): JSONContent[] { + const trimmed = raw.replace(/\n+$/, '') + return trimmed ? [{ type: 'text', text: trimmed }] : [] +} + +interface VerbatimNodeOptions { + name: string + /** Whether this node sits among block content (own line) or inline content (mid-paragraph). */ + inline: boolean + badgeLabel: string +} + +/** + * Shared shape for a node that holds a markdown construct's exact source text and re-emits it + * unchanged — parsing and rendering never inspect or transform the text, so there is nothing for + * these constructs to lose. `markdownTokenName`/`parseMarkdown`/`renderMarkdown` are read directly + * off the returned config by `@tiptap/markdown`'s `MarkdownManager` (see + * `node_modules/@tiptap/markdown/src/MarkdownManager.ts`), independent of the node's `name`. + */ +function verbatimNodeConfig({ name, inline, badgeLabel }: VerbatimNodeOptions) { + return { + name, + inline, + group: inline ? 'inline' : 'block', + content: 'text*', + marks: '', + code: true, + defining: !inline, + // Block verbatim nodes hold exact source text; `isolating` stops a boundary Backspace/Delete from + // joining across their edge, which would otherwise merge their raw markdown into an adjacent + // paragraph as HTML-escaped prose and destroy the node (silent data loss on save). + isolating: !inline, + selectable: true, + atom: false, + parseHTML() { + return [ + { + tag: `${inline ? 'span' : 'div'}[data-raw-markdown="${name}"]`, + preserveWhitespace: 'full' as const, + }, + ] + }, + renderHTML({ HTMLAttributes }: { HTMLAttributes: Record }) { + return [ + inline ? 'span' : 'div', + mergeAttributes(HTMLAttributes, { + 'data-raw-markdown': name, + 'data-raw-markdown-label': badgeLabel, + class: inline ? 'raw-markdown-inline' : 'raw-markdown-block', + }), + 0, + ] as const + }, + renderMarkdown(node: JSONContent) { + return verbatimText(node) + }, + } +} + +/** + * Tag names CommonMark/GFM treat as "block-starting" HTML (marked's own type-6 list — see + * `_tag` in `node_modules/marked/src/rules.ts`, verified against the CommonMark spec): a block + * opening with one of these ends at its *matching closing tag*, not at the first blank line. Tags + * NOT in this list (`em`, `a`, `span`, `code`, `kbd`, …) can legitimately start an ordinary + * paragraph (`hi there`), so they're deliberately left to marked's own stricter, single-line + * block-HTML detection below — claiming them here would risk swallowing a paragraph that merely + * starts with inline HTML. + */ +const BLOCK_HTML_TAG_NAMES = new Set([ + 'address', + 'article', + 'aside', + 'base', + 'basefont', + 'blockquote', + 'body', + 'caption', + 'center', + 'col', + 'colgroup', + 'dd', + 'details', + 'dialog', + 'dir', + 'div', + 'dl', + 'dt', + 'fieldset', + 'figcaption', + 'figure', + 'footer', + 'form', + 'frame', + 'frameset', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'head', + 'header', + 'hr', + 'html', + 'iframe', + 'legend', + 'li', + 'link', + 'main', + 'menu', + 'menuitem', + 'meta', + 'nav', + 'noframes', + 'ol', + 'optgroup', + 'option', + 'p', + 'param', + 'search', + 'section', + 'summary', + 'table', + 'tbody', + 'td', + 'tfoot', + 'th', + 'thead', + 'title', + 'tr', + 'track', + 'ul', +]) + +/** + * Marked's built-in block-HTML rule ends a `
`/`
`/… block at the *first blank line* + * (CommonMark's HTML-block-type-6 rule) — correct for normal rendering, but wrong for verbatim + * preservation: any real-world `
` with a paragraph inside would fragment into a raw chip, + * an ordinary (rendered) paragraph, and a second raw chip, stranding genuine content in between. + * This tokenizer instead scans to the tag's *matching* close via {@link findBalancedCloseEnd}, blank + * lines included, for tags in {@link BLOCK_HTML_TAG_NAMES}; anything else returns `undefined` and + * falls through to the existing `markdownTokenName: 'html'` handling below (marked's own block + * tokenizer, unchanged). Comments are matched the same way as the inline case — marked's own comment + * rule already spans blank lines correctly, but routing through one path keeps the two tokenizers + * symmetric and independently testable. CommonMark allows up to 3 leading spaces before a block-HTML + * opening line, so the leading indent is split off, matched against separately, and stitched back + * onto `raw` — everything after that first line (including the tag's own body) can be indented + * however the author wrote it, since the balanced scan doesn't care about column position there. + */ +function tokenizeRawHtmlBlockTag(src: string): MarkdownToken | undefined { + const indent = /^ {0,3}/.exec(src)?.[0] ?? '' + const rest = src.slice(indent.length) + + const comment = RAW_HTML_COMMENT_RE.exec(rest) + if (comment) { + const raw = indent + comment[0] + return { type: 'html', raw, text: raw, block: true } + } + + const open = OPEN_TAG_RE.exec(rest) + if (!open) return undefined + const tag = open[1].toLowerCase() + if (!BLOCK_HTML_TAG_NAMES.has(tag)) return undefined + // A handful of BLOCK_HTML_TAG_NAMES entries (link, meta, base, col, …) are void elements with no + // closing tag at all — treat them as complete right after the open tag (like an explicit `/>`), + // same as `tokenizeRawInlineHtml` already does via VOID_TAGS. Without this, scanning for a + // ``/`` that will never legitimately appear risks grabbing unrelated later content + // (or a stray same-name mention) as if it belonged to this block. + if (open[2] || VOID_TAGS.has(tag)) { + const raw = indent + open[0] + return { type: 'html', raw, text: raw, block: true } + } + + const end = findBalancedCloseEnd(rest, tag, open[0].length) + if (end < 0) return undefined + const raw = indent + rest.slice(0, end) + return { type: 'html', raw, text: raw, block: true } +} + +const SKIP_BLOCK_HTML_TAGS = /^<(img|br)\b[^>]*\/?>\s*$/i + +export const RawHtmlBlock = Node.create({ + ...verbatimNodeConfig({ name: 'rawHtmlBlock', inline: false, badgeLabel: 'Raw HTML' }), + markdownTokenName: 'html', + markdownTokenizer: { + name: 'rawHtmlBlockTag', + level: 'block' as const, + // Always -1 (never claims an early interrupt point): when `start` is omitted, `@tiptap/markdown` + // auto-generates one that calls `this.createLexer()` on every paragraph-continuation check, which + // corrupts the in-progress lexer's shared state (verified directly — every other construct on the + // page silently loses its content once a tokenizer without an explicit `start` is registered). + // The other custom tokenizers below all reference this comment rather than repeating it. + // + // The tokenizer above emits `type: 'html'` explicitly, so its tokens flow into the same + // `markdownTokenName: 'html'` parse registration as marked's own block-HTML tokens below — the + // distinct `name` here only avoids colliding with marked's own built-in `html` extension. + start: () => -1, + tokenize: tokenizeRawHtmlBlockTag, + }, + parseMarkdown(token: MarkdownToken) { + if (!token.block) return [] + const raw = token.raw ?? token.text ?? '' + if (!raw.trim()) return [] + // A lone ``/`
` tag block — leave it to the stock path (Image node / hard break), + // matching the same exclusion `round-trip-safety.ts` used to carve out for these two tags. + if (SKIP_BLOCK_HTML_TAGS.test(raw.trim())) return [] + return { type: 'rawHtmlBlock', content: verbatimParse(raw) } + }, +}) + +const FOOTNOTE_DEF_HEAD_RE = /^ {0,3}\[\^[^\]]+\]:/ +const FOOTNOTE_CONTINUATION_RE = /^ {4,}\S/ + +/** + * Consume a footnote definition's opening line plus any continuation lines GFM allows — indented by + * ≥4 spaces, optionally with blank lines between them (a multi-paragraph definition). Stops at the + * first line that is neither indented nor blank, and never consumes a blank line that isn't followed + * by further continuation (that blank line belongs to whatever block comes next). + */ +function tokenizeFootnoteDef(src: string): MarkdownToken | undefined { + const lines = src.split('\n') + if (!FOOTNOTE_DEF_HEAD_RE.test(lines[0])) return undefined + let lineCount = 1 + while (lineCount < lines.length) { + const line = lines[lineCount] + if (FOOTNOTE_CONTINUATION_RE.test(line)) { + lineCount += 1 + continue + } + if (line === '' && FOOTNOTE_CONTINUATION_RE.test(lines[lineCount + 1] ?? '')) { + lineCount += 2 + continue + } + break + } + const raw = lines.slice(0, lineCount).join('\n') + return { type: 'footnoteDef', raw, text: raw } +} + +/** Footnote definition (`[^id]: the note`, with optional ≥4-space-indented continuation lines) — + * marked has no footnote syntax at all, so without this tokenizer the definition is swallowed as a + * plain paragraph and the reference/definition link is lost. */ +export const FootnoteDef = Node.create({ + ...verbatimNodeConfig({ name: 'footnoteDef', inline: false, badgeLabel: 'Footnote' }), + markdownTokenName: 'footnoteDef', + markdownTokenizer: { + name: 'footnoteDef', + level: 'block' as const, + // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. The cost + // here is narrow and safe: a footnote def sharing a line-run with the preceding paragraph (no + // blank line between them) is picked up on the next block boundary instead of interrupting early. + start: () => -1, + tokenize: tokenizeFootnoteDef, + }, + parseMarkdown(token: MarkdownToken) { + const raw = token.raw ?? token.text ?? '' + if (!raw) return [] + return { type: 'footnoteDef', content: verbatimParse(raw) } + }, +}) + +const FOOTNOTE_REF_RE = /^\[\^[^\]]+\]/ + +/** Footnote reference (`text[^id]`) — verbatim passthrough, same rationale as {@link FootnoteDef}. */ +export const FootnoteRef = Node.create({ + ...verbatimNodeConfig({ name: 'footnoteRef', inline: true, badgeLabel: 'Footnote ref' }), + markdownTokenName: 'footnoteRef', + markdownTokenizer: { + name: 'footnoteRef', + level: 'inline' as const, + // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. + start: () => -1, + tokenize(src: string) { + const match = FOOTNOTE_REF_RE.exec(src) + if (!match) return undefined + return { type: 'footnoteRef', raw: match[0], text: match[0] } + }, + }, + parseMarkdown(token: MarkdownToken) { + const raw = token.raw ?? token.text ?? '' + if (!raw) return [] + return { type: 'footnoteRef', content: verbatimParse(raw) } + }, +}) + +/** + * Attempt to consume an inline HTML comment or a tag (with its matching close tag, or as a single + * void/self-closing element) starting at `src[0]`. Returns `undefined` for a tag this schema + * already has a real mark/node for ({@link HANDLED_INLINE_TAGS}) so it keeps parsing normally, and + * for an unterminated open tag (rare/malformed input — falls back to the stock, lossy behavior + * rather than risk mis-consuming the rest of the document). + */ +function tokenizeRawInlineHtml(src: string): MarkdownToken | undefined { + const comment = RAW_HTML_COMMENT_RE.exec(src) + if (comment) return { type: 'rawInlineHtml', raw: comment[0], text: comment[0] } + + const open = OPEN_TAG_RE.exec(src) + if (!open) return undefined + const tag = open[1].toLowerCase() + if (HANDLED_INLINE_TAGS.has(tag)) return undefined + if (open[2] || VOID_TAGS.has(tag)) { + return { type: 'rawInlineHtml', raw: open[0], text: open[0] } + } + + const end = findBalancedCloseEnd(src, tag, open[0].length) + if (end < 0) return undefined + const raw = src.slice(0, end) + return { type: 'rawInlineHtml', raw, text: raw } +} + +/** Inline raw HTML — ``, ``, ``, ``, `` (no Underline mark is registered), + * and any other tag this schema has no mark/node for, plus an inline-position HTML comment. Marked + * classifies inline HTML as its own `'html'` token type, and `@tiptap/markdown`'s inline parser + * hardcodes handling for that type *before* checking its extension registry (unlike block tokens) — + * so claiming it here needs a custom tokenizer, registered under a different token name + * (`rawInlineHtml`) so it's never confused with the stock `'html'` inline path. marked.js runs + * custom extension tokenizers before its own built-ins at both block and inline level (see + * `blockTokens`/`inlineTokens` in `node_modules/marked/lib/marked.esm.js`), so this reliably wins + * the race against marked's default inline HTML/tag tokenizer. */ +export const RawInlineHtml = Node.create({ + ...verbatimNodeConfig({ name: 'rawInlineHtml', inline: true, badgeLabel: 'Raw HTML' }), + markdownTokenName: 'rawInlineHtml', + markdownTokenizer: { + name: 'rawInlineHtml', + level: 'inline' as const, + // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. + start: () => -1, + tokenize: tokenizeRawInlineHtml, + }, + parseMarkdown(token: MarkdownToken) { + const raw = token.raw ?? token.text ?? '' + if (!raw) return [] + return { type: 'rawInlineHtml', content: verbatimParse(raw) } + }, +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet.tsx index 1aad444de29..ea4058895e7 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/raw-markdown-snippet.tsx @@ -1,479 +1,6 @@ -import type { JSONContent, MarkdownToken } from '@tiptap/core' -import { mergeAttributes, Node } from '@tiptap/core' import type { ReactNodeViewProps } from '@tiptap/react' import { NodeViewContent, NodeViewWrapper, ReactNodeViewRenderer } from '@tiptap/react' - -/** - * Constructs the schema has no node/mark for: raw HTML blocks (`
`, `
`, …), HTML - * comments, and footnotes. Before this file, all four made the *entire* document open read-only - * (see {@link isRoundTripSafe in ./round-trip-safety}) because the stock pipeline silently drops - * or mangles them. Each node below instead holds the exact source text as its content and - * re-emits it byte-for-byte on serialize — the same "hold raw source, re-render specially" shape - * `MarkdownCodeBlock` uses for Mermaid (see `./code-block.tsx`), just without the diagram render. - * - * Inline tags already covered by a real mark/node — `em`/`i`, `strong`/`b`, `s`/`del`/`strike`, - * `code`, `a`, `br`, `img` — are deliberately excluded from {@link RawInlineHtml} so they keep - * parsing into their proper mark (e.g. `x` → italic) instead of freezing as raw source. - */ -const HANDLED_INLINE_TAGS = new Set([ - 'br', - 'img', - 'em', - 'i', - 'strong', - 'b', - 's', - 'del', - 'strike', - 'code', - 'a', -]) - -const VOID_TAGS = new Set([ - 'area', - 'base', - 'br', - 'col', - 'embed', - 'hr', - 'img', - 'input', - 'link', - 'meta', - 'param', - 'source', - 'track', - 'wbr', -]) - -function verbatimText(node: JSONContent): string { - return (node.content ?? []).map((child) => child.text ?? '').join('') -} - -const RAW_HTML_COMMENT_RE = /^/ - -/** - * One HTML attribute: `name` or `name="value"`/`name='value'`/`name=bare`. The quoted-value - * alternatives are what matter — `[^"]*`/`[^']*` consume a literal `>` inside the quotes as part of - * the value, so an attribute like `data-example="a > b"` is treated as one unit instead of ending - * the tag match at the internal `>`. - */ -const ATTRS_RE_SOURCE = String.raw`(?:\s+[^\s"'=<>\`]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>\`]+))?)*` - -/** Matches one opening HTML tag, attributes included (see {@link ATTRS_RE_SOURCE}). Group 1 is the - * tag name, group 2 is the self-closing `/` if present — shared by inline and block tokenizing. */ -const OPEN_TAG_RE = new RegExp(`^<([a-z][\\w-]*)\\b${ATTRS_RE_SOURCE}\\s*(/)?>`, 'i') - -/** A fenced block's opening/closing marker may sit inside a blockquote (each line prefixed with - * up to 3 spaces then one or more `>` markers, each optionally followed by a space) and/or be - * independently indented up to 3 spaces with no blockquote at all (CommonMark's own fence-indent - * tolerance — matches `FENCE_OPEN`/`FENCE_CLOSE` in `./markdown-parse.ts`) — matched on both the - * open and close fence line so `> \`\`\`` and ` \`\`\`` both mask correctly. */ -const FENCE_PREFIX_SOURCE = '(?:[ ]{0,3}>[ ]?)*[ ]{0,3}' - -/** Same as {@link RAW_HTML_COMMENT_RE} but not anchored to the start of the string — used by - * {@link maskCodeRegions} to find a comment anywhere in the scanned text, not just at position 0. */ -const HTML_COMMENT_ANYWHERE_RE = //g - -/** - * Mask fenced code blocks, inline code spans, and HTML comments with same-length filler (newlines - * kept, everything else replaced with a space) so a tag-like mention *inside one of these* — - * `` `
` ``, a fenced example showing HTML syntax, or a comment documenting the tag - * (``) — is never mistaken for a real balancing tag while scanning. Mirrors - * the fenced/inline patterns `stripCode` in `./round-trip-safety.ts` matches (extended to also - * tolerate an indented and/or blockquoted fence marker via {@link FENCE_PREFIX_SOURCE}, since a raw - * HTML block can itself be indented or quoted), but preserves length/position (masks in place) - * instead of deleting, so match indices still map onto the original, unmodified `src` the caller - * slices from. - */ -function maskCodeRegions(src: string): string { - const fenceRe = new RegExp( - `^${FENCE_PREFIX_SOURCE}([\`~]{3,})[^\\n]*\\n[\\s\\S]*?^${FENCE_PREFIX_SOURCE}\\1[\`~]*[ \\t]*$`, - 'gm' - ) - return src - .replace(fenceRe, (m) => m.replace(/[^\n]/g, ' ')) - .replace(/`+[^`\n]*`+/g, (m) => ' '.repeat(m.length)) - .replace(HTML_COMMENT_ANYWHERE_RE, (m) => m.replace(/[^\n]/g, ' ')) -} - -/** - * Find the end of the close tag that balances the open tag of `tag` ending at `src[0, fromIndex)`, - * tracking nesting depth from `fromIndex` onward so `outer inner` consumes - * both levels instead of stopping at the first (inner) ``. Returns -1 if unterminated. A - * nested self-closing same-name tag (``) is skipped — it neither opens nor closes a level. - * Shared by the inline tokenizer (single line) and the block tokenizer (spans blank lines). - * - * Scans a {@link maskCodeRegions}-masked copy of `src` so a tag name mentioned inside code doesn't - * count as real markup — this narrows, but can't eliminate, the inherent ambiguity of regex-based - * (non-DOM) tag matching: a *bare, unescaped* mention of the same tag name in plain prose (not in - * code) is indistinguishable from a real closing tag here, exactly as it would be to a real HTML - * parser given the same ambiguous input (there is no valid way to "escape" a literal `` inside - * real HTML content other than an entity or code region). Verified this can't lose data even in that - * case — the result still reaches a stable fixpoint on save, just restructured — matching this file's - * "reject on doubt, but never require doubt-free input" gate (`isRoundTripSafe`). - */ -function findBalancedCloseEnd(src: string, tag: string, fromIndex: number): number { - const masked = maskCodeRegions(src) - const tagRe = new RegExp(`<(/?)${tag}\\b${ATTRS_RE_SOURCE}\\s*(/)?>`, 'gi') - tagRe.lastIndex = fromIndex - let depth = 1 - for (let match = tagRe.exec(masked); match; match = tagRe.exec(masked)) { - const isClose = match[1] === '/' - const isSelfClosing = Boolean(match[2]) - if (isSelfClosing) continue - if (isClose) { - depth -= 1 - if (depth === 0) return match.index + match[0].length - } else { - depth += 1 - } - } - return -1 -} - -/** - * Marked's own block tokenizer greedily consumes the blank-line run *after* an HTML block/comment - * or a def line as part of that token's own `raw` (the same behavior `PipeSafeTable` in - * `./extensions.ts` works around for tables) — storing it verbatim would double it up with the - * block joiner's own separator, growing by two newlines on every save. Block-level callers trim it; - * inline callers never carry one (inline tokens can't span a blank line), so trimming is a no-op there. - */ -function verbatimParse(raw: string): JSONContent[] { - const trimmed = raw.replace(/\n+$/, '') - return trimmed ? [{ type: 'text', text: trimmed }] : [] -} - -interface VerbatimNodeOptions { - name: string - /** Whether this node sits among block content (own line) or inline content (mid-paragraph). */ - inline: boolean - badgeLabel: string -} - -/** - * Shared shape for a node that holds a markdown construct's exact source text and re-emits it - * unchanged — parsing and rendering never inspect or transform the text, so there is nothing for - * these constructs to lose. `markdownTokenName`/`parseMarkdown`/`renderMarkdown` are read directly - * off the returned config by `@tiptap/markdown`'s `MarkdownManager` (see - * `node_modules/@tiptap/markdown/src/MarkdownManager.ts`), independent of the node's `name`. - */ -function verbatimNodeConfig({ name, inline, badgeLabel }: VerbatimNodeOptions) { - return { - name, - inline, - group: inline ? 'inline' : 'block', - content: 'text*', - marks: '', - code: true, - defining: !inline, - // Block verbatim nodes hold exact source text; `isolating` stops a boundary Backspace/Delete from - // joining across their edge, which would otherwise merge their raw markdown into an adjacent - // paragraph as HTML-escaped prose and destroy the node (silent data loss on save). - isolating: !inline, - selectable: true, - atom: false, - parseHTML() { - return [ - { - tag: `${inline ? 'span' : 'div'}[data-raw-markdown="${name}"]`, - preserveWhitespace: 'full' as const, - }, - ] - }, - renderHTML({ HTMLAttributes }: { HTMLAttributes: Record }) { - return [ - inline ? 'span' : 'div', - mergeAttributes(HTMLAttributes, { - 'data-raw-markdown': name, - 'data-raw-markdown-label': badgeLabel, - class: inline ? 'raw-markdown-inline' : 'raw-markdown-block', - }), - 0, - ] as const - }, - renderMarkdown(node: JSONContent) { - return verbatimText(node) - }, - } -} - -/** - * Tag names CommonMark/GFM treat as "block-starting" HTML (marked's own type-6 list — see - * `_tag` in `node_modules/marked/src/rules.ts`, verified against the CommonMark spec): a block - * opening with one of these ends at its *matching closing tag*, not at the first blank line. Tags - * NOT in this list (`em`, `a`, `span`, `code`, `kbd`, …) can legitimately start an ordinary - * paragraph (`hi there`), so they're deliberately left to marked's own stricter, single-line - * block-HTML detection below — claiming them here would risk swallowing a paragraph that merely - * starts with inline HTML. - */ -const BLOCK_HTML_TAG_NAMES = new Set([ - 'address', - 'article', - 'aside', - 'base', - 'basefont', - 'blockquote', - 'body', - 'caption', - 'center', - 'col', - 'colgroup', - 'dd', - 'details', - 'dialog', - 'dir', - 'div', - 'dl', - 'dt', - 'fieldset', - 'figcaption', - 'figure', - 'footer', - 'form', - 'frame', - 'frameset', - 'h1', - 'h2', - 'h3', - 'h4', - 'h5', - 'h6', - 'head', - 'header', - 'hr', - 'html', - 'iframe', - 'legend', - 'li', - 'link', - 'main', - 'menu', - 'menuitem', - 'meta', - 'nav', - 'noframes', - 'ol', - 'optgroup', - 'option', - 'p', - 'param', - 'search', - 'section', - 'summary', - 'table', - 'tbody', - 'td', - 'tfoot', - 'th', - 'thead', - 'title', - 'tr', - 'track', - 'ul', -]) - -/** - * Marked's built-in block-HTML rule ends a `
`/`
`/… block at the *first blank line* - * (CommonMark's HTML-block-type-6 rule) — correct for normal rendering, but wrong for verbatim - * preservation: any real-world `
` with a paragraph inside would fragment into a raw chip, - * an ordinary (rendered) paragraph, and a second raw chip, stranding genuine content in between. - * This tokenizer instead scans to the tag's *matching* close via {@link findBalancedCloseEnd}, blank - * lines included, for tags in {@link BLOCK_HTML_TAG_NAMES}; anything else returns `undefined` and - * falls through to the existing `markdownTokenName: 'html'` handling below (marked's own block - * tokenizer, unchanged). Comments are matched the same way as the inline case — marked's own comment - * rule already spans blank lines correctly, but routing through one path keeps the two tokenizers - * symmetric and independently testable. CommonMark allows up to 3 leading spaces before a block-HTML - * opening line, so the leading indent is split off, matched against separately, and stitched back - * onto `raw` — everything after that first line (including the tag's own body) can be indented - * however the author wrote it, since the balanced scan doesn't care about column position there. - */ -function tokenizeRawHtmlBlockTag(src: string): MarkdownToken | undefined { - const indent = /^ {0,3}/.exec(src)?.[0] ?? '' - const rest = src.slice(indent.length) - - const comment = RAW_HTML_COMMENT_RE.exec(rest) - if (comment) { - const raw = indent + comment[0] - return { type: 'html', raw, text: raw, block: true } - } - - const open = OPEN_TAG_RE.exec(rest) - if (!open) return undefined - const tag = open[1].toLowerCase() - if (!BLOCK_HTML_TAG_NAMES.has(tag)) return undefined - // A handful of BLOCK_HTML_TAG_NAMES entries (link, meta, base, col, …) are void elements with no - // closing tag at all — treat them as complete right after the open tag (like an explicit `/>`), - // same as `tokenizeRawInlineHtml` already does via VOID_TAGS. Without this, scanning for a - // ``/`` that will never legitimately appear risks grabbing unrelated later content - // (or a stray same-name mention) as if it belonged to this block. - if (open[2] || VOID_TAGS.has(tag)) { - const raw = indent + open[0] - return { type: 'html', raw, text: raw, block: true } - } - - const end = findBalancedCloseEnd(rest, tag, open[0].length) - if (end < 0) return undefined - const raw = indent + rest.slice(0, end) - return { type: 'html', raw, text: raw, block: true } -} - -const SKIP_BLOCK_HTML_TAGS = /^<(img|br)\b[^>]*\/?>\s*$/i - -export const RawHtmlBlock = Node.create({ - ...verbatimNodeConfig({ name: 'rawHtmlBlock', inline: false, badgeLabel: 'Raw HTML' }), - markdownTokenName: 'html', - markdownTokenizer: { - name: 'rawHtmlBlockTag', - level: 'block' as const, - // Always -1 (never claims an early interrupt point): when `start` is omitted, `@tiptap/markdown` - // auto-generates one that calls `this.createLexer()` on every paragraph-continuation check, which - // corrupts the in-progress lexer's shared state (verified directly — every other construct on the - // page silently loses its content once a tokenizer without an explicit `start` is registered). - // The other custom tokenizers below all reference this comment rather than repeating it. - // - // The tokenizer above emits `type: 'html'` explicitly, so its tokens flow into the same - // `markdownTokenName: 'html'` parse registration as marked's own block-HTML tokens below — the - // distinct `name` here only avoids colliding with marked's own built-in `html` extension. - start: () => -1, - tokenize: tokenizeRawHtmlBlockTag, - }, - parseMarkdown(token: MarkdownToken) { - if (!token.block) return [] - const raw = token.raw ?? token.text ?? '' - if (!raw.trim()) return [] - // A lone ``/`
` tag block — leave it to the stock path (Image node / hard break), - // matching the same exclusion `round-trip-safety.ts` used to carve out for these two tags. - if (SKIP_BLOCK_HTML_TAGS.test(raw.trim())) return [] - return { type: 'rawHtmlBlock', content: verbatimParse(raw) } - }, -}) - -const FOOTNOTE_DEF_HEAD_RE = /^ {0,3}\[\^[^\]]+\]:/ -const FOOTNOTE_CONTINUATION_RE = /^ {4,}\S/ - -/** - * Consume a footnote definition's opening line plus any continuation lines GFM allows — indented by - * ≥4 spaces, optionally with blank lines between them (a multi-paragraph definition). Stops at the - * first line that is neither indented nor blank, and never consumes a blank line that isn't followed - * by further continuation (that blank line belongs to whatever block comes next). - */ -function tokenizeFootnoteDef(src: string): MarkdownToken | undefined { - const lines = src.split('\n') - if (!FOOTNOTE_DEF_HEAD_RE.test(lines[0])) return undefined - let lineCount = 1 - while (lineCount < lines.length) { - const line = lines[lineCount] - if (FOOTNOTE_CONTINUATION_RE.test(line)) { - lineCount += 1 - continue - } - if (line === '' && FOOTNOTE_CONTINUATION_RE.test(lines[lineCount + 1] ?? '')) { - lineCount += 2 - continue - } - break - } - const raw = lines.slice(0, lineCount).join('\n') - return { type: 'footnoteDef', raw, text: raw } -} - -/** Footnote definition (`[^id]: the note`, with optional ≥4-space-indented continuation lines) — - * marked has no footnote syntax at all, so without this tokenizer the definition is swallowed as a - * plain paragraph and the reference/definition link is lost. */ -export const FootnoteDef = Node.create({ - ...verbatimNodeConfig({ name: 'footnoteDef', inline: false, badgeLabel: 'Footnote' }), - markdownTokenName: 'footnoteDef', - markdownTokenizer: { - name: 'footnoteDef', - level: 'block' as const, - // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. The cost - // here is narrow and safe: a footnote def sharing a line-run with the preceding paragraph (no - // blank line between them) is picked up on the next block boundary instead of interrupting early. - start: () => -1, - tokenize: tokenizeFootnoteDef, - }, - parseMarkdown(token: MarkdownToken) { - const raw = token.raw ?? token.text ?? '' - if (!raw) return [] - return { type: 'footnoteDef', content: verbatimParse(raw) } - }, -}) - -const FOOTNOTE_REF_RE = /^\[\^[^\]]+\]/ - -/** Footnote reference (`text[^id]`) — verbatim passthrough, same rationale as {@link FootnoteDef}. */ -export const FootnoteRef = Node.create({ - ...verbatimNodeConfig({ name: 'footnoteRef', inline: true, badgeLabel: 'Footnote ref' }), - markdownTokenName: 'footnoteRef', - markdownTokenizer: { - name: 'footnoteRef', - level: 'inline' as const, - // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. - start: () => -1, - tokenize(src: string) { - const match = FOOTNOTE_REF_RE.exec(src) - if (!match) return undefined - return { type: 'footnoteRef', raw: match[0], text: match[0] } - }, - }, - parseMarkdown(token: MarkdownToken) { - const raw = token.raw ?? token.text ?? '' - if (!raw) return [] - return { type: 'footnoteRef', content: verbatimParse(raw) } - }, -}) - -/** - * Attempt to consume an inline HTML comment or a tag (with its matching close tag, or as a single - * void/self-closing element) starting at `src[0]`. Returns `undefined` for a tag this schema - * already has a real mark/node for ({@link HANDLED_INLINE_TAGS}) so it keeps parsing normally, and - * for an unterminated open tag (rare/malformed input — falls back to the stock, lossy behavior - * rather than risk mis-consuming the rest of the document). - */ -function tokenizeRawInlineHtml(src: string): MarkdownToken | undefined { - const comment = RAW_HTML_COMMENT_RE.exec(src) - if (comment) return { type: 'rawInlineHtml', raw: comment[0], text: comment[0] } - - const open = OPEN_TAG_RE.exec(src) - if (!open) return undefined - const tag = open[1].toLowerCase() - if (HANDLED_INLINE_TAGS.has(tag)) return undefined - if (open[2] || VOID_TAGS.has(tag)) { - return { type: 'rawInlineHtml', raw: open[0], text: open[0] } - } - - const end = findBalancedCloseEnd(src, tag, open[0].length) - if (end < 0) return undefined - const raw = src.slice(0, end) - return { type: 'rawInlineHtml', raw, text: raw } -} - -/** Inline raw HTML — ``, ``, ``, ``, `` (no Underline mark is registered), - * and any other tag this schema has no mark/node for, plus an inline-position HTML comment. Marked - * classifies inline HTML as its own `'html'` token type, and `@tiptap/markdown`'s inline parser - * hardcodes handling for that type *before* checking its extension registry (unlike block tokens) — - * so claiming it here needs a custom tokenizer, registered under a different token name - * (`rawInlineHtml`) so it's never confused with the stock `'html'` inline path. marked.js runs - * custom extension tokenizers before its own built-ins at both block and inline level (see - * `blockTokens`/`inlineTokens` in `node_modules/marked/lib/marked.esm.js`), so this reliably wins - * the race against marked's default inline HTML/tag tokenizer. */ -export const RawInlineHtml = Node.create({ - ...verbatimNodeConfig({ name: 'rawInlineHtml', inline: true, badgeLabel: 'Raw HTML' }), - markdownTokenName: 'rawInlineHtml', - markdownTokenizer: { - name: 'rawInlineHtml', - level: 'inline' as const, - // See the comment on `RawHtmlBlock`'s `start` — omitting it corrupts the shared lexer. - start: () => -1, - tokenize: tokenizeRawInlineHtml, - }, - parseMarkdown(token: MarkdownToken) { - const raw = token.raw ?? token.text ?? '' - if (!raw) return [] - return { type: 'rawInlineHtml', content: verbatimParse(raw) } - }, -}) +import { FootnoteDef, RawHtmlBlock } from './raw-markdown-snippet-schema' const BLOCK_CONTROL_CLASS = 'pointer-events-none absolute top-1.5 right-2 select-none rounded-md bg-[var(--surface-4)] px-1.5 py-0.5 text-[10px] text-[var(--text-muted)] uppercase tracking-wide opacity-0 transition-opacity group-focus-within:opacity-100 group-hover:opacity-100' diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css index cfb5e9a2ec3..ec33493934b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css @@ -355,12 +355,18 @@ border-radius: 4px; } +/* `overflow: visible` (not `hidden`) so a collaborator's caret name label, which + * pops above the caret, can escape the table box instead of being clipped by it. + * With `table-layout: fixed; width: 100%` columns can't exceed the table, so there + * is nothing to clip here anyway; the only absolutely-positioned descendant is the + * decorative column-resize handle, whose 2px right sliver is clipped by the editor's + * own scroll container. */ .rich-markdown-prose table { width: 100%; border-collapse: collapse; table-layout: fixed; margin: 1rem 0; - overflow: hidden; + overflow: visible; } .rich-markdown-prose th, @@ -454,3 +460,116 @@ .rich-markdown-field-prose p.is-editor-empty:first-child::before { color: var(--text-muted); } + +/* + * Collaborative carets (TipTap CollaborationCaret). The caret bar and the name + * label's background are colored inline from each collaborator's identity color + * (the same `getUserColor` mechanism the canvas cursors use); the selection is a + * translucent tint of that color (set via selectionRender). The name label shows + * while the peer is active (JS toggles `--active` on each awareness change) or on + * hover, then fades after inactivity — matching Google Docs. `z-index` lifts the + * caret (and its label) above table cell backgrounds so a caret inside a table + * cell is not hidden behind adjacent cells. + */ +.rich-markdown-prose .collaboration-carets__caret { + position: relative; + margin-left: -1px; + margin-right: -1px; + border-left-width: 1px; + border-left-style: solid; + border-left-color: var(--caret-color); + border-right-width: 1px; + border-right-style: solid; + border-right-color: var(--caret-color); + word-break: normal; + z-index: 20; +} + +/* Dormant affordance: a small cap at the top of the caret in the collaborator's color, + * shaped like a collapsed presence name tag (same `rounded-xs` + notch corner as the + * tables/canvas tags) so the whole presence system reads as one language. It signals + * "someone's here — hover for who", and fades out when the full name tag takes over + * (peer active or on hover). */ +.rich-markdown-prose .collaboration-carets__caret::before { + content: ""; + position: absolute; + /* Seat the cap's square bottom-left corner on the pole's top-left, flush like a flag on its + * pole. `left: -1px` backs out the caret's 1px left border (the abs-positioning origin is the + * padding box, inside that border) so the cap's left edge lines up with the pole's left edge; + * `top` overlaps the pole's top a hair so the notch reads as continuous, no gap. */ + top: -2px; + left: -1px; + width: 8px; + height: 5px; + border-radius: 2px 2px 2px 0; + background-color: var(--caret-color); + transition: opacity 0.2s ease; +} + +.rich-markdown-prose .collaboration-carets__caret--active::before, +.rich-markdown-prose .collaboration-carets__caret:hover::before { + opacity: 0; +} + +/* Transparent hover hit-slop. The visible caret bar is only ~2px wide and the dormant cap is + * 8×5px, so "hover for who" (revealing the name label on a dormant caret) would be near + * impossible to trigger. This invisible strip widens the hover target across the caret's full + * height and over the cap — nothing visible changes, only the pointer area. `pointer-events: + * auto` is required so it catches the hover; kept narrow so it barely intrudes on selecting + * text next to a remote caret. */ +.rich-markdown-prose .collaboration-carets__caret::after { + content: ""; + position: absolute; + top: -4px; + bottom: 0; + left: -4px; + right: -4px; + pointer-events: auto; +} + +/* Matches the canvas cursor name tag (cursors.tsx): identity-color background with + * `--surface-1` text at `text-xs`/`font-medium`, so both presence surfaces read as + * one system. `--surface-1` is the base surface token (readable on every assigned + * identity color in both themes), not a hardcoded value. Hidden by default; the + * show/fade is driven by the `--active` class (see caret-presence.ts) and hover. */ +.rich-markdown-prose .collaboration-carets__label { + position: absolute; + top: -1.4em; + left: -1px; + max-width: 10rem; + overflow: hidden; + padding: 0.1rem 0.35rem; + border-radius: 2px 2px 2px 0; + /* 11px = the `text-xs` the canvas/tables presence tags use, for pixel parity. */ + font-size: 11px; + font-weight: 500; + line-height: 1.2; + white-space: nowrap; + text-overflow: ellipsis; + background-color: var(--caret-color); + color: var(--surface-1); + user-select: none; + pointer-events: none; + opacity: 0; + transition: opacity 0.2s ease; +} + +.rich-markdown-prose .collaboration-carets__caret--active .collaboration-carets__label, +.rich-markdown-prose .collaboration-carets__caret:hover .collaboration-carets__label { + opacity: 1; +} + +/* Near the editor's right edge the label is flipped to the caret's left so it never + * runs off (JS toggles `--flip` after measuring); mirror the tag's notch corner. */ +.rich-markdown-prose .collaboration-carets__caret--flip .collaboration-carets__label { + left: auto; + right: -1px; + border-radius: 2px 2px 0 2px; +} + +/* Remote text selection: a rounded translucent tint of the collaborator's identity + * color (the alpha fill is set inline by selectionRender). */ +.rich-markdown-prose .collaboration-carets__selection { + border-radius: 2px; + pointer-events: none; +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 9107dbd795d..4290dd696fe 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -1,21 +1,26 @@ 'use client' -import { memo, useEffect, useRef, useState } from 'react' +import { memo, useCallback, useEffect, useRef, useState } from 'react' import { cn, toast } from '@sim/emcn' -import type { JSONContent } from '@tiptap/core' +import { FILE_DOC_SEED, type JoinFileDocError } from '@sim/realtime-protocol/file-doc' +import type { Extensions, JSONContent } from '@tiptap/core' +import { isChangeOrigin } from '@tiptap/extension-collaboration' import { Fragment, Slice } from '@tiptap/pm/model' import { NodeSelection } from '@tiptap/pm/state' import { dropPoint } from '@tiptap/pm/transform' import type { Editor } from '@tiptap/react' import { EditorContent, useEditor } from '@tiptap/react' import { useRouter } from 'next/navigation' +import { useSession } from '@/lib/auth/auth-client' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { extractEmbeddedFileRef } from '@/lib/uploads/utils/embedded-image-ref' +import { isUntitledName } from '@/app/workspace/[workspaceId]/files/untitled-title' import { useUploadWorkspaceFile } from '@/hooks/queries/workspace-files' import type { SaveStatus } from '@/hooks/use-autosave' import { useFileContentSource } from '@/hooks/use-file-content-source' import { PreviewLoadingFrame } from '../preview-shared' import { useEditableFileContent } from '../use-editable-file-content' +import { useFileDocCollaboration } from './collaboration/use-file-doc-collaboration' import { createMarkdownEditorExtensions } from './editor-extensions' import { findHeadingPos } from './heading-anchors' import { @@ -38,11 +43,14 @@ import { LinkHoverCard } from './menus/link-hover-card' import { TableBubbleMenu } from './menus/table-menu' import { normalizeMarkdownContent } from './normalize-content' import { isRoundTripSafe } from './round-trip-safety' +import { firstHeadingTitle } from './title-heading' import '@sim/emcn/components/code/code.css' import './rich-markdown-editor.css' +const PLACEHOLDER = "Write something, or press '/' for commands…" + const EXTENSIONS = createMarkdownEditorExtensions({ - placeholder: "Write something, or press '/' for commands…", + placeholder: PLACEHOLDER, embeds: true, }) @@ -50,6 +58,9 @@ const EXTENSIONS = createMarkdownEditorExtensions({ const STREAM_REPARSE_THROTTLE_THRESHOLD = 40_000 const STREAM_REPARSE_THROTTLE_MS = 120 +/** Debounce before naming a still-untitled file after its leading heading, so it fires once typing settles. */ +const DERIVE_TITLE_DEBOUNCE_MS = 600 + interface RichMarkdownEditorProps { file: WorkspaceFileRecord workspaceId: string @@ -71,6 +82,19 @@ interface RichMarkdownEditorProps { previewContextKey?: string /** Disable the `@` tag-insertion menu (existing tags still render). Defaults off — the file editor keeps tagging. */ disableTagging?: boolean + /** + * Opt this surface into live collaborative editing. Set only by the Files page — + * the dedicated editing surface, which never streams agent output. The agent/Chat + * surface leaves it off, so collaboration and agent-streaming are disjoint by + * construction (they cannot both drive one editor and corrupt the shared doc). + */ + collaborative?: boolean + /** + * Called (debounced) with the document's leading-heading text while the file is still untitled, so the + * caller can name the file after it. Omitted on read-only/non-editable surfaces. See + * {@link isUntitledName}. + */ + onDeriveTitleFromHeading?: (headingText: string) => void } /** Inline WYSIWYG markdown editor: agent output streams in read-only, then the same instance becomes editable on settle. */ @@ -89,7 +113,26 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ disableStreamingAutoScroll = false, previewContextKey, disableTagging, + collaborative = false, + onDeriveTitleFromHeading, }: RichMarkdownEditorProps) { + const { data: session, isPending: isSessionPending } = useSession() + const userId = session?.user?.id ?? '' + const userName = session?.user?.name?.trim() || 'Collaborator' + + /** + * Client-autosave gate. For a NON-collaborative file this is `true` (the client owns durability and + * autosaves the markdown). For a collaborative file it stays `false`: the realtime relay persists the + * shared document to markdown server-side, so the client must never also autosave — a stale keystroke + * saving over a server/copilot edit is exactly the clobber the server path closes. The child reports + * the right value up via `onCollabReadyChange`. + * + * Initialize from the `collaborative` prop (NOT unconditionally `true`): a collaborative file must + * start with autosave OFF, or a save could fire in the window before the child mounts and reports — + * re-clobbering exactly what this closes. The child turns it on for the non-collaborative fallback. + */ + const [collabReady, setCollabReady] = useState(!collaborative) + const { content, setDraftContent, @@ -108,9 +151,14 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ saveRef, discardRef, normalizeBaseline: normalizeMarkdownContent, + canAutosave: collabReady, }) - if (isContentLoading) return + // Wait for the session too: the child decides collaboration ONCE at mount from + // `userId`, so mounting before the session resolves would latch collaboration off + // for a cold-loaded file (both users would then solo-save, last-write-wins). + if (isContentLoading || isSessionPending) + return if (hasContentError) { return ( @@ -128,12 +176,17 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ content={content} isStreaming={isStreamInteractionLocked} canEdit={canEdit} + userId={userId} + userName={userName} autoFocus={autoFocus} streamIsIncremental={streamIsIncremental} disableStreamingAutoScroll={disableStreamingAutoScroll} disableTagging={disableTagging} + collaborative={collaborative} onChange={setDraftContent} onSaveShortcut={saveImmediately} + onCollabReadyChange={setCollabReady} + onDeriveTitleFromHeading={onDeriveTitleFromHeading} /> ) }) @@ -146,13 +199,22 @@ interface LoadedRichMarkdownEditorProps { /** True while agent output is streaming in: the editor renders it read-only and syncs each chunk. */ isStreaming: boolean canEdit: boolean + /** Current user id + display name, for the collaborative caret identity. */ + userId: string + userName: string autoFocus?: boolean /** See {@link RichMarkdownEditorProps.streamIsIncremental}. */ streamIsIncremental?: boolean disableStreamingAutoScroll?: boolean disableTagging?: boolean + /** See {@link RichMarkdownEditorProps.collaborative}. */ + collaborative?: boolean onChange: (markdown: string) => void onSaveShortcut: () => Promise + /** Reports whether the collaborative document is synced+seeded (autosave gate). */ + onCollabReadyChange: (ready: boolean) => void + /** See {@link RichMarkdownEditorProps.onDeriveTitleFromHeading}. */ + onDeriveTitleFromHeading?: (headingText: string) => void } interface SettledContent { @@ -172,12 +234,17 @@ export function LoadedRichMarkdownEditor({ content, isStreaming, canEdit, + userId, + userName, autoFocus, streamIsIncremental, disableStreamingAutoScroll, disableTagging, + collaborative = false, onChange, onSaveShortcut, + onCollabReadyChange, + onDeriveTitleFromHeading, }: LoadedRichMarkdownEditorProps) { /** Whether this editor mounted mid-stream — if so it starts empty and syncs streamed chunks until settle. */ const streamingAtMountRef = useRef(isStreaming) @@ -187,11 +254,48 @@ export function LoadedRichMarkdownEditor({ if (!streamingAtMountRef.current && settledRef.current === null) { settledRef.current = lockSettled(content) } - const isEditable = canEdit && !isStreaming && (settledRef.current?.verdict ?? false) + /** + * Collaboration is decided once at mount from synchronously-available inputs + * (`settledRef` is set just above) via `useState`-init, and never changes — TipTap + * fixes the extension set at editor creation, so it cannot turn on later. Enabled + * only on a `collaborative` surface (the Files page, which never streams) for an + * editable, round-trip-safe, non-streaming workspace document with a known user. + */ + const [collaborationEnabled] = useState( + () => + collaborative && + canEdit && + !streamingAtMountRef.current && + (settledRef.current?.verdict ?? false) && + Boolean(userId) && + (file.storageContext ?? 'workspace') === 'workspace' + ) + /** + * Whether the collaborative document is safe to edit + persist: synced and seeded. + * Starts `false` for a collaborative document — so the editor is read-only and + * autosave gated until the shared content has arrived (a user must not type into an + * empty, unsynced doc, which the seed would then discard) — and `true` for a local one. + */ + const [collabReady, setCollabReady] = useState(!collaborationEnabled) + const isEditable = + canEdit && !isStreaming && (settledRef.current?.verdict ?? false) && collabReady - /** Seed the doc once via lazy init — chunked parse is linear vs the editor's ~O(n²) whole-body markdown parse. */ + const collaboration = useFileDocCollaboration({ + fileId: file.id, + userId, + userName, + enabled: collaborationEnabled, + }) + + /** + * Initial editor content. When collaborating, the Y.Doc is the source of truth — + * start empty and let the server-seeded Yjs sync fill it (below); otherwise seed from the + * parsed markdown (chunked parse is linear vs the editor's ~O(n²) whole-body parse). + */ const [initialContent] = useState(() => - streamingAtMountRef.current ? '' : parseMarkdownToDoc(splitFrontmatter(content).body) + streamingAtMountRef.current || collaborationEnabled + ? '' + : parseMarkdownToDoc(splitFrontmatter(content).body) ) /** * The body currently shown in the editor: seeded from a settled mount, updated on local edits (via @@ -206,6 +310,31 @@ export function LoadedRichMarkdownEditor({ onChangeRef.current = onChange const onSaveShortcutRef = useRef(onSaveShortcut) onSaveShortcutRef.current = onSaveShortcut + + /** + * The frontmatter to re-attach to the body on save. For a collaborative doc it lives in the CRDT + * (config map, seeded/updated server-side), so a server edit that changes it is reflected rather + * than reverted by this editor's stale open-time copy; falls back to the locked `settledRef` copy + * before the seed lands and for non-collaborative documents. + */ + const resolveSaveFrontmatter = useCallback((): string => { + const fromDoc = collaboration?.doc + .getMap(FILE_DOC_SEED.configMap) + .get(FILE_DOC_SEED.frontmatterKey) + if (typeof fromDoc === 'string') return fromDoc + return settledRef.current?.frontmatter ?? '' + }, [collaboration]) + + /** + * While the file is still unnamed, name it after its leading heading: `onDeriveTitleFromHeading` is + * called (debounced) so the caller can rename the file, and `fileNameRef` lets the onUpdate handler + * read the current name without re-subscribing. See {@link isUntitledName}. + */ + const onDeriveTitleFromHeadingRef = useRef(onDeriveTitleFromHeading) + onDeriveTitleFromHeadingRef.current = onDeriveTitleFromHeading + const fileNameRef = useRef(file.name) + fileNameRef.current = file.name + const deriveTitleTimerRef = useRef | null>(null) /** * Read in the RAF tick so an already-scheduled tick still sees the latest edit kind (it can change * between sessions within one turn, e.g. an append followed by a rewrite). @@ -289,8 +418,27 @@ export function LoadedRichMarkdownEditor({ } } + /** + * Extensions: the shared module set for the local path, or a per-instance set + * carrying this document's Collaboration + CollaborationCaret. Built once (collab + * is decided at mount), since `useEditor` fixes the extension set at creation. + */ + const [extensions] = useState(() => + collaboration + ? createMarkdownEditorExtensions({ + placeholder: PLACEHOLDER, + embeds: true, + collaboration: { + doc: collaboration.doc, + awareness: collaboration.awareness, + user: collaboration.user, + }, + }) + : EXTENSIONS + ) + const editor = useEditor({ - extensions: EXTENSIONS, + extensions, editable: isEditable, enablePasteRules: false, autofocus: streamingAtMountRef.current ? false : autoFocus ? 'end' : false, @@ -441,14 +589,179 @@ export function LoadedRichMarkdownEditor({ return false }, }, - onUpdate: ({ editor }) => { + onUpdate: ({ editor, transaction }) => { const md = postProcessSerializedMarkdown(editor.getMarkdown()) lastSyncedBodyRef.current = md - onChangeRef.current(applyFrontmatter(settledRef.current?.frontmatter ?? '', md)) + onChangeRef.current(applyFrontmatter(resolveSaveFrontmatter(), md)) + // While the file is still untitled, name it after its leading heading once typing settles — but + // only for the LOCAL user's own edits. `isChangeOrigin` is true for a remote Yjs change (a peer + // typing); bail BEFORE touching the timer so a remote edit never cancels or reschedules the local + // user's pending rename (and every client doesn't schedule the same rename from a peer's + // not-yet-synced heading). It is false for local edits and non-collaborative surfaces. + if (isChangeOrigin(transaction)) return + // Local edit: restart the debounce. Clearing first cancels a stale rename if the heading was + // removed/changed before it fired; the timer re-derives the title from the live doc rather than a + // value captured now, so it can never name the file after a heading the user has since changed. + // `editor.isEditable` is the autosave gate (canEdit + settled + collab-ready), so a view-only + // viewer or the not-yet-editable mount seed never schedules a rename. + if (deriveTitleTimerRef.current) clearTimeout(deriveTitleTimerRef.current) + if ( + !editor.isEditable || + !isUntitledName(fileNameRef.current) || + firstHeadingTitle(editor.state.doc) === null + ) + return + deriveTitleTimerRef.current = setTimeout(() => { + const liveEditor = editorInstanceRef.current + if (!liveEditor || !liveEditor.isEditable || !isUntitledName(fileNameRef.current)) return + const title = firstHeadingTitle(liveEditor.state.doc) + if (title) onDeriveTitleFromHeadingRef.current?.(title) + }, DERIVE_TITLE_DEBOUNCE_MS) }, }) editorInstanceRef.current = editor + useEffect( + () => () => { + if (deriveTitleTimerRef.current) clearTimeout(deriveTitleTimerRef.current) + }, + [] + ) + + /** + * The loaded markdown to seed the shared doc from, held by pointer so the parse + * runs once at seed time rather than every render. + */ + const seedContentRef = useRef(content) + seedContentRef.current = content + + /** + * The collaborative document lifecycle. In one effect because the three concerns + * are one state machine keyed off the same provider events: + * - **observe** readiness: the server seeds the doc authoritatively (content + + * `initialContentLoaded` flag in ONE Yjs update), so the client only watches for + * synced AND seeded — it never imports content itself on the happy path; + * - **gate** the parent's autosave until the doc is synced AND seeded, so an + * empty/still-syncing doc can never overwrite the real file's markdown mirror; + * - **fall back** on a fatal join: seed the loaded content so it is SHOWN, but + * leave the editor read-only + gated. Every non-retryable failure (auth, access + * denied, not found, client-id conflict) either can't save or is moot, so the + * safe fallback is a read-only view of the content rather than editable-but- + * unsavable — which would silently drop the user's edits. + * + * `ready` (synced+seeded) gates BOTH the editor's editability (a user must never + * type into an empty/unsynced doc) and the parent's autosave. Non-collaborative + * documents are never gated. The server seeds the doc authoritatively (content + the + * seed flag arrive together), so the client only observes readiness. `provider.joinError` + * is latched, so a fatal rejection that fired before this subscription is not missed. + */ + useEffect(() => { + const setReady = (ready: boolean) => { + // Child-local: gates editability (a user must never type into an unsynced/unseeded doc). + setCollabReady(ready) + // Parent: gates CLIENT autosave. In a collaborative session the relay persists the doc to + // markdown server-side (debounced + on last-disconnect), so the client must NOT also autosave — + // a stale keystroke saving over a server/copilot edit is the clobber the server path closes. + // Only the non-collaborative (solo) path client-autosaves. + onCollabReadyChange(collaboration ? false : ready) + } + if (!collaboration) { + setReady(true) + return + } + const { provider, doc } = collaboration + if (!editor) { + setReady(false) + return + } + const config = doc.getMap(FILE_DOC_SEED.configMap) + + const seedFromLoaded = () => { + if (config.get(FILE_DOC_SEED.flag) === true) return + doc.transact(() => { + editor.commands.setContent( + parseMarkdownToDoc(splitFrontmatter(seedContentRef.current).body), + { contentType: 'json', emitUpdate: false } + ) + config.set(FILE_DOC_SEED.flag, true) + }) + } + + if (!provider) { + setReady(false) + return + } + + // The server seeds the document (content + the `initialContentLoaded` flag), so readiness is + // simply "synced AND seeded" — no client-side seed import on the happy path. `seedFromLoaded` + // remains only for the offline fallback below (a non-retryable join error / readiness timeout), + // where it locally renders the file read-only since the server can't be reached. + const report = () => setReady(provider.synced && config.get(FILE_DOC_SEED.flag) === true) + const onJoinError = (error: JoinFileDocError) => { + if (error.retryable === false) seedFromLoaded() + } + + // A server edit that changes ONLY the frontmatter (e.g. copilot) updates the config map but not + // the body fragment, so TipTap's `onUpdate` never fires and the autosave draft would keep the + // stale open-time frontmatter — an explicit save could then revert the live change. Re-attach the + // new frontmatter to the current body and push a fresh draft whenever it changes on its own. + let lastFrontmatter = config.get(FILE_DOC_SEED.frontmatterKey) + const syncFrontmatter = () => { + const current = config.get(FILE_DOC_SEED.frontmatterKey) + if (current === lastFrontmatter) return + lastFrontmatter = current + // Null body ref ⇒ no body has synced yet (e.g. this fired before the seed's own `onUpdate`); + // that path re-attaches the frontmatter itself, so there is nothing to do here. + if (lastSyncedBodyRef.current !== null) { + onChangeRef.current( + applyFrontmatter(typeof current === 'string' ? current : '', lastSyncedBodyRef.current) + ) + } + } + + provider.on('synced', report) + provider.on('join-error', onJoinError) + config.observe(report) + config.observe(syncFrontmatter) + report() + if (provider.joinError) onJoinError(provider.joinError) + + return () => { + provider.off('synced', report) + provider.off('join-error', onJoinError) + config.unobserve(report) + config.unobserve(syncFrontmatter) + // Report NOT ready on teardown — the safe direction. If this effect ever re-runs while mounted + // (a future dep change), briefly gating autosave off is harmless; reporting `true` here could + // ungate it while the doc is unready. + onCollabReadyChange(false) + } + }, [collaboration, editor, onCollabReadyChange, setCollabReady]) + + /** + * Owns editability for the collaborative lifecycle: `useEditor`'s `editable` is only + * the initial value, and the streaming/settle effect stays inert in collab mode — so + * re-apply here whenever collaboration readiness (synced + seeded) flips `isEditable`. + */ + useEffect(() => { + if (!editor || !collaborationEnabled) return + if (editor.isEditable === isEditable) return + // Defer out of the render/commit phase. `isEditable` flips from collab readiness (synced + seeded), + // which is driven by a Yjs `config.observe` firing synchronously inside `Y.applyUpdate` — so this + // effect can run while React is mid-render. `setEditable` dispatches a TipTap transaction that the + // React binding commits with `flushSync`, which throws ("cannot flush while rendering") in that + // window. A microtask runs right after the current commit, before paint; re-check liveness/value + // since either can change before it fires. + let cancelled = false + queueMicrotask(() => { + if (cancelled || editor.isDestroyed) return + if (editor.isEditable !== isEditable) editor.setEditable(isEditable) + }) + return () => { + cancelled = true + } + }, [editor, collaborationEnabled, isEditable]) + /** * Wire the `/Image` slash command to the hidden picker (per-editor storage, since the extension set is * shared across instances). Reads only refs, so the handler stays stable across the editor's life. @@ -471,8 +784,17 @@ export function LoadedRichMarkdownEditor({ const pendingStreamBodyRef = useRef(null) const streamRafRef = useRef(null) const lastStreamParseAtRef = useRef(0) + const settleRunSeqRef = useRef(0) + const pendingCollapseRef = useRef(false) useEffect(() => { if (!editor) return + // Collaboration and agent-streaming are disjoint surfaces: collaboration is enabled + // only on the Files page, which never streams agent output, so in collab mode this + // reconcile loop stays fully inert. It must not run even defensively — its + // `setContent` would sync a full-document replace into the shared Y.Doc (the + // ySyncPlugin writes it regardless of `emitUpdate: false`), wiping peers' edits. + // Editability is owned by the reactive effect above. + if (collaborationEnabled) return const syncEditorBody = (body: string) => { if (body === lastSyncedBodyRef.current) return lastSyncedBodyRef.current = body @@ -481,9 +803,29 @@ export function LoadedRichMarkdownEditor({ emitUpdate: false, }) } + // Editor view mutations flush synchronously through the @tiptap/react binding (setContent mounts + // the React node views via flushSync — tiptap#3764), so calling them directly in this effect body + // throws "flushSync ... cannot flush while rendering" when the effect runs mid-render. Defer to a + // microtask (after commit, before paint). The streaming rAF tick below already runs off-render. + // + // Tag each run: a deferred mutation applies only if it is still the latest run (this effect has + // several early-return exits, so a run-token beats a per-exit cleanup flag) and the editor is + // alive. This drops a superseded run's microtask when React ran the next pass — a newer stream or + // settle — before the microtask flushed, so it can't overwrite the newer state. + const runSeq = ++settleRunSeqRef.current + const runOffRender = (mutate: () => void) => { + queueMicrotask(() => { + if (runSeq !== settleRunSeqRef.current || editor.isDestroyed) return + mutate() + }) + } if (isStreaming) { wasStreamingRef.current = true - if (editor.isEditable) editor.setEditable(false) + if (editor.isEditable) { + runOffRender(() => { + if (editor.isEditable) editor.setEditable(false) + }) + } const body = splitFrontmatter(content).body if (body === lastSyncedBodyRef.current) return pendingStreamBodyRef.current = body @@ -532,23 +874,47 @@ export function LoadedRichMarkdownEditor({ if (isInitialSettle || wasStreamingRef.current) { wasStreamingRef.current = false settledRef.current = lockSettled(content) - syncEditorBody(splitFrontmatter(content).body) - // `setContent` maps any pre-existing selection onto the new doc rather than clearing it — a - // select-all survives as "select everything," permanently painting every divider/image with the - // `rich-leaf-in-selection` decoration (keymap.ts) until the user clicks elsewhere. This must run - // on every settle regardless of whether `setContent` ran just above: the last streaming tick - // already syncs `lastSyncedBodyRef` to the final body before settle, so `body` usually already - // equals it here — collapsing only inside that `if` would skip the common streamed-content case - // entirely. `setTextSelection` (not `.focus()`) so this never steals DOM focus from whatever the - // user is doing outside the editor. - editor.commands.setTextSelection(editor.state.doc.content.size) - editor.setEditable(canEdit && settledRef.current.verdict) - if (isInitialSettle && autoFocus) editor.commands.focus('end') + const settledVerdict = settledRef.current.verdict + const shouldFocus = isInitialSettle && autoFocus + // A settle owes a selection collapse. Track it as a ref, not just inline in this microtask: if a + // newer run bumps the token before this microtask fires, this settle's task is dropped — but the + // debt survives, and the run that supersedes it (settle OR the steady-sync path below) clears it. + pendingCollapseRef.current = true + // One ordered microtask: set body → collapse selection → re-apply editability. The collapse is + // load-bearing and runs on every settle even when the body is unchanged — setContent maps a + // pre-existing selection onto the new doc, so a prior select-all survives as "select everything", + // permanently painting every divider/image with the rich-leaf-in-selection decoration (keymap.ts) + // until the user clicks away. setTextSelection (not .focus()) never steals DOM focus. + runOffRender(() => { + syncEditorBody(splitFrontmatter(content).body) + pendingCollapseRef.current = false + editor.commands.setTextSelection(editor.state.doc.content.size) + editor.setEditable(canEdit && settledVerdict && collabReady) + if (shouldFocus) editor.commands.focus('end') + }) return } - syncEditorBody(splitFrontmatter(content).body) - if (settledRef.current) editor.setEditable(canEdit && settledRef.current.verdict) - }, [editor, content, isStreaming, canEdit, autoFocus, disableStreamingAutoScroll]) + const settled = settledRef.current + runOffRender(() => { + syncEditorBody(splitFrontmatter(content).body) + // Honor a collapse a superseded settle owed but never applied (its microtask was dropped when this + // run bumped the token), so a post-stream select-all can't keep the leaf-in-selection decoration. + if (pendingCollapseRef.current) { + pendingCollapseRef.current = false + editor.commands.setTextSelection(editor.state.doc.content.size) + } + if (settled) editor.setEditable(canEdit && settled.verdict && collabReady) + }) + }, [ + editor, + content, + isStreaming, + canEdit, + autoFocus, + disableStreamingAutoScroll, + collaborationEnabled, + collabReady, + ]) useEffect( () => () => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.test.ts new file mode 100644 index 00000000000..f6c46bd027f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.test.ts @@ -0,0 +1,63 @@ +/** + * @vitest-environment jsdom + */ +import { Editor } from '@tiptap/core' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createMarkdownEditorExtensions } from './editor-extensions' +import { firstHeadingTitle } from './title-heading' + +function editorWith(markdown: string): Editor { + const editor = new Editor({ extensions: createMarkdownEditorExtensions({ placeholder: '' }) }) + if (markdown) editor.commands.setContent(markdown, { contentType: 'markdown' }) + return editor +} + +describe('firstHeadingTitle', () => { + beforeEach(() => { + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ) + Element.prototype.scrollIntoView = vi.fn() + document.elementFromPoint = vi.fn(() => null) + }) + + it('returns the leading H1 text', () => { + const editor = editorWith('# Q3 Planning\n\nbody') + expect(firstHeadingTitle(editor.state.doc)).toBe('Q3 Planning') + editor.destroy() + }) + + it('returns the text of any leading heading level', () => { + const editor = editorWith('## Sub title') + expect(firstHeadingTitle(editor.state.doc)).toBe('Sub title') + editor.destroy() + }) + + it('returns null when the first block is a paragraph, not a heading', () => { + const editor = editorWith('just text\n\n# later heading') + expect(firstHeadingTitle(editor.state.doc)).toBeNull() + editor.destroy() + }) + + it('returns null for an empty leading heading', () => { + const editor = editorWith('') + editor.commands.setContent({ type: 'doc', content: [{ type: 'heading', attrs: { level: 1 } }] }) + expect(firstHeadingTitle(editor.state.doc)).toBeNull() + editor.destroy() + }) + + it('returns null for a whitespace-only leading heading (trim boundary)', () => { + const editor = editorWith('') + editor.commands.setContent({ + type: 'doc', + content: [{ type: 'heading', attrs: { level: 1 }, content: [{ type: 'text', text: ' ' }] }], + }) + expect(firstHeadingTitle(editor.state.doc)).toBeNull() + editor.destroy() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.ts new file mode 100644 index 00000000000..3877519ca97 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/title-heading.ts @@ -0,0 +1,9 @@ +import type { Node as ProseMirrorNode } from '@tiptap/pm/model' + +/** The text of the document's leading heading (any level), or null when the first block isn't a heading. */ +export function firstHeadingTitle(doc: ProseMirrorNode): string | null { + const first = doc.firstChild + if (!first || first.type.name !== 'heading') return null + const text = first.textContent.trim() + return text.length > 0 ? text : null +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts index 984d2b2e1de..f8dac76bbdd 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts @@ -61,6 +61,13 @@ interface UseEditableFileContentOptions { * the at-rest baseline, never while an agent stream is in flight. Stable reference required. */ normalizeBaseline?: (raw: string) => string + /** + * Extra gate on autosave (and draft persistence). When `false`, saving is + * suppressed even when otherwise eligible — the collaborative editor uses it to + * hold saves until the shared document is synced AND seeded, so an empty or + * partially-synced doc can never overwrite the real file. Defaults to `true`. + */ + canAutosave?: boolean } interface EditableFileContent { @@ -136,6 +143,7 @@ export function useEditableFileContent({ saveRef, discardRef, normalizeBaseline, + canAutosave = true, }: UseEditableFileContentOptions): EditableFileContent { const onDirtyChangeRef = useRef(onDirtyChange) const onSaveStatusChangeRef = useRef(onSaveStatusChange) @@ -234,7 +242,7 @@ export function useEditableFileContent({ [workspaceId, file.id, markSavedContent] ) - const autosaveEnabled = canEdit && isInitialized && !isStreamInteractionLocked + const autosaveEnabled = canEdit && isInitialized && !isStreamInteractionLocked && canAutosave const { saveStatus, saveImmediately, isDirty, discard } = useAutosave({ content, @@ -249,9 +257,16 @@ export function useEditableFileContent({ ), }) + // When the client can't autosave it isn't the durability owner: the collaborative editor holds + // `canAutosave` permanently false because the relay persists the doc server-side (debounced + on + // last-disconnect), so `savedContent` never advances and raw `isDirty` would latch true after any + // local OR remote keystroke — surfacing a spurious "Unsaved changes" navigation prompt whose + // "Discard" discards nothing real. With nothing the user can save, there is nothing to warn about. + const isDirtyForCaller = canAutosave && isDirty + useEffect(() => { - onDirtyChangeRef.current?.(isDirty) - }, [isDirty]) + onDirtyChangeRef.current?.(isDirtyForCaller) + }, [isDirtyForCaller]) useEffect(() => { onSaveStatusChangeRef.current?.( @@ -299,6 +314,6 @@ export function useEditableFileContent({ hasContentError: streamingContent === undefined && Boolean(error) && !isInitialized, saveStatus, saveImmediately, - isDirty, + isDirty: isDirtyForCaller, } } diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index da8ac589312..82aad11ad2a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -77,8 +77,11 @@ import { isPreviewable, isTextEditable, } from '@/app/workspace/[workspaceId]/files/components/file-viewer' +import { FileDocAvatars } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-avatars' +import { FileDocRoomProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context' import { FilesListContextMenu } from '@/app/workspace/[workspaceId]/files/components/files-list-context-menu' import { ShareModal } from '@/app/workspace/[workspaceId]/files/components/share-modal' +import { useWorkspaceFilesRoom } from '@/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room' import { filesFilterParsers, filesFilterUrlKeys, @@ -86,6 +89,12 @@ import { filesSortParams, filesUrlKeys, } from '@/app/workspace/[workspaceId]/files/search-params' +import { + DEFAULT_UNTITLED_NAME, + deriveMarkdownFileName, + isUntitledName, + uniqueMarkdownName, +} from '@/app/workspace/[workspaceId]/files/untitled-title' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks' import { usePinItem, usePinnedIds, useUnpinItem } from '@/hooks/queries/pinned-items' @@ -208,6 +217,11 @@ export function Files() { const canEdit = userPermissions.canEdit === true const { config: permissionConfig } = usePermissionConfig() + // Joined for the live file tree: a `workspace-files-changed` broadcast invalidates the + // browser. "Who's in this file" comes from the file-doc room (see FileDocRoomProvider), + // not from who's browsing the Files section. + useWorkspaceFilesRoom(workspaceId) + useEffect(() => { if (permissionConfig.hideFilesTab) { router.replace(`/workspace/${workspaceId}`) @@ -358,6 +372,34 @@ export function Files() { const selectedFileRef = useRef(selectedFile) selectedFileRef.current = selectedFile + /** + * While a file is still untitled, name it after the leading heading the user types in its editor. The + * editor reports the heading text (debounced); here we re-check the file is still untitled, derive a + * unique `.md` name among its folder siblings, and rename. A no-op once the file has a real name. + */ + const handleDeriveTitleFromHeading = useCallback( + (headingText: string) => { + const currentFile = selectedFileRef.current + if (!currentFile || !isUntitledName(currentFile.name)) return + const derived = deriveMarkdownFileName(headingText) + if (!derived) return + const siblingNames = new Set( + filesRef.current + .filter( + (f) => + (f.folderId ?? null) === (currentFile.folderId ?? null) && f.id !== currentFile.id + ) + .map((f) => f.name) + ) + const name = uniqueMarkdownName(derived, siblingNames) + if (name === currentFile.name) return + renameFile + .mutateAsync({ workspaceId, fileId: currentFile.id, name }) + .catch((err) => logger.error('Failed to auto-name file from heading:', err)) + }, + [workspaceId] + ) + const shareFile = shareFileId ? (files.find((f) => f.id === shareFileId) ?? null) : null const shareModal = shareFile ? ( (f.folderId ?? null) === currentFolderId).map((f) => f.name) ) - let name = 'untitled.md' - let counter = 1 - while (existingNames.has(name)) { - name = `untitled (${counter}).md` - counter++ - } + const name = uniqueMarkdownName(DEFAULT_UNTITLED_NAME, existingNames) const mimeType = getMimeTypeFromExtension('md') const blob = new Blob([''], { type: mimeType }) @@ -1979,35 +2016,43 @@ export function Files() { if (selectedFile) { return ( <> - - - + {/* The room provider scopes "who's in this file" presence to the open document: the + editor (inside FileViewer) publishes the server-authenticated roster and the + header's FileDocAvatars reads it — both must be descendants. */} + + + } + /> + - - + + + + invalidateWorkspaceFileBrowsers(queryClient, workspaceId) + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/untitled-title.test.ts b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.test.ts new file mode 100644 index 00000000000..e9ee0ba2e43 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from 'vitest' +import { + DEFAULT_UNTITLED_NAME, + deriveMarkdownFileName, + isUntitledName, + uniqueMarkdownName, +} from './untitled-title' + +describe('untitled format single-source-of-truth', () => { + // Guards against DEFAULT_UNTITLED_NAME / uniqueMarkdownName drifting from the isUntitledName regex: + // the default name and its deduped siblings must always read back as "untitled". + it('recognizes the default name and its deduped siblings as untitled', () => { + expect(isUntitledName(DEFAULT_UNTITLED_NAME)).toBe(true) + const second = uniqueMarkdownName(DEFAULT_UNTITLED_NAME, new Set([DEFAULT_UNTITLED_NAME])) + expect(second).toBe('untitled (1).md') + expect(isUntitledName(second)).toBe(true) + }) +}) + +describe('isUntitledName', () => { + it.each([ + ['untitled.md', true], + ['untitled (1).md', true], + ['untitled (23).md', true], + ['Untitled.md', false], + ['untitled.txt', false], + ['untitled', false], + ['my notes.md', false], + ['untitled draft.md', false], + ['untitled ().md', false], + ])('%s → %s', (name, expected) => { + expect(isUntitledName(name)).toBe(expected) + }) +}) + +describe('deriveMarkdownFileName', () => { + it('turns heading text into a .md file name', () => { + expect(deriveMarkdownFileName('Q3 Planning')).toBe('Q3 Planning.md') + }) + it('strips filesystem-illegal characters and collapses whitespace', () => { + expect(deriveMarkdownFileName('Roadmap: Q3 / Q4 *draft*')).toBe('Roadmap Q3 Q4 draft.md') + }) + it('keeps hyphens and dots inside the title', () => { + expect(deriveMarkdownFileName('v1.2 - release-notes')).toBe('v1.2 - release-notes.md') + }) + it('returns null when nothing usable remains', () => { + expect(deriveMarkdownFileName(' ')).toBeNull() + expect(deriveMarkdownFileName('///')).toBeNull() + }) + + it('does not double the extension when the heading already ends in .md', () => { + expect(deriveMarkdownFileName('README.md')).toBe('README.md') + expect(deriveMarkdownFileName('notes.MD')).toBe('notes.MD') + }) + it('hard-caps the length (no ellipsis) before the extension', () => { + const result = deriveMarkdownFileName('a'.repeat(200)) + expect(result).toBe(`${'a'.repeat(100)}.md`) + }) + + it('re-trims when the hard cap lands on a space (no "foo .md")', () => { + // 99 non-space chars + space at index 99 → truncate(100) leaves a trailing space to re-trim away. + const result = deriveMarkdownFileName(`${'a'.repeat(99)} bcd`) + expect(result).toBe(`${'a'.repeat(99)}.md`) + }) +}) + +describe('uniqueMarkdownName', () => { + it('returns the name unchanged when free', () => { + expect(uniqueMarkdownName('notes.md', new Set())).toBe('notes.md') + }) + it('appends an incrementing suffix before the extension when taken', () => { + expect(uniqueMarkdownName('notes.md', new Set(['notes.md']))).toBe('notes (1).md') + expect(uniqueMarkdownName('notes.md', new Set(['notes.md', 'notes (1).md']))).toBe( + 'notes (2).md' + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts new file mode 100644 index 00000000000..4800fb13e9a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts @@ -0,0 +1,57 @@ +import { truncate } from '@sim/utils/string' + +/** + * The name a freshly-created markdown file is given in `handleCreateFile`: `untitled.md`, or + * `untitled (n).md` when that is taken. A file keeps this "unnamed" status until it is renamed — + * while unnamed, typing a leading heading names the file (one direction only; the reverse + * name→heading seed was removed as unsafe on the shared editor). See {@link isUntitledName}. + */ +export const DEFAULT_UNTITLED_NAME = 'untitled.md' + +const UNTITLED_NAME_RE = /^untitled(?: \(\d+\))?\.md$/ + +/** Longest title kept when deriving a file name from a heading, before the `.md` extension. */ +const MAX_DERIVED_TITLE_LENGTH = 100 + +/** + * Filename characters disallowed across the common platforms (`\ / : * ? " < > |`) plus C0 control + * characters, replaced with a space when deriving a file name from heading text. + */ +const ILLEGAL_FILENAME_CHARS = /[\\/:*?"<>|\x00-\x1f]/g + +/** True when `name` is still the auto-assigned untitled markdown name (`untitled.md`, `untitled (2).md`). */ +export function isUntitledName(name: string): boolean { + return UNTITLED_NAME_RE.test(name) +} + +/** + * Derives a markdown file name from heading text — illegal filename characters dropped, whitespace + * collapsed, trimmed, hard-capped at {@link MAX_DERIVED_TITLE_LENGTH}, and suffixed with `.md`. + * Returns null when nothing usable remains (e.g. a heading of only slashes), so the caller keeps the + * current name. + */ +export function deriveMarkdownFileName(headingText: string): string | null { + const base = headingText.replace(ILLEGAL_FILENAME_CHARS, ' ').replace(/\s+/g, ' ').trim() + if (!base) return null + // Re-trim after the hard cap: truncation can land mid-word and leave a trailing space (`"foo .md"`). + const capped = truncate(base, MAX_DERIVED_TITLE_LENGTH, '').trim() + if (!capped) return null + // A heading that already ends in `.md` (e.g. `# README.md`) must not become `README.md.md`. + return /\.md$/i.test(capped) ? capped : `${capped}.md` +} + +/** + * Makes `name` unique among `existingNames` by appending ` (n)` before the `.md` extension — the same + * scheme `handleCreateFile` uses for the default untitled name. + */ +export function uniqueMarkdownName(name: string, existingNames: ReadonlySet): string { + if (!existingNames.has(name)) return name + const withoutExt = name.replace(/\.md$/i, '') + let counter = 1 + let candidate = `${withoutExt} (${counter}).md` + while (existingNames.has(candidate)) { + counter++ + candidate = `${withoutExt} (${counter}).md` + } + return candidate +} diff --git a/apps/sim/app/workspace/[workspaceId]/hooks/use-workspace-invalidation-room.ts b/apps/sim/app/workspace/[workspaceId]/hooks/use-workspace-invalidation-room.ts new file mode 100644 index 00000000000..0c652fcb8d6 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/hooks/use-workspace-invalidation-room.ts @@ -0,0 +1,111 @@ +'use client' + +import { useEffect, useRef } from 'react' +import { createLogger } from '@sim/logger' +import type { RoomType } from '@sim/realtime-protocol/rooms' +import { useSocket } from '@/app/workspace/providers/socket-provider' + +const logger = createLogger('WorkspaceInvalidationRoom') + +/** Retry cap + base delay for a retryable join failure on an otherwise-live socket. */ +const MAX_JOIN_RETRIES = 3 +const JOIN_RETRY_BASE_MS = 1000 + +interface JoinErrorPayload { + workspaceId: string + error: string + code: string + retryable?: boolean +} + +/** + * Joins a workspace-scoped, presence-free "invalidation room" over the shared socket and runs + * `onChanged` whenever the server broadcasts `${roomType}-changed` for this workspace, so the list + * refetches without waiting for staleness. Shared core behind {@link useWorkspaceFilesRoom} and + * {@link useWorkspaceTablesRoom}; event names derive from `roomType`. + * + * These rooms carry no presence — "who's in a resource" comes from the per-resource room, not from + * who's browsing the section. Mutations happen server-side (HTTP + copilot) and fan out this signal. + */ +export function useWorkspaceInvalidationRoom( + workspaceId: string, + roomType: RoomType, + onChanged: () => void +): void { + const { socket } = useSocket() + // Held by ref so a caller passing a fresh closure each render never re-subscribes the socket. + const onChangedRef = useRef(onChanged) + onChangedRef.current = onChanged + + useEffect(() => { + if (!socket || !workspaceId) return + + const joinEvent = `join-${roomType}` + const successEvent = `${joinEvent}-success` + const errorEvent = `${joinEvent}-error` + const leaveEvent = `leave-${roomType}` + const changedEvent = `${roomType}-changed` + + let retries = 0 + let retryTimer: ReturnType | null = null + + const join = () => socket.emit(joinEvent, { workspaceId }) + + // A fresh (re)connect gets a fresh retry budget, so a prior full exhaustion doesn't leave the + // socket unable to retry a failed re-join until the next success. Cancel any retry still pending + // from before the reconnect so it can't fire a duplicate join after this immediate one. + const handleConnect = () => { + retries = 0 + if (retryTimer) { + clearTimeout(retryTimer) + retryTimer = null + } + join() + } + + const handleJoinSuccess = (data: { workspaceId: string }) => { + if (data.workspaceId !== workspaceId) return + retries = 0 + // Cancel any retry scheduled by a prior retryable error so it can't fire an extra + // join after we're already in. + if (retryTimer) { + clearTimeout(retryTimer) + retryTimer = null + } + } + const handleJoinError = (data: JoinErrorPayload) => { + if (data.workspaceId !== workspaceId) return + logger.warn(`Failed to join ${roomType} room`, { code: data.code, error: data.error }) + if (data.retryable && retries < MAX_JOIN_RETRIES) { + retries += 1 + // Clear any still-pending retry before scheduling a new one, so reconnect churn can't + // orphan a timer that fires an extra join(). + if (retryTimer) clearTimeout(retryTimer) + retryTimer = setTimeout(join, JOIN_RETRY_BASE_MS * retries) + } + } + const handleChanged = (data: { workspaceId: string }) => { + if (data.workspaceId === workspaceId) onChangedRef.current() + } + + // Join now if the socket is already connected; `connect` covers (re)connects. + if (socket.connected) join() + socket.on('connect', handleConnect) + socket.on(successEvent, handleJoinSuccess) + socket.on(errorEvent, handleJoinError) + socket.on(changedEvent, handleChanged) + + return () => { + if (retryTimer) clearTimeout(retryTimer) + socket.off('connect', handleConnect) + socket.off(successEvent, handleJoinSuccess) + socket.off(errorEvent, handleJoinError) + socket.off(changedEvent, handleChanged) + + // Leave the room, scoped to THIS workspace: the server no-ops if the socket has already + // switched to another workspace's room (so a workspace A→B switch, where B's join runs first + // and auto-leaves A, can't have A's leave evict B). + socket.emit(leaveEvent, { workspaceId }) + } + }, [socket, workspaceId, roomType]) +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx new file mode 100644 index 00000000000..5461082f3de --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/remote-selection-overlay.tsx @@ -0,0 +1,312 @@ +'use client' + +import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' +import { createPortal } from 'react-dom' +import { getUserColor, withAlpha } from '@/lib/workspaces/colors' +import { + isCellInSelection, + type NormalizedSelection, +} from '@/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/utils' +import type { RemoteTableSelection } from '@/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room' + +/** A measured remote selection, positioned in the grid content wrapper's space. */ +interface SelectionBox { + socketId: string + userName: string + color: string + editing: boolean + top: number + left: number + width: number + height: number + /** Viewport-space top/left of the selection, for the body-portaled name label. */ + viewportTop: number + viewportLeft: number + /** Resolved anchor/focus cell indices (undefined when off-window). Coverage by the local + * selection is derived from these in render (see {@link isSelectionCovered}) — no DOM + * re-measure when only the local caret moves. */ + anchorRow: number | undefined + anchorCol: number | undefined + focusRow: number | undefined + focusCol: number | undefined +} + +interface RemoteSelectionOverlayProps { + remoteSelections: RemoteTableSelection[] + /** Column id → its rendered column index (matches the cells' `data-col`). */ + columnIndexById: Map + /** Row id → its index in the current row list, to test local-selection coverage. */ + rowIndexById: Map + /** The local user's own normalized selection, so a co-selected remote cell defers to it. */ + localSelection: NormalizedSelection | null + /** The grid's scroll container (`data-table-scroll`), queried for cell rects. */ + scrollElement: HTMLElement | null +} + +/** The cell `` for a (rowId, columnIndex), or undefined when virtualized off-window. */ +function cellRect( + scrollEl: HTMLElement, + rowId: string, + columnIndex: number | undefined +): DOMRect | undefined { + if (columnIndex === undefined) return undefined + // `rowId` is a remote peer's value — escape it so a hostile id can't break the + // selector and throw (`columnIndex` is a local numeric index, already safe). + const cell = scrollEl.querySelector( + `[data-row-id="${CSS.escape(rowId)}"][data-col="${columnIndex}"]` + ) + return cell?.getBoundingClientRect() +} + +/** + * Whether a remote selection defers to the local one: true only when the local selection + * fully contains it (both corners inside), so a partial overlap still shows. + */ +function isSelectionCovered( + anchorRow: number | undefined, + anchorCol: number | undefined, + focusRow: number | undefined, + focusCol: number | undefined, + bounds: NormalizedSelection | null +): boolean { + return ( + bounds !== null && + isCellInSelection(anchorRow, anchorCol, bounds) && + isCellInSelection(focusRow, focusCol, bounds) + ) +} + +/** + * Renders remote collaborators' cell selections over the table grid — a colored + * border per user (Google-Sheets style), a darker fill while they are editing, and + * their name on hover. Mounted inside the grid's `relative` content wrapper, so + * content-space coordinates scroll with the grid automatically. + * + * Positions are measured from the live cell rects (the same `[data-row-id][data-col]` + * idiom the reveal effect uses), keyed by stable ids so each client renders under its + * own sort/scroll. A selection whose rows are virtualized off-window is simply not + * drawn. The layer is `pointer-events-none` so it never intercepts cell clicks; the + * name-on-hover is driven by hit-testing pointer moves against the measured boxes. + */ +export function RemoteSelectionOverlay({ + remoteSelections, + columnIndexById, + rowIndexById, + localSelection, + scrollElement, +}: RemoteSelectionOverlayProps) { + const rootRef = useRef(null) + const [boxes, setBoxes] = useState([]) + const [hoveredSocketId, setHoveredSocketId] = useState(null) + + // Latest data read by the subscribe-once effect + the pointer hit-test, so neither + // re-subscribes on every incoming selection delta. + const boxesRef = useRef([]) + boxesRef.current = boxes + const remoteSelectionsRef = useRef(remoteSelections) + remoteSelectionsRef.current = remoteSelections + const columnIndexByIdRef = useRef(columnIndexById) + columnIndexByIdRef.current = columnIndexById + const rowIndexByIdRef = useRef(rowIndexById) + rowIndexByIdRef.current = rowIndexById + // Read only by the pointer hit-test (never in render) to skip a locally-covered box. + const localSelectionRef = useRef(localSelection) + localSelectionRef.current = localSelection + // Cached content-wrapper origin, refreshed on each measure (scroll/resize/data change), + // so the pointer hit-test never forces a layout read per mouse move. + const originRef = useRef({ top: 0, left: 0 }) + + const measure = useCallback(() => { + const scrollEl = scrollElement + const root = rootRef.current + if (!scrollEl || !root) return + const origin = root.getBoundingClientRect() + originRef.current = { top: origin.top, left: origin.left } + const next: SelectionBox[] = [] + for (const selection of remoteSelectionsRef.current) { + const { anchor, focus, editing } = selection.cell + const anchorCol = columnIndexByIdRef.current.get(anchor.columnId) + const focusCol = columnIndexByIdRef.current.get(focus.columnId) + const anchorRow = rowIndexByIdRef.current.get(anchor.rowId) + const focusRow = rowIndexByIdRef.current.get(focus.rowId) + const rects = [ + cellRect(scrollEl, anchor.rowId, anchorCol), + cellRect(scrollEl, focus.rowId, focusCol), + ].filter((rect): rect is DOMRect => rect !== undefined) + if (rects.length === 0) continue + + const viewportTop = Math.min(...rects.map((r) => r.top)) + const viewportLeft = Math.min(...rects.map((r) => r.left)) + const top = viewportTop - origin.top + const left = viewportLeft - origin.left + const bottom = Math.max(...rects.map((r) => r.bottom)) - origin.top + const right = Math.max(...rects.map((r) => r.right)) - origin.left + next.push({ + socketId: selection.socketId, + userName: selection.userName, + color: getUserColor(selection.userId), + editing: editing === true, + top, + left, + width: right - left, + height: bottom - top, + viewportTop, + viewportLeft, + anchorRow, + anchorCol, + focusRow, + focusCol, + }) + } + setBoxes(next) + }, [scrollElement]) + + // Subscribe once per scroll element: re-measure on scroll/resize, and hit-test pointer + // moves against the cached boxes/origin — no layout read per move, stays pointer-events-none. + useEffect(() => { + const scrollEl = scrollElement + if (!scrollEl) return + + let raf = 0 + const schedule = () => { + if (!raf) + raf = requestAnimationFrame(() => { + raf = 0 + measure() + }) + } + const handleMove = (event: PointerEvent) => { + const { top, left } = originRef.current + const x = event.clientX - left + const y = event.clientY - top + const hit = boxesRef.current.find( + (b) => + x >= b.left && + x <= b.left + b.width && + y >= b.top && + y <= b.top + b.height && + // Skip a box the local selection covers — it isn't drawn, so hovering it must not + // pop a name tag over a cell with no visible remote selection. + !isSelectionCovered( + b.anchorRow, + b.anchorCol, + b.focusRow, + b.focusCol, + localSelectionRef.current + ) + ) + setHoveredSocketId((prev) => + prev === (hit?.socketId ?? null) ? prev : (hit?.socketId ?? null) + ) + } + const handleLeave = () => setHoveredSocketId(null) + + // No measure() here — the re-measure layout effect below runs on mount and whenever + // `measure` changes (it depends on `scrollElement`), so it already covers the initial + // and scroll-element-changed measures without a redundant pass. + scrollEl.addEventListener('scroll', schedule, { passive: true }) + scrollEl.addEventListener('pointermove', handleMove, { passive: true }) + scrollEl.addEventListener('pointerleave', handleLeave) + const resizeObserver = new ResizeObserver(schedule) + resizeObserver.observe(scrollEl) + // Also observe the content layer (this overlay fills it): a column resize or a + // row-count change grows/shrinks the content without resizing the scroll container, + // yet moves cell rects — so measure off the content, not just the viewport. + if (rootRef.current) resizeObserver.observe(rootRef.current) + // Re-measure when rows are added/removed/reordered/virtualized (a live refetch moves + // cells without a scroll/resize) — childList only, so a cell-content edit doesn't fire. + const tbody = scrollEl.querySelector('tbody') + const rowObserver = new MutationObserver(schedule) + if (tbody) rowObserver.observe(tbody, { childList: true }) + + return () => { + scrollEl.removeEventListener('scroll', schedule) + scrollEl.removeEventListener('pointermove', handleMove) + scrollEl.removeEventListener('pointerleave', handleLeave) + resizeObserver.disconnect() + rowObserver.disconnect() + if (raf) cancelAnimationFrame(raf) + } + }, [scrollElement, measure]) + + // Re-measure when the remote selections or column layout change (listeners stay + // subscribed). Layout effect so positions update before paint — no one-frame lag as a + // peer moves. NOT keyed on `localSelection`: moving the local caret changes only which + // boxes are `covered`, which the cheap in-memory pass below handles without a reflow. + useLayoutEffect(() => { + measure() + }, [remoteSelections, columnIndexById, measure]) + + // Re-derived in render so it reacts to `localSelection`: when the local selection grows to + // cover the hovered box (its outline is no longer drawn) without another pointer move, the + // floating name tag must drop rather than linger over cells with no visible remote selection. + const hoveredBox = hoveredSocketId + ? boxes.find( + (box) => + box.socketId === hoveredSocketId && + !isSelectionCovered( + box.anchorRow, + box.anchorCol, + box.focusRow, + box.focusCol, + localSelection + ) + ) + : undefined + + return ( + <> +
+ {boxes.map((box) => + // A cell the local user also has selected shows only the local selection — the + // remote box isn't drawn (its `boxes` entry still drives the hover name). The + // border is an inset box-shadow (no layout width, so it never stacks with an + // adjacent cell's border) plus a subtle fill, darker while the peer is editing. + isSelectionCovered( + box.anchorRow, + box.anchorCol, + box.focusRow, + box.focusCol, + localSelection + ) ? null : ( +
+ ) + )} +
+ {/* The name label portals to the body so it floats on top of the grid (and its + sticky header) instead of being clipped by the overlay's overflow-hidden; it's + placed in viewport space, its bottom-left tabbed onto the selection's top-left. */} + {hoveredBox && + createPortal( + // Same chrome as the workflow-canvas presence label (see cursors.tsx): the + // identity color is the only per-user value; text color, font, radius, and + // padding all reuse the canvas tokens so tables + canvas presence look identical. + // `rounded-bl-none` tabs the label's bottom-left corner onto the selection's + // top-left, the one deviation from the free-floating canvas cursor tag. +
+ {hoveredBox.userName} +
, + document.body + )} + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index ed93cdaca87..ac28796895b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -5,6 +5,7 @@ import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } fr import { cn, toast, useToast } from '@sim/emcn' import { Loader, TableX } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' +import type { TableCellSelection } from '@sim/realtime-protocol/table-presence' import { getErrorMessage } from '@sim/utils/errors' import { useVirtualizer } from '@tanstack/react-virtual' import { useParams } from 'next/navigation' @@ -22,6 +23,7 @@ import type { import { getColumnId } from '@/lib/table/column-keys' import { TABLE_LIMITS } from '@/lib/table/constants' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' +import type { RemoteTableSelection } from '@/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room' import type { BlockedTableAction } from '@/app/workspace/[workspaceId]/tables/[tableId]/lock-copy' import { useTimezone } from '@/hooks/queries/general-settings' import { @@ -57,6 +59,7 @@ import { ExpandedCellPopover } from './cells' import { ADD_COL_WIDTH, COL_WIDTH, SELECTION_TINT_BG } from './constants' import { DataRow } from './data-row' import { ColumnHeaderMenu, WorkflowGroupMetaCell } from './headers' +import { RemoteSelectionOverlay } from './remote-selection-overlay' import { TableFind } from './table-find' import { AddRowButton, SelectAllCheckbox, TableColGroup } from './table-primitives' import type { DisplayColumn } from './types' @@ -69,6 +72,7 @@ import { computeNormalizedSelection, type ExecStatusMix, expandToDisplayColumns, + isCellInSelection, moveCell, ROW_SELECTION_ALL, ROW_SELECTION_NONE, @@ -153,6 +157,10 @@ interface TableGridProps { workspaceId?: string tableId?: string embedded?: boolean + /** Remote collaborators' cell selections, rendered as presence overlays. */ + remoteSelections: RemoteTableSelection[] + /** Broadcast the local viewer's cell selection to the table presence room. */ + emitCellSelection: (cell: TableCellSelection) => void /** The table's mutation locks; gates row/schema affordances alongside `canEdit`. */ locks?: TableLocks /** @@ -369,6 +377,8 @@ export function TableGrid({ workspaceId: propWorkspaceId, tableId: propTableId, embedded, + remoteSelections, + emitCellSelection, locks, onBlockedAction, sidebarReservedPx, @@ -436,6 +446,13 @@ export function TableGrid({ const columnWidthsRef = useRef(columnWidths) columnWidthsRef.current = columnWidths const [resizingColumn, setResizingColumn] = useState(null) + const resizingColumnRef = useRef(resizingColumn) + resizingColumnRef.current = resizingColumn + /** True from a committed local width change until its metadata PUT settles. Keeps a + * concurrent peer-triggered definition refetch (the value-less `metadata` event forces a + * refetch that can carry the not-yet-persisted server widths) from momentarily reverting + * the just-written widths. */ + const pendingWidthWriteRef = useRef(false) const [columnOrder, setColumnOrder] = useState(null) const columnOrderRef = useRef(columnOrder) columnOrderRef.current = columnOrder @@ -801,6 +818,24 @@ export function TableGrid({ return expandToDisplayColumns(ordered, tableWorkflowGroups) }, [columns, columnOrder, hiddenColumns, tableWorkflowGroups]) + /** Column id → its rendered index (matches the cells' `data-col`), for placing overlays. + * Only built when collaborators are present (the overlay it feeds is gated on that too), + * so solo editing never pays the map build. */ + const columnIndexById = useMemo(() => { + const map = new Map() + if (remoteSelections.length > 0) displayColumns.forEach((col, index) => map.set(col.key, index)) + return map + }, [displayColumns, remoteSelections.length]) + + /** Row id → its index in the current row list, for testing local-selection coverage. + * Only built when collaborators are present (the overlay is gated on that too), so + * solo editing never pays the O(n) map build on a refetch. */ + const rowIndexById = useMemo(() => { + const map = new Map() + if (remoteSelections.length > 0) rows.forEach((row, index) => map.set(row.id, index)) + return map + }, [rows, remoteSelections.length]) + const workflowGroupById = useMemo( () => new Map(tableWorkflowGroups.map((g) => [g.id, g])), [tableWorkflowGroups] @@ -975,6 +1010,30 @@ export function TableGrid({ ? (rowsRef.current[selectionFocus.rowIndex]?.id ?? null) : null + // Broadcast the local viewer's cell selection to the presence room. Resolves the + // index-based selection to stable (rowId, columnId), re-running on `rows`/`displayColumns` + // too so a peer's row insert/delete/reorder re-broadcasts the shifted id under the same + // index (the emitter dedups an unchanged result). `editing` marks the active cell so + // peers darken it (the "someone is typing here" signal). + useEffect(() => { + const resolve = (coord: CellCoord | null) => { + if (!coord) return null + const rowId = rows[coord.rowIndex]?.id + const columnId = displayColumns[coord.colIndex]?.key + return rowId && columnId ? { rowId, columnId } : null + } + const anchor = resolve(selectionAnchor) + if (!anchor) { + emitCellSelection(null) + return + } + // A single-cell click leaves `selectionFocus` null; the grid treats that as a + // one-cell selection at the anchor (`focus ?? anchor`). Mirror that — otherwise the + // most common selection would never broadcast and would clear the prior outline. + const focus = resolve(selectionFocus) ?? anchor + emitCellSelection({ anchor, focus, editing: editingCell !== null }) + }, [selectionAnchor, selectionFocus, editingCell, rows, displayColumns, emitCellSelection]) + const { data: findData, isFetching: isFindFetching } = useFindTableRows({ workspaceId, tableId, @@ -1345,12 +1404,7 @@ export function TableGrid({ selectionAnchorRef.current, selectionFocusRef.current ) - const isWithinSelection = - sel !== null && - rowIndex >= sel.startRow && - rowIndex <= sel.endRow && - colIndex >= sel.startCol && - colIndex <= sel.endCol + const isWithinSelection = sel !== null && isCellInSelection(rowIndex, colIndex, sel) if (!isWithinSelection) { setSelectionAnchor({ rowIndex, colIndex }) @@ -1567,7 +1621,11 @@ export function TableGrid({ const handleColumnResizeEnd = useCallback(() => { setResizingColumn(null) - updateMetadataRef.current({ columnWidths: columnWidthsRef.current }) + pendingWidthWriteRef.current = true + updateMetadataRef.current( + { columnWidths: columnWidthsRef.current }, + { onSettled: () => (pendingWidthWriteRef.current = false) } + ) }, []) const handleColumnAutoResize = useCallback((columnKey: string) => { @@ -1628,7 +1686,11 @@ export function TableGrid({ setColumnWidths((prev) => ({ ...prev, [columnKey]: newWidth })) const updated = { ...columnWidthsRef.current, [columnKey]: newWidth } columnWidthsRef.current = updated - updateMetadataRef.current({ columnWidths: updated }) + pendingWidthWriteRef.current = true + updateMetadataRef.current( + { columnWidths: updated }, + { onSettled: () => (pendingWidthWriteRef.current = false) } + ) }, []) const handleColumnDragStart = useCallback((columnName: string) => { @@ -1933,28 +1995,45 @@ export function TableGrid({ return } if (!source) return - // After first load: only re-seed `columnOrder` when the *set of columns* - // changes (e.g. a workflow group adds/removes outputs server-side). Pure - // reorders are left alone so an in-flight optimistic drag isn't clobbered - // by a refetch returning the pre-drag order. + // After first load a collaborator (or our own committed edit) reshaped the layout. + // Re-apply it live from the active layout source (a view's config when one is active, + // else the table's own metadata), but never clobber the gesture the local user is + // mid-way through — their in-progress value leads the server's. Each field is guarded + // by reference: React Query structural sharing keeps an unchanged sub-object + // referentially stable, so an unrelated change (e.g. a peer's pin) doesn't re-apply + // widths/order. + // Width: keep the column being actively resized on its live local value. + const serverWidths = source.columnWidths + if (serverWidths && serverWidths !== columnWidthsRef.current) { + const resizing = resizingColumnRef.current + const localWidth = resizing ? columnWidthsRef.current[resizing] : undefined + if (resizing && localWidth !== undefined) { + setColumnWidths({ ...serverWidths, [resizing]: localWidth }) + } else if (!pendingWidthWriteRef.current) { + setColumnWidths(serverWidths) + } + // else: a just-committed local width write is still in flight — local leads until + // its onSettled invalidation brings back the server's committed (merged) widths. + } + // Pins toggle instantly (no in-progress gesture) — apply on change. + const serverPins = source.pinnedColumns + if (serverPins && serverPins !== pinnedColumnsRef.current) { + setPinnedColumns(serverPins) + } + // Order: apply the server order live (a peer reorder or our own committed edit), + // unless a local column drag is in flight (an optimistic reorder would otherwise be + // reverted to the pre-drag order the refetch returns). Preserve our own just-appended + // ids whose patch is still in flight by appending them — `viewLayout` gets a new + // identity on every save, so a refetch/view-save predating the append must not drop + // them; `displayColumns` harmlessly skips any id with no matching column. const serverOrder = source.columnOrder - if (serverOrder) { + if (serverOrder && serverOrder !== columnOrderRef.current && !dragColumnNameRef.current) { const localOrder = columnOrderRef.current if (!localOrder) { setColumnOrder(serverOrder) } else { - // Re-seed only when the server knows an id the local order lacks — a real - // schema change (a workflow group gained outputs). Ids present locally but - // NOT on the server are our own just-appended columns whose patch is still - // in flight: `viewLayout` gets a new identity on every save, so a refetch - // carrying the pre-append order would otherwise roll them back, and the - // append effect can't re-fire because `columns` is unchanged. Ids the - // server drops stay in the local order harmlessly — `displayColumns` - // skips any id with no matching column. - const localSet = new Set(localOrder) - if (serverOrder.some((id) => !localSet.has(id))) { - setColumnOrder(serverOrder) - } + const localOnly = localOrder.filter((id) => !serverOrder.includes(id)) + setColumnOrder(localOnly.length > 0 ? [...serverOrder, ...localOnly] : serverOrder) } } }, [tableData?.metadata, viewLayout, viewLayoutKey]) @@ -4260,6 +4339,15 @@ export function TableGrid({ })()} + {remoteSelections.length > 0 && ( + + )} {resizingColumn && (
= sel.startRow && + row <= sel.endRow && + col >= sel.startCol && + col <= sel.endCol + ) +} + /** A run of consecutive `displayColumns` rendered together in the meta header row. */ export type HeaderGroup = | { kind: 'plain'; size: 1; startColIndex: number } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/index.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/index.ts index 76addf44c49..190e846254b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/index.ts @@ -1,3 +1,4 @@ export * from './use-context-menu' export * from './use-table' export * from './use-table-event-stream' +export * from './use-table-room' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts index 67f8a2b54c6..0d4b46f1fb8 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts @@ -379,14 +379,32 @@ export function useTableEventStream({ else if (entry.event?.kind === 'dispatch') applyDispatch(entry.event) else if (entry.event?.kind === 'job') applyJob(entry.event) else if (entry.event?.kind === 'usageLimitReached') applyUsageLimit(entry.event) + // A collaborator's manual edit: refetch rows (debounced) so the winning + // last-write value shows live, in this client's own wire format. + else if (entry.event?.kind === 'edit') scheduleRowsInvalidate() + // A collaborator changed the table structure: mirror the local + // invalidateTableSchema set — the definition (exact, so rows stay on the + // debounce), the run-state + enrichment sibling queries under detail (a group + // delete/restructure can otherwise leave a stale running badge or enrichment + // panel), the tables list (column/row counts), and the debounced rows. + else if (entry.event?.kind === 'schema') { + void queryClient.invalidateQueries({ queryKey: tableKeys.detail(tableId), exact: true }) + void queryClient.invalidateQueries({ queryKey: tableKeys.activeDispatches(tableId) }) + void queryClient.invalidateQueries({ queryKey: tableKeys.enrichmentDetails(tableId) }) + void queryClient.invalidateQueries({ queryKey: tableKeys.lists() }) + scheduleRowsInvalidate() + } + // A collaborator changed the column layout (width/pin/order): refetch the + // definition alone (it carries the metadata) — the grid re-applies it without + // a rows refetch. Exact, so rows/run-state stay put. + else if (entry.event?.kind === 'metadata') { + void queryClient.invalidateQueries({ queryKey: tableKeys.detail(tableId), exact: true }) + } + // A collaborator toggled a table lock: re-read the definition so every open + // viewer's gating updates. `exact` avoids refetching every rows page + // (rowsRoot nests under detail); no row data changed. else if (entry.event?.kind === 'definition') { - // A lock/schema change on the definition — re-read it so every open - // viewer's gating updates. `exact` avoids refetching every rows page - // (rowsRoot nests under detail); no row data changed. - void queryClient.invalidateQueries({ - queryKey: tableKeys.detail(tableId), - exact: true, - }) + void queryClient.invalidateQueries({ queryKey: tableKeys.detail(tableId), exact: true }) } } catch (err) { logger.warn('Failed to parse table event', { tableId, err }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts new file mode 100644 index 00000000000..df950dab585 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-room.ts @@ -0,0 +1,233 @@ +'use client' + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { createLogger } from '@sim/logger' +import { presenceEventName, ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { + type JoinTableError, + type JoinTableSuccess, + TABLE_PRESENCE_EVENTS, + type TableCellSelection, + type TableCellSelectionBroadcast, + type TablePresenceUser, +} from '@sim/realtime-protocol/table-presence' +import { generateShortId } from '@sim/utils/id' +import type { PresenceAvatarUser } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' +import { useSocket } from '@/app/workspace/providers/socket-provider' + +const logger = createLogger('TableRoom') + +/** Retry cap + base delay for a retryable join failure on an otherwise-live socket. */ +const MAX_JOIN_RETRIES = 3 +const JOIN_RETRY_BASE_MS = 1000 +/** Trailing-throttle window for broadcasting local selection changes (smooths drag-select). */ +const SELECTION_EMIT_THROTTLE_MS = 50 + +/** The `table:presence-update` broadcast name, derived from the room type. */ +const TABLE_PRESENCE_UPDATE_EVENT = presenceEventName(ROOM_TYPES.TABLE) + +/** A remote viewer's current cell selection, ready to render as a presence overlay. */ +export interface RemoteTableSelection { + socketId: string + userId: string + userName: string + cell: NonNullable +} + +interface UseTableRoomResult { + /** Collaborators viewing this table, excluding the current socket (for avatars). */ + otherUsers: PresenceAvatarUser[] + /** Remote viewers that currently have a cell selected (for overlays). */ + remoteSelections: RemoteTableSelection[] + /** Broadcast the local viewer's current cell selection (`null` clears it). Throttled. */ + emitCellSelection: (cell: TableCellSelection) => void +} + +/** + * Joins the table presence room for live collaborator avatars + cell-selection + * highlights. Presence rides the shared socket (`useSocket`); table data is + * unchanged (it flows through the one-way durable event stream). The full roster + * arrives via the presence broadcast (join/leave); individual selection moves + * arrive as lower-latency {@link TABLE_PRESENCE_EVENTS.CELL_SELECTION} deltas that + * patch the matching roster entry. + */ +export function useTableRoom(tableId: string): UseTableRoomResult { + const { socket, currentSocketId } = useSocket() + + const [presenceUsers, setPresenceUsers] = useState([]) + + const tabSessionIdRef = useRef('') + if (!tabSessionIdRef.current) tabSessionIdRef.current = generateShortId() + + // The local viewer's current selection, re-broadcast on (re)join. Emits only fire on a + // selection change, and the server drops a CELL_SELECTION for a socket not yet in the + // room — so a selection made before the join completes (or held across a reconnect) + // would otherwise never reach peers until it next changes. + const currentCellRef = useRef(null) + + useEffect(() => { + if (!socket || !tableId) return + + let retries = 0 + let retryTimer: ReturnType | null = null + + const join = () => { + socket.emit(TABLE_PRESENCE_EVENTS.JOIN, { tableId, tabSessionId: tabSessionIdRef.current }) + } + + // A fresh (re)connect gets a fresh retry budget, so a prior full exhaustion doesn't leave the + // socket unable to retry a failed re-join until the next success. + const handleConnect = () => { + retries = 0 + join() + } + + const handleJoinSuccess = (data: JoinTableSuccess) => { + if (data.tableId !== tableId) return + retries = 0 + if (retryTimer) { + clearTimeout(retryTimer) + retryTimer = null + } + setPresenceUsers(data.presenceUsers ?? []) + // Re-send our current selection now that the room is joined (earlier emits were + // dropped server-side), so peers see it without waiting for the next change. + if (currentCellRef.current) { + socket.emit(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { cell: currentCellRef.current }) + } + } + const handleJoinError = (data: JoinTableError) => { + if (data.tableId !== tableId) return + logger.warn('Failed to join table room', { code: data.code, error: data.error }) + if (data.retryable && retries < MAX_JOIN_RETRIES) { + retries += 1 + // Clear any still-pending retry before scheduling a new one, so reconnect churn can't + // orphan a timer that fires an extra join(). + if (retryTimer) clearTimeout(retryTimer) + retryTimer = setTimeout(join, JOIN_RETRY_BASE_MS * retries) + } + } + const handlePresence = (users: TablePresenceUser[]) => { + // Take membership from the roster snapshot but keep the `cell` we already hold for + // a known socket. Trade-off: a snapshot can lag the lower-latency CELL_SELECTION + // deltas, so a blind replace could revert a fresher selection (the common case). + // The cost is that a *dropped* delta is no longer healed by the next snapshot, only + // by the peer's next delta — fine, since deltas flow continuously during selection + // and a cleared selection also sends `null` via delta. + setPresenceUsers((prev) => { + const cellBySocket = new Map(prev.map((user) => [user.socketId, user.cell])) + return (users ?? []).map((user) => + cellBySocket.has(user.socketId) + ? { ...user, cell: cellBySocket.get(user.socketId) } + : user + ) + }) + } + const handleCellSelection = (data: TableCellSelectionBroadcast) => { + // Patch the matching roster entry's selection. The peer is always already in + // the roster: the server broadcasts their join (→ presence-update) before they + // can select, and Socket.IO preserves that order — so a delta for an unknown + // socket only means a dropped broadcast, which the next presence-update heals. + setPresenceUsers((prev) => + prev.map((user) => (user.socketId === data.socketId ? { ...user, cell: data.cell } : user)) + ) + } + + // Join now if the socket is already connected; `connect` covers (re)connects. + if (socket.connected) join() + socket.on('connect', handleConnect) + socket.on(TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, handleJoinSuccess) + socket.on(TABLE_PRESENCE_EVENTS.JOIN_ERROR, handleJoinError) + socket.on(TABLE_PRESENCE_UPDATE_EVENT, handlePresence) + socket.on(TABLE_PRESENCE_EVENTS.CELL_SELECTION, handleCellSelection) + + return () => { + if (retryTimer) clearTimeout(retryTimer) + socket.off('connect', handleConnect) + socket.off(TABLE_PRESENCE_EVENTS.JOIN_SUCCESS, handleJoinSuccess) + socket.off(TABLE_PRESENCE_EVENTS.JOIN_ERROR, handleJoinError) + socket.off(TABLE_PRESENCE_UPDATE_EVENT, handlePresence) + socket.off(TABLE_PRESENCE_EVENTS.CELL_SELECTION, handleCellSelection) + setPresenceUsers([]) + // Leave scoped to THIS table so a table A→B switch (B joins first, auto-leaving + // A) can't have A's deferred leave evict the fresh B membership. + socket.emit(TABLE_PRESENCE_EVENTS.LEAVE, { tableId }) + } + }, [socket, tableId]) + + const socketRef = useRef(socket) + socketRef.current = socket + /** Presence is disabled when no table id is bound (e.g. the embedded Chat panel surface). */ + const enabledRef = useRef(false) + enabledRef.current = Boolean(tableId) + const lastEmitRef = useRef(0) + const trailingTimerRef = useRef | null>(null) + const pendingCellRef = useRef(null) + /** Key of the last selection sent, to skip re-emitting an unchanged selection. */ + const lastSentKeyRef = useRef(null) + + // Reset the throttle when the table changes (or on unmount): a pending selection for + // the table we're leaving must not flush into the next table's room after a switch. + useEffect( + () => () => { + if (trailingTimerRef.current) { + clearTimeout(trailingTimerRef.current) + trailingTimerRef.current = null + } + pendingCellRef.current = null + currentCellRef.current = null + lastSentKeyRef.current = null + lastEmitRef.current = 0 + }, + [tableId] + ) + + const emitCellSelection = useCallback((cell: TableCellSelection) => { + // No room joined (empty tableId, e.g. embedded mode) — never broadcast; the server + // would drop it anyway. Local selection UI is unaffected (grid-owned state). + if (!enabledRef.current) return + // Skip re-emitting an unchanged selection: the caller re-resolves on every data + // refetch (so a peer's row insert re-broadcasts the shifted rowId), but most refetches + // don't move the selection — dedup those, and the no-selection state on table open. + const key = cell === null ? 'null' : JSON.stringify(cell) + if (key === lastSentKeyRef.current) return + lastSentKeyRef.current = key + currentCellRef.current = cell + pendingCellRef.current = cell + const flush = () => { + lastEmitRef.current = Date.now() + trailingTimerRef.current = null + socketRef.current?.emit(TABLE_PRESENCE_EVENTS.CELL_SELECTION, { + cell: pendingCellRef.current, + }) + } + const elapsed = Date.now() - lastEmitRef.current + if (elapsed >= SELECTION_EMIT_THROTTLE_MS) { + flush() + } else if (!trailingTimerRef.current) { + trailingTimerRef.current = setTimeout(flush, SELECTION_EMIT_THROTTLE_MS - elapsed) + } + }, []) + + const otherUsers = useMemo( + () => presenceUsers.filter((user) => user.socketId !== currentSocketId), + [presenceUsers, currentSocketId] + ) + const remoteSelections = useMemo( + () => + otherUsers + .filter( + (user): user is TablePresenceUser & { cell: NonNullable } => + user.cell != null + ) + .map((user) => ({ + socketId: user.socketId, + userId: user.userId, + userName: user.userName, + cell: user.cell, + })), + [otherUsers] + ) + + return { otherUsers, remoteSelections, emitCellSelection } +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index 0540b5346b5..4ff7ca9cc19 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -28,6 +28,7 @@ import { Resource, type SortConfig, } from '@/app/workspace/[workspaceId]/components' +import { PresenceAvatars } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' import { LogDetails } from '@/app/workspace/[workspaceId]/logs/components' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { ImportCsvDialog } from '@/app/workspace/[workspaceId]/tables/components/import-csv-dialog' @@ -72,7 +73,7 @@ import { } from './components' import { COLUMN_SIDEBAR_WIDTH } from './components/table-grid/constants' import { COLUMN_TYPE_ICONS } from './components/table-grid/headers' -import { useTable, useTableEventStream } from './hooks' +import { useTable, useTableEventStream, useTableRoom } from './hooks' import { type BlockedTableAction, describeBlockedAction, lockedNouns } from './lock-copy' import { ALL_VIEW_PARAM, @@ -230,6 +231,12 @@ export function Table({ } useTableEventStream({ tableId, workspaceId, onUsageLimitReached }) + // Live table presence (cell-selection carets + avatars). Runs in both modes so the + // mothership chat panel shows collaborators' live selections too. The avatar stack lives + // only in the `!embedded` Resource.Header, so the embedded panel gets carets without avatars + // for free — matching the panel's own-chrome layout. + const { otherUsers: presenceUsers, remoteSelections, emitCellSelection } = useTableRoom(tableId) + const [slideout, dispatch] = useReducer(slideoutReducer, { kind: 'none' }) const [showDeleteTableConfirm, setShowDeleteTableConfirm] = useState(false) const [showLockSettings, setShowLockSettings] = useState(false) @@ -1324,6 +1331,9 @@ export function Table({ breadcrumbs={breadcrumbs} aside={
+ {presenceUsers.length > 0 && ( + + )} {selection.totalRunning > 0 || selection.hasActiveDispatch ? ( { + queryClient.invalidateQueries({ queryKey: tableKeys.lists() }) + queryClient.invalidateQueries({ queryKey: folderKeys.resource('table') }) + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx b/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx index 94eecb940bd..81a44336bbd 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx @@ -48,6 +48,7 @@ import { TablesListContextMenu, } from '@/app/workspace/[workspaceId]/tables/components' import { TableContextMenu } from '@/app/workspace/[workspaceId]/tables/components/table-context-menu' +import { useWorkspaceTablesRoom } from '@/app/workspace/[workspaceId]/tables/hooks/use-workspace-tables-room' import { tablesParsers, tablesSortParams, @@ -112,6 +113,10 @@ export function Tables() { const userPermissions = useUserPermissionsContext() const canEdit = userPermissions.canEdit === true + // Joined for the live tables list: a `workspace-tables-changed` broadcast (fanned out by the table + // mutation service) invalidates the list so this view refetches without waiting for staleness. + useWorkspaceTablesRoom(workspaceId) + const { data: tables = EMPTY_TABLES, error } = useTablesList(workspaceId) const { data: members } = useWorkspaceMembersQuery(workspaceId) const pinnedTableIds = usePinnedIds(workspaceId, 'table') diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/workflow-item/avatars/avatars.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/workflow-item/avatars/avatars.tsx index 8626149a174..19b29832c61 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/workflow-item/avatars/avatars.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/workflow-item/avatars/avatars.tsx @@ -1,8 +1,7 @@ 'use client' -import { type CSSProperties, useMemo } from 'react' -import { Avatar, AvatarFallback, AvatarImage, Tooltip } from '@sim/emcn' -import { getUserColor } from '@/lib/workspaces/colors' +import { useMemo } from 'react' +import { PresenceAvatars } from '@/app/workspace/[workspaceId]/components/presence/presence-avatars' import { useSocket } from '@/app/workspace/providers/socket-provider' import { SIDEBAR_WIDTH } from '@/stores/constants' import { usePresenceStore } from '@/stores/presence/store' @@ -21,64 +20,11 @@ interface AvatarsProps { workflowId: string } -interface PresenceUser { - socketId: string - userId: string - userName?: string - avatarUrl?: string | null -} - -interface UserAvatarProps { - user: PresenceUser - index: number -} - /** - * Individual user avatar using emcn Avatar component. - * Falls back to colored circle with initials if image fails to load. - */ -function UserAvatar({ user, index }: UserAvatarProps) { - const color = getUserColor(user.userId) - const initials = user.userName ? user.userName.charAt(0).toUpperCase() : '?' - - const avatarElement = ( - - {user.avatarUrl && ( - - )} - - {initials} - - - ) - - if (user.userName) { - return ( - - {avatarElement} - - {user.userName} - - - ) - } - - return avatarElement -} - -/** - * Displays user avatars for presence in a workflow item. - * Only shows avatars for the currently active workflow. - * - * @param props - Component props - * @returns Avatar stack for workflow presence + * Presence avatars for a workflow sidebar item. Owns the workflow-specific + * concerns — the presence source, the current-workflow filter, and the + * sidebar-width-driven visible count — and delegates the stack rendering to the + * shared {@link PresenceAvatars}, so it looks identical to every other surface. */ export function Avatars({ workflowId }: AvatarsProps) { const { currentWorkflowId, currentSocketId } = useSocket() @@ -86,8 +32,8 @@ export function Avatars({ workflowId }: AvatarsProps) { const sidebarWidth = useSidebarStore((state) => state.sidebarWidth) /** - * Calculate max visible avatars based on sidebar width. - * Scales between MIN_COUNT and MAX_COUNT as sidebar expands. + * Scale the max visible avatars between MIN_COUNT and MAX_COUNT as the sidebar + * widens. */ const maxVisible = useMemo(() => { const widthDelta = sidebarWidth - SIDEBAR_WIDTH.MIN @@ -97,56 +43,13 @@ export function Avatars({ workflowId }: AvatarsProps) { }, [sidebarWidth]) /** - * Only show presence for the currently active workflow. - * Filter out the current socket connection (allows same user's other tabs to appear). + * Only show presence for the currently active workflow, excluding the current + * socket (so the user's own other tabs still appear). */ const workflowUsers = useMemo(() => { - if (currentWorkflowId !== workflowId) { - return [] - } + if (currentWorkflowId !== workflowId) return [] return presenceUsers.filter((user) => user.socketId !== currentSocketId) }, [presenceUsers, currentWorkflowId, workflowId, currentSocketId]) - /** - * Calculate visible users and overflow count. - * Shows up to maxVisible avatars, with overflow indicator for any remaining. - * Users are reversed so new avatars appear on the left (keeping right side stable). - */ - const { visibleUsers, overflowCount } = useMemo(() => { - if (workflowUsers.length === 0) { - return { visibleUsers: [], overflowCount: 0 } - } - - const visible = workflowUsers.slice(0, maxVisible) - const overflow = Math.max(0, workflowUsers.length - maxVisible) - - // Reverse so rightmost avatars stay stable as new ones are revealed on the left - return { visibleUsers: [...visible].reverse(), overflowCount: overflow } - }, [workflowUsers, maxVisible]) - - if (visibleUsers.length === 0) { - return null - } - - return ( -
- {overflowCount > 0 && ( - - - - - +{overflowCount} - - - - - {overflowCount} more user{overflowCount > 1 ? 's' : ''} - - - )} - {visibleUsers.map((user, index) => ( - - ))} -
- ) + return } diff --git a/apps/sim/hooks/queries/workspace-file-folders.ts b/apps/sim/hooks/queries/workspace-file-folders.ts index 8b8cb87fe6e..40010ab03c8 100644 --- a/apps/sim/hooks/queries/workspace-file-folders.ts +++ b/apps/sim/hooks/queries/workspace-file-folders.ts @@ -40,7 +40,7 @@ async function fetchWorkspaceFileFolders( return data.folders } -function invalidateWorkspaceFileBrowsers( +export function invalidateWorkspaceFileBrowsers( queryClient: ReturnType, workspaceId: string ) { diff --git a/apps/sim/lib/api/contracts/file-doc.ts b/apps/sim/lib/api/contracts/file-doc.ts new file mode 100644 index 00000000000..a46bce7f5d3 --- /dev/null +++ b/apps/sim/lib/api/contracts/file-doc.ts @@ -0,0 +1,119 @@ +import { z } from 'zod' +import { defineRouteContract } from '@/lib/api/contracts/types' + +export const buildFileDocSeedBodySchema = z.object({ + workspaceId: z.string().min(1, 'workspaceId is required'), + fileId: z.string().min(1, 'fileId is required'), +}) +export type BuildFileDocSeedBody = z.input + +export const buildFileDocSeedResponseSchema = z.object({ + /** + * The base64-encoded Yjs update that seeds the file's collaborative document, or `null` when the + * file is missing/deleted (the caller treats that as an empty document). + */ + update: z.string().nullable(), +}) +export type BuildFileDocSeedResponse = z.output + +/** + * Internal, `x-api-key`-gated: the realtime relay asks the app to build a server-authoritative seed + * for a file's collaborative document (markdown → Yjs, through the exact client engine). Only the app + * can do this — it owns the editor extensions + blob access — so realtime delegates instead of + * embedding a second markdown/ProseMirror stack. + */ +export const buildFileDocSeedContract = defineRouteContract({ + method: 'POST', + path: '/api/internal/file-doc/seed', + body: buildFileDocSeedBodySchema, + response: { + mode: 'json', + schema: buildFileDocSeedResponseSchema, + }, +}) + +export const mergeFileDocBodySchema = z.object({ + /** For logging/observability only — the merge itself is stateless. */ + fileId: z.string().min(1, 'fileId is required'), + /** + * Base64-encoded `Y.encodeStateAsUpdate` of the live document as the relay currently holds it. The + * bound is generous headroom over a max-size collab doc's Yjs state (base64-inflated), not a tuned + * limit — collab is gated to ≤256 KB documents client-side. + */ + docState: z + .string() + .min(1, 'docState is required') + .max(16 * 1024 * 1024, 'docState is too large'), + /** The full markdown body the caller (e.g. copilot) wants the document to become. */ + markdown: z.string().max(8 * 1024 * 1024, 'markdown is too large'), +}) +export type MergeFileDocBody = z.input + +export const mergeFileDocResponseSchema = z.object({ + /** + * Base64-encoded Yjs update — the minimal CRDT diff that transforms the supplied document state + * into `markdown`. The relay applies it to the live doc, which merges it with any concurrent user + * edits and relays it to every connected editor. Empty (a no-op update) when nothing changed. + */ + update: z.string(), +}) +export type MergeFileDocResponse = z.output + +/** + * Internal, `x-api-key`-gated: the realtime relay asks the app to merge new markdown into a live + * collaborative document as a minimal CRDT diff (Stage C — copilot writing into an open doc). The app + * owns the conversion engine; the relay owns the doc — so the relay ships the current doc state here, + * gets back the diff, and applies + relays it. Stateless (no blob/DB access): the caller supplies both + * the current state and the target markdown. + */ +export const mergeFileDocContract = defineRouteContract({ + method: 'POST', + path: '/api/internal/file-doc/merge', + body: mergeFileDocBodySchema, + response: { + mode: 'json', + schema: mergeFileDocResponseSchema, + }, +}) + +export const persistFileDocBodySchema = z.object({ + workspaceId: z.string().min(1, 'workspaceId is required'), + fileId: z.string().min(1, 'fileId is required'), + /** Attribution only (blob metadata) — the last collaborator to touch the live doc. */ + userId: z.string().min(1, 'userId is required'), + /** + * Base64-encoded `Y.encodeStateAsUpdate` of the live document as the relay currently holds it — the + * bound matches the merge contract (generous base64 headroom over a ≤256 KB collab doc's Yjs state). + */ + docState: z + .string() + .min(1, 'docState is required') + .max(16 * 1024 * 1024, 'docState is too large'), +}) +export type PersistFileDocBody = z.input + +export const persistFileDocResponseSchema = z.object({ + /** + * `true` when the document was projected to markdown and written durably to the file; `false` when + * the file was missing/deleted (nothing to write). A transport/conversion failure is a non-2xx. + */ + persisted: z.boolean(), +}) +export type PersistFileDocResponse = z.output + +/** + * Internal, `x-api-key`-gated: the realtime relay asks the app to project a live collaborative + * document back to durable markdown (Yjs → markdown, through the exact editor engine) and write it to + * the file. The relay owns the live doc but not the conversion engine or blob/DB access, so it ships + * the current doc state here. Called debounced during editing and when the last collaborator leaves — + * the server-authoritative durable path that replaces the editor's client-side autosave. + */ +export const persistFileDocContract = defineRouteContract({ + method: 'POST', + path: '/api/internal/file-doc/persist', + body: persistFileDocBodySchema, + response: { + mode: 'json', + schema: persistFileDocResponseSchema, + }, +}) diff --git a/apps/sim/lib/collab-doc/README.md b/apps/sim/lib/collab-doc/README.md new file mode 100644 index 00000000000..4613bb9c3b7 --- /dev/null +++ b/apps/sim/lib/collab-doc/README.md @@ -0,0 +1,53 @@ +# `@/lib/collab-doc` — server-side collaborative-document conversion + +Server-side conversion between a file's **markdown** (the durable source of truth) and its +collaborative **Yjs document**, so the server can own the doc: seed it, project it back to +markdown, and let the agent write into it while a user is typing. + +## Why this exists + +Collaborative file editing had two writers with no shared CRDT: copilot `edit_content` wrote +markdown straight to the file while the user typed into an ephemeral, client-seeded Yjs doc. They +couldn't reconcile — the agent's edit didn't stream into the editor, and last-writer clobbered. The +fix is a **server-authoritative Yjs doc** both sides write into, with markdown as a projection. + +## What Stage A (this module) provides + +| Function | Purpose | +|---|---| +| `markdownToYDoc(md)` | Cold-start seed: file markdown → a fresh `Y.Doc`. | +| `yDocToMarkdown(ydoc)` | Projection: `Y.Doc` → the file's canonical markdown. | +| `applyMarkdownToYDoc(ydoc, md)` | Agent write: merge new content into a live `Y.Doc` as a minimal CRDT diff (no clobber). | + +### Design decisions (why it's not hacky) + +- **Parity by construction.** The markdown↔ProseMirror step reuses the *exact* client engine + (`parseMarkdownToDoc` / `serializeDocToMarkdown`, `@tiptap/markdown` on the shared extension set) — + not a second markdown implementation — so the server can never diverge from what the editor + renders. The custom-fidelity constructs (tables, footnotes, raw HTML, `sim:` mentions) are covered + by the same code that covers them in the browser; the round-trip test asserts equivalence. +- **Same Yjs binding as the browser.** ProseMirror↔Yjs uses `@tiptap/y-tiptap` (what TipTap's + Collaboration extension uses), pinned to the same version and sharing the same `prosemirror-model` + / `yjs` instances (peer deps) — so the structure the server produces is byte-compatible with the + client, targeting the same `'default'` fragment. +- **Merge, not replace.** `applyMarkdownToYDoc` uses `updateYFragment` (the primitive `ySyncPlugin` + runs on every keystroke) to apply only the diff, so Yjs reconciles the agent's write with in-flight + remote edits. The test proves an agent write and a concurrent remote edit both survive. +- **Server-only, DOM via jsdom.** The markdown engine builds a (never-mounted) TipTap editor that + needs a DOM; on the server it's backed by a single lazily-created `jsdom` window. Lazy-required so + the client bundle never pulls jsdom in. + +## Server-authoritative seeding (shipped alongside this module) + +The realtime relay seeds each room's document from this module over an internal endpoint +(`buildFileDocSeed` → `POST /api/internal/file-doc/seed` → `ensureServerSeed`), which let the entire +client-seeder subsystem (election / deadlines / `triedSeeders` / `MAX_SEED_ROUNDS` / the +`SEED_REQUEST` handshake) be deleted. The client's connect-deadline offline fallback is deliberately +**kept** — it is unrelated to seeding. No feature flag: the cutover is all-at-once. + +## Remaining stages (future PRs) + +- **Durable persistence.** A DB column for the Yjs binary + debounced snapshotting, so a document + survives with no collaborators connected instead of being re-seeded from markdown on cold open. +- **Copilot into the doc + projection.** `edit_content` calls `applyMarkdownToYDoc` when a doc is + live; a debounced `yDocToMarkdown` projection keeps the file's markdown current. diff --git a/apps/sim/lib/collab-doc/collab-state.ts b/apps/sim/lib/collab-doc/collab-state.ts new file mode 100644 index 00000000000..97828bece51 --- /dev/null +++ b/apps/sim/lib/collab-doc/collab-state.ts @@ -0,0 +1,61 @@ +import { createHash } from 'crypto' +import { db } from '@sim/db' +import { workspaceFileCollabState } from '@sim/db/schema' +import { eq } from 'drizzle-orm' + +/** + * The cached-collab-state cache (`workspace_file_collab_state`) lets a cold room open load the file's + * last-persisted Yjs binary directly instead of re-converting markdown → Yjs on every open — the + * Hocuspocus load-document pattern. See {@link workspaceFileCollabState} for the full rationale. + */ + +/** sha256 (hex) of a markdown buffer — the freshness tag matching a cached doc state to the live file. */ +export function hashMarkdown(markdown: Buffer): string { + return createHash('sha256').update(markdown).digest('hex') +} + +/** + * Load a file's cached Yjs binary IF it is still fresh — i.e. was derived from markdown whose hash + * matches `sourceHash` (the current file's markdown). Returns the binary to apply directly, or `null` + * when there is no cache or it is stale (the markdown changed externally since it was saved), in which + * case the caller re-converts from markdown. Applying the stored binary — rather than rebuilding a Y.Doc + * from markdown — is what preserves the CRDT's client ids and prevents duplicated content on reconnect. + */ +export async function loadFreshCollabDocState( + fileId: string, + sourceHash: string +): Promise { + const [row] = await db + .select({ + docState: workspaceFileCollabState.docState, + sourceHash: workspaceFileCollabState.sourceHash, + }) + .from(workspaceFileCollabState) + .where(eq(workspaceFileCollabState.fileId, fileId)) + .limit(1) + + if (!row || row.sourceHash !== sourceHash) return null + return new Uint8Array(row.docState) +} + +/** + * Persist a collaborative doc's Yjs binary as the file's cold-start state, tagged with the hash of the + * markdown it was derived from. Upsert — one row per file. Called from the server-side persist right + * after the markdown is written, so the cached binary and its `sourceHash` are always consistent with + * the file that was just saved. + */ +export async function saveCollabDocState( + fileId: string, + docState: Uint8Array, + sourceHash: string +): Promise { + const state = Buffer.from(docState) + const updatedAt = new Date() + await db + .insert(workspaceFileCollabState) + .values({ fileId, docState: state, sourceHash, updatedAt }) + .onConflictDoUpdate({ + target: workspaceFileCollabState.fileId, + set: { docState: state, sourceHash, updatedAt }, + }) +} diff --git a/apps/sim/lib/collab-doc/converter.test.ts b/apps/sim/lib/collab-doc/converter.test.ts new file mode 100644 index 00000000000..ba5606600b6 --- /dev/null +++ b/apps/sim/lib/collab-doc/converter.test.ts @@ -0,0 +1,108 @@ +/** + * @vitest-environment jsdom + */ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import { describe, expect, it } from 'vitest' +import * as Y from 'yjs' +import { + applyFrontmatter, + postProcessSerializedMarkdown, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { serializeMarkdownBody } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' +import { + applyMarkdownToYDoc, + markdownToYDoc, + yDocToFileMarkdown, + yDocToMarkdown, +} from './converter' + +/** Representative markdown covering the custom-fidelity constructs (tables, code, lists, marks). */ +const SAMPLES = [ + '# Title\n\nA paragraph with **bold**, *italic*, and `code`.', + '- one\n- two\n - nested\n\n1. first\n2. second', + '> a quote\n\n```ts\nconst x = 1\n```', + '| a | b |\n| --- | --- |\n| 1 | 2 |\n| pipe \\| here | y |', + 'Text with a [link](https://example.com) and a break.\n\nSecond paragraph.', + // Custom-fidelity constructs — the exact cases a second markdown engine would diverge on, and + // that only survive because the server reuses the client's own @tiptap/markdown engine. + 'A footnote reference[^1].\n\n[^1]: the footnote body.', + 'Before.\n\n
untouched raw html
\n\nAfter.', + '- [ ] todo\n- [x] done', +] + +describe('collab-doc converter', () => { + it('round-trips markdown through the Yjs doc identically to the client engine', () => { + for (const md of SAMPLES) { + // yDocToMarkdown(markdownToYDoc(md)) must equal the client's own canonical serialization — + // both go through the exact same @tiptap/markdown engine, so the Yjs hop must be lossless. + expect(yDocToMarkdown(markdownToYDoc(md))).toBe(serializeMarkdownBody(md)) + } + }) + + it('projects an empty doc without throwing', () => { + expect(yDocToMarkdown(markdownToYDoc(''))).toBe(serializeMarkdownBody('')) + }) + + it('applies new content into an existing doc (agent write)', () => { + const ydoc = markdownToYDoc('# Hello\n\nWorld.') + applyMarkdownToYDoc(ydoc, '# Hello\n\nWorld and then some more.') + expect(yDocToMarkdown(ydoc)).toBe(serializeMarkdownBody('# Hello\n\nWorld and then some more.')) + }) + + it('clears an existing doc to empty without throwing (agent write of empty content)', () => { + const ydoc = markdownToYDoc('# Hello\n\nWorld.') + expect(() => applyMarkdownToYDoc(ydoc, '')).not.toThrow() + expect(yDocToMarkdown(ydoc)).toBe(serializeMarkdownBody('')) + }) + + it('merges an agent write with a concurrent remote edit (CRDT, no clobber)', () => { + // Two clients start from the same state. + const server = markdownToYDoc('# Doc\n\nAlpha paragraph.\n\nBeta paragraph.') + const remote = new Y.Doc() + Y.applyUpdate(remote, Y.encodeStateAsUpdate(server)) + + // The remote user edits the FIRST paragraph's text directly on the shared type, concurrently… + const remoteFrag = remote.getXmlFragment('default') + remote.transact(() => { + // second child (index 1) is the first paragraph; append " EDITED" to its text node. + const firstPara = remoteFrag.get(1) as Y.XmlElement + const textNode = firstPara.get(0) as Y.XmlText + textNode.insert(textNode.toString().length, ' EDITED') + }) + + // …while the agent rewrites the SECOND paragraph via the converter on the server doc. + applyMarkdownToYDoc( + server, + '# Doc\n\nAlpha paragraph.\n\nBeta paragraph, expanded by the agent.' + ) + + // Exchange updates both ways (as the relay would) and both edits must survive the merge. + Y.applyUpdate(server, Y.encodeStateAsUpdate(remote, Y.encodeStateVector(server))) + Y.applyUpdate(remote, Y.encodeStateAsUpdate(server, Y.encodeStateVector(remote))) + + const merged = yDocToMarkdown(server) + expect(merged).toContain('Alpha paragraph. EDITED') + expect(merged).toContain('expanded by the agent') + }) + + it('yDocToFileMarkdown matches the client save composition (frontmatter + postProcess body pass)', () => { + // A server-side persist must be byte-identical to the editor save — `applyFrontmatter(frontmatter, + // postProcessSerializedMarkdown(body))` — or it drifts from a client save / the dirty-check baseline + // and churns the blob. This guards against the postProcess pass being dropped from the server path. + const frontmatter = '---\ntitle: Doc\n---\n' + const doc = markdownToYDoc('- one\n- two\n\n> [!NOTE]\n> hi') + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.frontmatterKey, frontmatter) + const expected = applyFrontmatter( + frontmatter, + postProcessSerializedMarkdown(yDocToMarkdown(doc)) + ) + expect(yDocToFileMarkdown(doc)).toBe(expected) + doc.destroy() + }) + + it('yDocToFileMarkdown re-attaches empty frontmatter when the config map has none', () => { + const doc = markdownToYDoc('plain body\n') + expect(yDocToFileMarkdown(doc)).toBe(postProcessSerializedMarkdown(yDocToMarkdown(doc))) + doc.destroy() + }) +}) diff --git a/apps/sim/lib/collab-doc/converter.ts b/apps/sim/lib/collab-doc/converter.ts new file mode 100644 index 00000000000..990e0f906c6 --- /dev/null +++ b/apps/sim/lib/collab-doc/converter.ts @@ -0,0 +1,128 @@ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import { getSchema } from '@tiptap/core' +import { Node as ProseMirrorNode, type Schema } from '@tiptap/pm/model' +import { + initProseMirrorDoc, + prosemirrorJSONToYDoc, + updateYFragment, + yDocToProsemirrorJSON, +} from '@tiptap/y-tiptap' +import type * as Y from 'yjs' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + applyFrontmatter, + postProcessSerializedMarkdown, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { + parseMarkdownToDoc, + serializeDocToMarkdown, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' + +/** + * Server-side conversion between a file's markdown and its collaborative Yjs document. + * + * The markdown ↔ ProseMirror step reuses the EXACT client engine (`parseMarkdownToDoc` / + * `serializeDocToMarkdown`, both driven by `@tiptap/markdown` on the shared extension set), so the + * server can never diverge from what the editor renders — parity by construction, not by a second + * markdown implementation. The ProseMirror ↔ Yjs step uses `@tiptap/y-tiptap` (the same binding + * TipTap's Collaboration extension uses in the browser), so the Yjs structure the server produces is + * byte-compatible with the client's. + * + * The TipTap editor the markdown engine builds needs a DOM; on the server we back it with `jsdom` + * (see {@link ensureDomForTipTap}). This module is server-only by construction — it must never reach + * the client bundle (jsdom + the full editor would bloat it and break in the browser). It is kept + * out of that bundle by `require`-ing `jsdom` lazily (never a static top-level import) and by being + * imported only from server code (the seed builder + its internal route); there is no `import + * 'server-only'` marker because this repo does not use that package. + */ + +/** + * The Yjs `XmlFragment` name TipTap's Collaboration extension binds to. The client configures + * `Collaboration.configure({ document })` with no explicit `field`, so it uses TipTap's default, + * `'default'`. The server MUST target the same fragment or the client would sync an empty document. + */ +const COLLAB_DOC_FIELD = 'default' + +let cachedSchema: Schema | null = null + +/** The shared ProseMirror schema, built headlessly from the exact client extension set. */ +function markdownSchema(): Schema { + if (!cachedSchema) cachedSchema = getSchema(createMarkdownContentExtensions()) + return cachedSchema +} + +/** + * Ensure a DOM exists for the TipTap editor the markdown engine constructs. In a `jsdom`/browser + * environment `window` + `document` already exist and this is a no-op; in a plain Node server it + * installs a single shared jsdom window's globals. Cheap and idempotent — TipTap only needs + * `window`/`document`/`navigator` to build its (never-mounted) editor for parse/serialize. + * + * Gate on `window` (what TipTap's `elementFromString` actually checks), not just `document`, and hold + * NO cached "ready" flag: the Next server runtime exposes a partial `document` with NO `window`, and a + * `document`-only guard (plus a sticky flag) skipped this setup — leaving TipTap to throw "there is no + * window object available". Re-checking the globals every call means a partial stub can never wedge it. + * When `window` is missing we install a coherent jsdom window+document pair, overwriting any such stub. + */ +function ensureDomForTipTap(): void { + if (typeof window !== 'undefined' && typeof document !== 'undefined') return + // Lazy require so the client bundle never pulls jsdom in. Bind to `jsdomWindow`, NOT `window` — a + // local `const window` would shadow the global and put the `typeof window` guard above in its + // temporal dead zone ("Cannot access 'window' before initialization"). + const { JSDOM } = require('jsdom') as typeof import('jsdom') + const { window: jsdomWindow } = new JSDOM('') + const g = globalThis as unknown as Record + g.window = jsdomWindow + g.document = jsdomWindow.document + g.navigator ??= jsdomWindow.navigator +} + +/** Convert a file's markdown to a fresh collaborative {@link Y.Doc} (cold-start seed). */ +export function markdownToYDoc(markdown: string): Y.Doc { + ensureDomForTipTap() + const json = parseMarkdownToDoc(markdown) + return prosemirrorJSONToYDoc(markdownSchema(), json, COLLAB_DOC_FIELD) +} + +/** Project a collaborative {@link Y.Doc}'s BODY back to markdown (no frontmatter). */ +export function yDocToMarkdown(ydoc: Y.Doc): string { + ensureDomForTipTap() + const json = yDocToProsemirrorJSON(ydoc, COLLAB_DOC_FIELD) + return serializeDocToMarkdown(json) +} + +/** + * Project a collaborative {@link Y.Doc} back to the file's FULL canonical markdown — the body from the + * CRDT re-joined with the frontmatter carried in the config map. Mirrors the editor's save path EXACTLY + * (`applyFrontmatter(resolveSaveFrontmatter(), postProcessSerializedMarkdown(editor.getMarkdown()))`), + * INCLUDING the `postProcessSerializedMarkdown` body fidelity pass (empty list markers, callout + * un-escaping, trailing whitespace) — so a server-side persist is byte-identical to a client save and + * matches the client's dirty-check baseline, with no spurious blob churn on the round-trip. + */ +export function yDocToFileMarkdown(ydoc: Y.Doc): string { + const frontmatter = ydoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.frontmatterKey) + return applyFrontmatter( + typeof frontmatter === 'string' ? frontmatter : '', + postProcessSerializedMarkdown(yDocToMarkdown(ydoc)) + ) +} + +/** + * Apply new markdown content into an EXISTING collaborative {@link Y.Doc} as a minimal CRDT diff, + * merging with any concurrent user edits rather than replacing the document. This is how the agent + * writes into a live doc: `updateYFragment` computes exactly the changes between the fragment's + * current content and the target and applies them as Yjs operations — the same primitive TipTap's + * `ySyncPlugin` uses on every keystroke — so Yjs reconciles them with in-flight remote edits. + */ +export function applyMarkdownToYDoc(ydoc: Y.Doc, markdown: string): void { + ensureDomForTipTap() + const schema = markdownSchema() + const target = ProseMirrorNode.fromJSON(schema, parseMarkdownToDoc(markdown)) + const fragment = ydoc.getXmlFragment(COLLAB_DOC_FIELD) + // `updateYFragment` diffs against the fragment's CURRENT content, so it needs the fragment↔PM + // binding metadata (the element/mark mapping the live editor's ySyncPlugin normally maintains). + // `initProseMirrorDoc` reconstructs it from the fragment's present state. + const { meta } = initProseMirrorDoc(fragment, schema) + ydoc.transact(() => { + updateYFragment(ydoc, fragment, target, meta) + }) +} diff --git a/apps/sim/lib/collab-doc/merge.test.ts b/apps/sim/lib/collab-doc/merge.test.ts new file mode 100644 index 00000000000..bb4b28196c5 --- /dev/null +++ b/apps/sim/lib/collab-doc/merge.test.ts @@ -0,0 +1,70 @@ +/** + * @vitest-environment jsdom + */ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import { describe, expect, it } from 'vitest' +import * as Y from 'yjs' +import { serializeMarkdownBody } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' +import { markdownToYDoc, yDocToMarkdown } from './converter' +import { buildFileDocMergeUpdate } from './merge' + +describe('buildFileDocMergeUpdate', () => { + it('produces a diff that turns the live doc into the target markdown', () => { + const live = markdownToYDoc('# Title\n\nOriginal.') + const update = buildFileDocMergeUpdate(Y.encodeStateAsUpdate(live), '# Title\n\nRewritten.') + Y.applyUpdate(live, update) + expect(yDocToMarkdown(live)).toBe(serializeMarkdownBody('# Title\n\nRewritten.')) + }) + + it('strips frontmatter from the body but stores it in the config for the editor to re-attach', () => { + const live = markdownToYDoc('# Title\n\nBody.') + const update = buildFileDocMergeUpdate( + Y.encodeStateAsUpdate(live), + '---\ntitle: X\n---\n\n# Title\n\nBody rewritten.' + ) + Y.applyUpdate(live, update) + const md = yDocToMarkdown(live) + expect(md).not.toContain('title: X') + expect(md).toBe(serializeMarkdownBody('# Title\n\nBody rewritten.')) + // The updated frontmatter is carried in the config map, so an open editor re-attaches the NEW + // value on autosave instead of reverting to its stale open-time copy. + expect(live.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.frontmatterKey)).toContain( + 'title: X' + ) + }) + + it('returns an empty (no-op) diff when the markdown already matches', () => { + const live = markdownToYDoc('# Same\n\nBody.') + const before = Y.encodeStateVector(live) + const update = buildFileDocMergeUpdate(Y.encodeStateAsUpdate(live), yDocToMarkdown(live)) + Y.applyUpdate(live, update) + // Nothing new was introduced: the doc's state vector is unchanged by applying the diff. + expect(Y.encodeStateVector(live)).toEqual(before) + }) + + it('merges a copilot rewrite with a concurrent remote edit (no clobber)', () => { + // The relay captures the live doc's state and asks the app for the diff… + const live = markdownToYDoc('# Doc\n\nAlpha paragraph.\n\nBeta paragraph.') + const capturedState = Y.encodeStateAsUpdate(live) + + // …meanwhile a remote user edits the FIRST paragraph directly on the live doc, AFTER the capture. + const frag = live.getXmlFragment('default') + live.transact(() => { + const firstPara = frag.get(1) as Y.XmlElement + const textNode = firstPara.get(0) as Y.XmlText + textNode.insert(textNode.toString().length, ' EDITED') + }) + + // Copilot rewrites the SECOND paragraph; the diff is computed against the CAPTURED (older) state. + const update = buildFileDocMergeUpdate( + capturedState, + '# Doc\n\nAlpha paragraph.\n\nBeta paragraph, expanded by the agent.' + ) + // Applying the diff to the now-advanced live doc must preserve BOTH edits. + Y.applyUpdate(live, update) + + const merged = yDocToMarkdown(live) + expect(merged).toContain('Alpha paragraph. EDITED') + expect(merged).toContain('expanded by the agent') + }) +}) diff --git a/apps/sim/lib/collab-doc/merge.ts b/apps/sim/lib/collab-doc/merge.ts new file mode 100644 index 00000000000..3db9c73365e --- /dev/null +++ b/apps/sim/lib/collab-doc/merge.ts @@ -0,0 +1,39 @@ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import * as Y from 'yjs' +import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { applyMarkdownToYDoc } from './converter' + +/** + * Compute the minimal Yjs diff that turns `docState` — the live collaborative document as the realtime + * relay currently holds it — into `markdown`. This is the Stage C "copilot writes into the open doc" + * primitive: the relay owns the doc but not the conversion engine, so it ships the current state here + * and applies the returned diff, which Yjs merges with any concurrent user edits before relaying it to + * every connected editor. + * + * `applyMarkdownToYDoc` performs a real `updateYFragment` diff (not a replace), so unrelated + * paragraphs the user is editing are preserved. The returned update is relative to the document's + * state at call time (`Y.encodeStateAsUpdate(doc, before)`), so it is exactly the change to apply — and + * is empty (a no-op update) when `markdown` already matches the document. + */ +export function buildFileDocMergeUpdate(docState: Uint8Array, markdown: string): Uint8Array { + const doc = new Y.Doc() + try { + Y.applyUpdate(doc, docState) + const before = Y.encodeStateVector(doc) + // The collaborative body never includes frontmatter (callers pass full file content, e.g. + // copilot's `finalContent`), so merge only the body, and update the frontmatter in the config map + // — the editor re-attaches THAT on autosave, so a frontmatter change is reflected rather than + // reverted by an open editor's stale, open-time copy. + const { frontmatter, body } = splitFrontmatter(markdown) + applyMarkdownToYDoc(doc, body) + // Only write when it actually changed (treating "unset" as "") so an unchanged edit stays a true + // no-op diff rather than churning the config map on every merge. + const config = doc.getMap(FILE_DOC_SEED.configMap) + if ((config.get(FILE_DOC_SEED.frontmatterKey) ?? '') !== frontmatter) { + config.set(FILE_DOC_SEED.frontmatterKey, frontmatter) + } + return Y.encodeStateAsUpdate(doc, before) + } finally { + doc.destroy() + } +} diff --git a/apps/sim/lib/collab-doc/persist.ts b/apps/sim/lib/collab-doc/persist.ts new file mode 100644 index 00000000000..c83c0c6ee60 --- /dev/null +++ b/apps/sim/lib/collab-doc/persist.ts @@ -0,0 +1,62 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import * as Y from 'yjs' +import { getWorkspaceFile, updateWorkspaceFileContent } from '@/lib/uploads/contexts/workspace' +import { hashMarkdown, saveCollabDocState } from './collab-state' +import { yDocToFileMarkdown } from './converter' + +const logger = createLogger('FileDocPersist') + +/** + * Project a live collaborative document back to durable markdown and write it to the file. This is the + * server-authoritative durable path — the realtime relay owns the live Yjs doc but not the conversion + * engine or blob/DB access, so it ships the doc state here and the app persists it. Called debounced + * while the doc is being edited and when the last collaborator leaves; it replaces the editor's + * client-side autosave, so a copilot (or any server) edit can never be clobbered by a stale keystroke + * saving over it. + * + * Returns `false` when the file is genuinely absent (deleted) — nothing to write. Any other failure + * (conversion / blob / DB) THROWS so the caller surfaces a non-2xx and can retry on the next debounce. + * + * `userId` is attribution only (blob metadata) — the last collaborator to touch the doc; the caller is + * already trusted via the internal `x-api-key` gate, so this does not re-authorize. + */ +export async function persistFileDoc( + workspaceId: string, + fileId: string, + userId: string, + docState: Uint8Array +): Promise { + const record = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) + if (!record) return false + + const ydoc = new Y.Doc() + let markdownBuffer: Buffer + try { + Y.applyUpdate(ydoc, docState) + markdownBuffer = Buffer.from(yDocToFileMarkdown(ydoc), 'utf-8') + } finally { + ydoc.destroy() + } + + // `syncLiveDoc: false`: this write IS the projection of the live doc back to markdown, so merging it + // back into that same doc would be a persist → merge → persist self-loop. The doc already holds this + // exact content; peers are already converged through the relay. + await updateWorkspaceFileContent(workspaceId, fileId, userId, markdownBuffer, undefined, { + syncLiveDoc: false, + }) + + // Cache the Yjs binary (tagged with the exact markdown just written) so a later cold room open loads + // it directly instead of re-converting markdown → Yjs. Best-effort: the markdown IS the durable file, + // so a cache failure only means the next cold open re-converts — never lost data. + try { + await saveCollabDocState(fileId, docState, hashMarkdown(markdownBuffer)) + } catch (error) { + logger.warn(`Failed to cache collab doc state for file ${fileId}`, { + error: getErrorMessage(error), + }) + } + + logger.info(`Persisted live collaborative document to file ${fileId} (workspace ${workspaceId})`) + return true +} diff --git a/apps/sim/lib/collab-doc/seed.test.ts b/apps/sim/lib/collab-doc/seed.test.ts new file mode 100644 index 00000000000..a3785a1c5ec --- /dev/null +++ b/apps/sim/lib/collab-doc/seed.test.ts @@ -0,0 +1,127 @@ +/** + * @vitest-environment jsdom + */ +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' + +const { mockGetWorkspaceFile, mockFetchBuffer, mockLoadFresh } = vi.hoisted(() => ({ + mockGetWorkspaceFile: vi.fn(), + mockFetchBuffer: vi.fn(), + mockLoadFresh: vi.fn(), +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + getWorkspaceFile: mockGetWorkspaceFile, + fetchWorkspaceFileBuffer: mockFetchBuffer, +})) + +// The DB-backed cold-start cache is exercised in its own suite; here we default it to a MISS so these +// tests cover the markdown → Yjs conversion path (the cache-hit fast path has its own test below). +vi.mock('./collab-state', () => ({ + hashMarkdown: () => 'test-source-hash', + loadFreshCollabDocState: mockLoadFresh, +})) + +import { serializeMarkdownBody } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' +import { yDocToMarkdown } from './converter' +import { buildFileDocSeed } from './seed' + +describe('buildFileDocSeed', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetWorkspaceFile.mockResolvedValue({ + id: 'file-1', + name: 'note.md', + key: 'k', + context: 'workspace', + }) + // Default: no cached binary → the conversion path runs (the case these tests cover). + mockLoadFresh.mockResolvedValue(null) + }) + + it('builds a seed whose applied update reproduces the file body (through the client engine)', async () => { + mockFetchBuffer.mockResolvedValue(Buffer.from('# Title\n\nHello **world**.', 'utf-8')) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + expect(seed).not.toBeNull() + + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(yDocToMarkdown(doc)).toBe(serializeMarkdownBody('# Title\n\nHello **world**.')) + }) + + it('cold-start fast path: returns the cached binary directly without re-converting when it is fresh', async () => { + // A cached binary derived from the current markdown → seed returns it verbatim (no conversion), the + // Hocuspocus load-document path that preserves the CRDT's client ids across reopens. + const cachedDoc = new Y.Doc() + cachedDoc.getText('marker').insert(0, 'cached') + const cached = Y.encodeStateAsUpdate(cachedDoc) + mockFetchBuffer.mockResolvedValue(Buffer.from('# Anything', 'utf-8')) + mockLoadFresh.mockResolvedValue(cached) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + + expect(seed?.update).toBe(cached) + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(doc.getText('marker').toString()).toBe('cached') + }) + + it('falls through to conversion when the cache read fails (never blocks a cold open)', async () => { + // The cache is a best-effort optimization over the durable markdown we already hold; a transient DB + // error or a not-yet-migrated cache table must convert, not abort the seed. + mockFetchBuffer.mockResolvedValue(Buffer.from('# Title\n\ntext.', 'utf-8')) + mockLoadFresh.mockRejectedValue(new Error('cache table missing')) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + expect(seed).not.toBeNull() + + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(yDocToMarkdown(doc)).toBe(serializeMarkdownBody('# Title\n\ntext.')) + }) + + it('strips frontmatter — only the body seeds the collaborative doc', async () => { + mockFetchBuffer.mockResolvedValue(Buffer.from('---\ntitle: X\n---\n\n# Body\n\ntext.', 'utf-8')) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + const md = yDocToMarkdown(doc) + expect(md).not.toContain('title: X') + expect(md).toBe(serializeMarkdownBody('# Body\n\ntext.')) + }) + + it('marks the seeded doc as initial-content-loaded so the client needs no seeder handshake', async () => { + mockFetchBuffer.mockResolvedValue(Buffer.from('# Body', 'utf-8')) + const seed = await buildFileDocSeed('ws-1', 'file-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + }) + + it('carries the frontmatter in the config map (not the body)', async () => { + mockFetchBuffer.mockResolvedValue(Buffer.from('---\ntitle: X\n---\n\n# Body', 'utf-8')) + const seed = await buildFileDocSeed('ws-1', 'file-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, seed!.update) + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.frontmatterKey)).toContain( + 'title: X' + ) + // …and the frontmatter is NOT in the collaborative body. + expect(yDocToMarkdown(doc)).not.toContain('title: X') + }) + + it('returns null for a missing file', async () => { + mockGetWorkspaceFile.mockResolvedValue(null) + expect(await buildFileDocSeed('ws-1', 'missing')).toBeNull() + }) + + it('requests the file with throwOnError so a read failure is not mistaken for an empty file', async () => { + mockGetWorkspaceFile.mockRejectedValue(new Error('db down')) + // Propagates instead of returning null — the relay must retry, never seed blank over a real file. + await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toThrow('db down') + expect(mockGetWorkspaceFile).toHaveBeenCalledWith('ws-1', 'file-1', { throwOnError: true }) + }) +}) diff --git a/apps/sim/lib/collab-doc/seed.ts b/apps/sim/lib/collab-doc/seed.ts new file mode 100644 index 00000000000..4fbfaa544c3 --- /dev/null +++ b/apps/sim/lib/collab-doc/seed.ts @@ -0,0 +1,81 @@ +import { createLogger } from '@sim/logger' +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import { getErrorMessage } from '@sim/utils/errors' +import * as Y from 'yjs' +import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contexts/workspace' +import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { hashMarkdown, loadFreshCollabDocState } from './collab-state' +import { markdownToYDoc } from './converter' + +const logger = createLogger('FileDocSeed') + +/** + * The largest file we will build a collaborative seed for. Beyond this the editor uses its + * non-collaborative path anyway; converting a huge document server-side would be wasted work. + */ +const MAX_SEED_BYTES = 5 * 1024 * 1024 + +/** A collaborative document's initial state, encoded as a Yjs update the relay can apply. */ +export interface FileDocSeed { + /** `Y.encodeStateAsUpdate` of the seeded document — apply with `Y.applyUpdate`. */ + update: Uint8Array +} + +/** + * Build the server-side seed for a file's collaborative document: load the file's current markdown + * and convert it — through the exact client engine (see {@link markdownToYDoc}) — into a Yjs update. + * + * This is what makes seeding server-authoritative: the realtime relay applies this to a fresh room's + * document instead of electing a client to import the content, so the whole client-seeder subsystem + * (election / deadlines / retries) goes away. The frontmatter is stripped exactly as the client's + * seed did — it is file metadata, not part of the collaborative body. + * + * Returns `null` ONLY when the file is genuinely absent (deleted/never-existed). A transient read + * failure THROWS (`throwOnError`) rather than returning `null`, so the relay retries instead of + * mistaking a DB blip for an empty file and seeding blank content over the real document. + */ +export async function buildFileDocSeed( + workspaceId: string, + fileId: string +): Promise { + const record = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) + if (!record) return null + + const buffer = await fetchWorkspaceFileBuffer(record, { maxBytes: MAX_SEED_BYTES }) + + // Cold-start fast path: if we hold a cached Yjs binary derived from THIS exact markdown, apply it + // directly (the Hocuspocus load-document pattern) instead of re-converting. This preserves the CRDT's + // client ids across reopens — no duplicated content, no split-brain — and skips the server-side + // headless conversion. A stale/absent cache (markdown edited externally, or first ever open) falls + // through to the conversion below, and the next persist refreshes the cache. + // + // Best-effort read: the cache is an optimization over the durable markdown we already hold, so a + // transient DB error (or a not-yet-migrated cache table) must fall through to conversion rather than + // block the cold open — symmetric with persist's best-effort cache write. + try { + const cached = await loadFreshCollabDocState(fileId, hashMarkdown(buffer)) + if (cached) return { update: cached } + } catch (error) { + logger.warn(`Failed to read cached collab doc state for file ${fileId}`, { + error: getErrorMessage(error), + }) + } + + const markdown = buffer.toString('utf-8') + const { frontmatter, body } = splitFrontmatter(markdown) + + const ydoc = markdownToYDoc(body) + try { + const config = ydoc.getMap(FILE_DOC_SEED.configMap) + // Mark the document seeded IN the same doc, so the client's readiness gate + // (`synced && initialContentLoaded === true`) recognizes a server-seeded doc without any + // client-seeder handshake, and a stray re-election can never seed on top of it. + config.set(FILE_DOC_SEED.flag, true) + // Carry the frontmatter in the doc (not the body) so it merges across clients and a later + // server-side edit can update it — the editor re-attaches this on autosave. + config.set(FILE_DOC_SEED.frontmatterKey, frontmatter) + return { update: Y.encodeStateAsUpdate(ydoc) } + } finally { + ydoc.destroy() + } +} diff --git a/apps/sim/lib/copilot/tools/server/files/create-file.ts b/apps/sim/lib/copilot/tools/server/files/create-file.ts index 1d5e3629546..cd950eb3406 100644 --- a/apps/sim/lib/copilot/tools/server/files/create-file.ts +++ b/apps/sim/lib/copilot/tools/server/files/create-file.ts @@ -64,6 +64,9 @@ export const createFileServerTool: BaseServerTool table, requestId ) + signalTableRowsChanged(args.tableId) return { success: true, @@ -608,6 +610,7 @@ export const userTableServerTool: BaseServerTool table, requestId ) + signalTableRowsChanged(args.tableId) return { success: true, @@ -751,6 +754,7 @@ export const userTableServerTool: BaseServerTool // doesn't, so the guard never trips here. Defensive narrowing. return { success: false, message: 'Row update was skipped' } } + signalTableRowsChanged(args.tableId) // Auto-dispatch for user edits is handled inside `updateRow` // (mode: 'new' for newly-cleared groups + cancel+rerun for in-flight // downstream groups). Firing a second mode: 'incomplete' dispatch @@ -790,6 +794,7 @@ export const userTableServerTool: BaseServerTool return { success: false, message: `Table ${args.tableId} not found` } } await deleteRow(deleteRowTable, args.rowId, requestId) + signalTableRowsChanged(args.tableId) return { success: true, @@ -893,6 +898,7 @@ export const userTableServerTool: BaseServerTool }, requestId ) + if (result.affectedCount > 0) signalTableRowsChanged(args.tableId) return { success: true, @@ -998,6 +1004,7 @@ export const userTableServerTool: BaseServerTool } finally { await releaseJobClaim(table.id, inlineDeleteId).catch(() => {}) } + if (result.affectedCount > 0) signalTableRowsChanged(args.tableId) recordAudit({ workspaceId, @@ -1081,6 +1088,7 @@ export const userTableServerTool: BaseServerTool table, requestId ) + if (result.affectedCount > 0) signalTableRowsChanged(args.tableId) return { success: true, @@ -1120,6 +1128,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, rowIds, workspaceId }, requestId ) + if (result.deletedCount > 0) signalTableRowsChanged(args.tableId) recordAudit({ workspaceId, @@ -1428,6 +1437,7 @@ export const userTableServerTool: BaseServerTool table, requestId ) + signalTableRowsChanged(table.id) logger.info('Rows replaced from file', { tableId: table.id, @@ -1456,6 +1466,7 @@ export const userTableServerTool: BaseServerTool } const inserted = await batchInsertAll(table.id, coerced, table, workspaceId, context) + if (inserted > 0) signalTableRowsChanged(table.id) logger.info('Rows imported from file', { tableId: table.id, @@ -1519,6 +1530,7 @@ export const userTableServerTool: BaseServerTool ? { ...col, options: normalizeSelectOptionsInput(col.options) } : { ...col, options: undefined } const updated = await addTableColumn(args.tableId, columnToAdd, requestId) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Added column "${col.name}" (${col.type}) to table`, @@ -1548,6 +1560,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, oldName: colName, newName: newColName }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Renamed column "${colName}" to "${newColName}"`, @@ -1579,6 +1592,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, columnName: names[0] }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Deleted column "${names[0]}"`, @@ -1590,6 +1604,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, columnNames: names }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Deleted ${names.length} columns: ${names.join(', ')}`, @@ -1695,6 +1710,7 @@ export const userTableServerTool: BaseServerTool requestId ) } + if (result) signalTableSchemaChanged(args.tableId) return { success: true, message: `Updated column "${colName}"`, @@ -1724,6 +1740,7 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) assertNotAborted() const renamed = await renameTable(args.tableId, newName, requestId, context.userId) + signalTableSchemaChanged(args.tableId) return { success: true, @@ -1847,6 +1864,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, group, outputColumns, autoRun, actorUserId: context.userId }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Added workflow group "${name ?? groupId}" with ${outputs.length} output column(s)`, @@ -1919,6 +1937,7 @@ export const userTableServerTool: BaseServerTool }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Updated workflow group ${groupId}`, @@ -1940,6 +1959,7 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) assertNotAborted() const updated = await deleteWorkflowGroup({ tableId: args.tableId, groupId }, requestId) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Deleted workflow group ${groupId}`, @@ -1977,6 +1997,7 @@ export const userTableServerTool: BaseServerTool }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Added output to workflow group ${groupId}`, @@ -2005,6 +2026,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, groupId, columnName }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Removed output "${columnName}" from workflow group ${groupId}`, @@ -2218,6 +2240,7 @@ export const userTableServerTool: BaseServerTool { tableId: args.tableId, group, outputColumns, autoRun, actorUserId: context.userId }, requestId ) + signalTableSchemaChanged(args.tableId) return { success: true, message: `Added enrichment "${name}" with ${outputs.length} output column(s)${ diff --git a/apps/sim/lib/copilot/vfs/resource-writer.ts b/apps/sim/lib/copilot/vfs/resource-writer.ts index 33ad1c3530e..f9f6a5eddd8 100644 --- a/apps/sim/lib/copilot/vfs/resource-writer.ts +++ b/apps/sim/lib/copilot/vfs/resource-writer.ts @@ -164,6 +164,12 @@ export async function writeWorkspaceFileByPath(args: { target: WorkspaceFileWriteTarget buffer: Buffer inferredMimeType: string + /** + * Forwarded to {@link updateWorkspaceFileContent} on an overwrite. Defaults to `true` (stream a + * markdown overwrite into any open collaborative editor). Pass `false` for a write whose content is + * only a placeholder — e.g. `create_file`'s empty shell, whose real content lands via a later write. + */ + syncLiveDoc?: boolean }): Promise { const contentType = args.target.mimeType || args.inferredMimeType if (args.target.mode === 'overwrite') { @@ -177,7 +183,8 @@ export async function writeWorkspaceFileByPath(args: { existing.id, args.userId, args.buffer, - contentType || existing.type + contentType || existing.type, + { syncLiveDoc: args.syncLiveDoc } ) return { diff --git a/apps/sim/lib/folders/config.ts b/apps/sim/lib/folders/config.ts index 4c6b3d6333e..02e803f5749 100644 --- a/apps/sim/lib/folders/config.ts +++ b/apps/sim/lib/folders/config.ts @@ -306,6 +306,8 @@ async function archiveTableChildren(context: CascadeChildrenContext): Promise ({ env: { REDIS_URL: undefined } })) +vi.mock('@/lib/core/config/redis', () => ({ getRedisClient: () => null })) + +import { + appendEvent, + type EventLogConfig, + type EventLogEntry, + getLatestEventId, + readEventsSince, + resetEventLogMemoryForTesting, +} from '@/lib/realtime/event-log' + +interface TestEntry extends EventLogEntry { + eventId: number + streamId: string + value: string +} + +const config: EventLogConfig = { prefix: 'test:stream:', ttlSeconds: 3600, cap: 3, readChunk: 500 } + +function serializerFor(streamId: string, value: string) { + return { + entryPrefix: '{"eventId":', + entrySuffix: `,"streamId":${JSON.stringify(streamId)},"value":${JSON.stringify(value)}}`, + buildEntry: (eventId: number): TestEntry => ({ eventId, streamId, value }), + } +} + +describe('event-log (memory fallback)', () => { + beforeEach(() => resetEventLogMemoryForTesting()) + + it('assigns monotonically increasing event ids', async () => { + const first = await appendEvent(config, 's1', serializerFor('s1', 'a')) + const second = await appendEvent(config, 's1', serializerFor('s1', 'b')) + expect(first?.eventId).toBe(1) + expect(second?.eventId).toBe(2) + }) + + it('isolates streams by id', async () => { + await appendEvent(config, 's1', serializerFor('s1', 'a')) + const other = await appendEvent(config, 's2', serializerFor('s2', 'x')) + expect(other?.eventId).toBe(1) + expect(await getLatestEventId(config, 's1')).toBe(1) + expect(await getLatestEventId(config, 's2')).toBe(1) + }) + + it('reads only events after the cursor', async () => { + await appendEvent(config, 's1', serializerFor('s1', 'a')) + await appendEvent(config, 's1', serializerFor('s1', 'b')) + const result = await readEventsSince(config, 's1', 1) + expect(result.status).toBe('ok') + if (result.status === 'ok') { + expect(result.events).toHaveLength(1) + expect(result.events[0].eventId).toBe(2) + expect(result.events[0].value).toBe('b') + } + }) + + it('tails from the latest id and returns nothing for a fresh cursor', async () => { + await appendEvent(config, 's1', serializerFor('s1', 'a')) + await appendEvent(config, 's1', serializerFor('s1', 'b')) + const latest = await getLatestEventId(config, 's1') + const result = await readEventsSince(config, 's1', latest) + expect(result).toEqual({ status: 'ok', events: [] }) + }) + + it('reports pruned when the cursor falls behind the cap-trimmed buffer', async () => { + // cap = 3; append 5, so the earliest retained id is 3. + for (const v of ['a', 'b', 'c', 'd', 'e']) { + await appendEvent(config, 's1', serializerFor('s1', v)) + } + const result = await readEventsSince(config, 's1', 1) + expect(result.status).toBe('pruned') + if (result.status === 'pruned') expect(result.earliestEventId).toBe(3) + }) + + it('reports pruned for a non-zero cursor against a never-seen stream', async () => { + const result = await readEventsSince(config, 'missing', 5) + expect(result.status).toBe('pruned') + }) +}) diff --git a/apps/sim/lib/realtime/event-log.ts b/apps/sim/lib/realtime/event-log.ts new file mode 100644 index 00000000000..f7b470c33a2 --- /dev/null +++ b/apps/sim/lib/realtime/event-log.ts @@ -0,0 +1,283 @@ +/** + * Generic durable event log over Redis (sorted set + monotonic id + TTL), with an + * in-memory fallback for dev/tests. This is the reusable core extracted from the + * Tables cell-event buffer; a domain adapter (e.g. `lib/table/events.ts`) supplies + * its Redis key prefix and how to serialize an entry, and gets append/read/tail + * semantics for free — including replay-on-reconnect and prune detection. + * + * The core is deliberately domain-neutral: it only knows an entry has a numeric + * `eventId`. Everything else in the entry is opaque bytes the adapter owns, so a + * domain can keep its exact wire shape (and its existing Redis keys) unchanged. + * + * Modeled after `apps/sim/lib/execution/event-buffer.ts` but stripped of what an + * always-on stream doesn't need (no id-reservation batching, no write-queue + * serialization, no per-entity terminal lifecycle, no byte budgeting). + */ + +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { env } from '@/lib/core/config/env' +import { getRedisClient } from '@/lib/core/config/redis' + +const logger = createLogger('EventLog') + +/** + * Atomic append: INCR the seq counter to mint a new eventId, splice it into the + * adapter-supplied entry JSON, ZADD it, refresh TTLs, trim to cap, and record the + * resulting earliestEventId in meta — one round-trip. Without atomicity a slow + * reader could observe the trim before the meta update and miss the prune signal. + * + * KEYS: [events, seq, meta] + * ARGV: [ttlSec, cap, updatedAtIso, entryPrefix, entrySuffix] + * The new eventId is spliced between prefix/suffix to form the entry JSON. + * Returns the new eventId. + */ +const APPEND_EVENT_SCRIPT = ` +local eventId = redis.call('INCR', KEYS[2]) +local entry = ARGV[4] .. eventId .. ARGV[5] +redis.call('ZADD', KEYS[1], eventId, entry) +redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1])) +redis.call('EXPIRE', KEYS[2], tonumber(ARGV[1])) +redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -tonumber(ARGV[2]) - 1) +local oldest = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES') +if oldest[2] then + redis.call('HSET', KEYS[3], 'earliestEventId', tostring(math.floor(tonumber(oldest[2]))), 'updatedAt', ARGV[3]) + redis.call('EXPIRE', KEYS[3], tonumber(ARGV[1])) +end +return eventId +` + +/** Configuration for a durable event-log stream family (e.g. Tables). */ +export interface EventLogConfig { + /** + * Redis key prefix, e.g. `table:stream:`. STABLE per family — renaming it resets + * the seq counter, which silently strands live clients holding a higher + * in-memory `lastEventId` (their `?from=` never matches). Never change it. + */ + prefix: string + ttlSeconds: number + cap: number + /** Max entries returned by one read; the SSE route drains in chunks. */ + readChunk: number +} + +/** Minimal shape the core requires; adapters extend it with their own fields. */ +export interface EventLogEntry { + eventId: number +} + +/** + * How an adapter serializes one entry. `entryPrefix`/`entrySuffix` are spliced + * around the minted `eventId` in Lua (`prefix + eventId + suffix`) for the Redis + * write; `buildEntry` returns the equivalent object. It is the canonical entry + * builder for BOTH paths — the in-memory fallback AND the Redis success path, + * which returns `buildEntry(eventId)` rather than re-parsing the stored string — + * so it MUST produce the byte-identical shape to the spliced JSON, or dev/no-Redis + * behaves differently from prod. + */ +export interface EntrySerializer { + entryPrefix: string + entrySuffix: string + buildEntry: (eventId: number) => E +} + +export type EventLogReadResult = + | { status: 'ok'; events: E[] } + | { status: 'pruned'; earliestEventId: number | undefined } + | { status: 'unavailable'; error: string } + +function eventsKey(config: EventLogConfig, streamId: string) { + return `${config.prefix}${streamId}:events` +} +function seqKey(config: EventLogConfig, streamId: string) { + return `${config.prefix}${streamId}:seq` +} +function metaKey(config: EventLogConfig, streamId: string) { + return `${config.prefix}${streamId}:meta` +} + +interface MemoryStream { + events: E[] + earliestEventId?: number + nextEventId: number + expiresAt: number +} + +/** In-memory fallback keyed by `${prefix}${streamId}`, shared across all families. */ +const memoryStreams = new Map>() + +function memoryKey(config: EventLogConfig, streamId: string) { + return `${config.prefix}${streamId}` +} + +function canUseMemoryBuffer(): boolean { + return typeof window === 'undefined' && !env.REDIS_URL +} + +function pruneExpiredMemoryStreams(now = Date.now()): void { + for (const [key, stream] of memoryStreams) { + if (stream.expiresAt <= now) memoryStreams.delete(key) + } +} + +function getMemoryStream(config: EventLogConfig, streamId: string): MemoryStream { + pruneExpiredMemoryStreams() + const key = memoryKey(config, streamId) + let stream = memoryStreams.get(key) + if (!stream) { + stream = { events: [], nextEventId: 1, expiresAt: Date.now() + config.ttlSeconds * 1000 } + memoryStreams.set(key, stream) + } + return stream +} + +/** + * Append an event. Fire-and-forget from the caller — never throws, returns null on + * failure. A Redis blip must not fail the originating mutation. + */ +export async function appendEvent( + config: EventLogConfig, + streamId: string, + serializer: EntrySerializer +): Promise { + const redis = getRedisClient() + if (!redis) { + if (canUseMemoryBuffer()) { + try { + const stream = getMemoryStream(config, streamId) + const entry = serializer.buildEntry(stream.nextEventId++) + stream.events.push(entry) + if (stream.events.length > config.cap) { + stream.events = stream.events.slice(-config.cap) + stream.earliestEventId = stream.events[0]?.eventId + } + stream.expiresAt = Date.now() + config.ttlSeconds * 1000 + return entry + } catch (error) { + logger.warn('appendEvent: memory append failed', { + streamId, + error: toError(error).message, + }) + return null + } + } + return null + } + try { + const result = await redis.eval( + APPEND_EVENT_SCRIPT, + 3, + eventsKey(config, streamId), + seqKey(config, streamId), + metaKey(config, streamId), + config.ttlSeconds, + config.cap, + new Date().toISOString(), + serializer.entryPrefix, + serializer.entrySuffix + ) + const eventId = typeof result === 'number' ? result : Number(result) + if (!Number.isFinite(eventId)) return null + return serializer.buildEntry(eventId) + } catch (error) { + logger.warn('appendEvent: Redis append failed', { streamId, error: toError(error).message }) + return null + } +} + +/** + * The latest eventId assigned for a stream, or 0 when empty/expired. Used by the + * stream route to tail from "now" when a client connects without a replay cursor. + * Redis errors propagate so the route errors the stream instead of replaying the + * whole buffer over freshly-fetched state. + */ +export async function getLatestEventId(config: EventLogConfig, streamId: string): Promise { + const redis = getRedisClient() + if (!redis) { + if (canUseMemoryBuffer()) { + const stream = memoryStreams.get(memoryKey(config, streamId)) + return stream ? stream.nextEventId - 1 : 0 + } + return 0 + } + const raw = await redis.get(seqKey(config, streamId)) + if (!raw) return 0 + const parsed = Number.parseInt(raw, 10) + return Number.isFinite(parsed) && parsed > 0 ? parsed : 0 +} + +/** + * Read events where eventId > afterEventId. Returns 'pruned' if the caller has + * fallen off the back of the buffer (TTL expired or cap rolled past their cursor); + * the caller should full-refetch and resume from the new earliest id. + */ +export async function readEventsSince( + config: EventLogConfig, + streamId: string, + afterEventId: number +): Promise> { + const redis = getRedisClient() + if (!redis) { + if (canUseMemoryBuffer()) { + pruneExpiredMemoryStreams() + const stream = memoryStreams.get(memoryKey(config, streamId)) + if (!stream) { + if (afterEventId > 0) return { status: 'pruned', earliestEventId: undefined } + return { status: 'ok', events: [] } + } + if (stream.earliestEventId !== undefined && afterEventId + 1 < stream.earliestEventId) { + return { status: 'pruned', earliestEventId: stream.earliestEventId } + } + return { + status: 'ok', + events: stream.events + .filter((entry) => entry.eventId > afterEventId) + .slice(0, config.readChunk) as E[], + } + } + return { status: 'unavailable', error: 'Redis client unavailable' } + } + try { + const meta = await redis.hgetall(metaKey(config, streamId)) + const earliestEventId = + meta?.earliestEventId !== undefined ? Number(meta.earliestEventId) : undefined + if (earliestEventId !== undefined && afterEventId + 1 < earliestEventId) { + return { status: 'pruned', earliestEventId } + } + const raw = await redis.zrangebyscore( + eventsKey(config, streamId), + afterEventId + 1, + '+inf', + 'LIMIT', + 0, + config.readChunk + ) + if (raw.length === 0 && afterEventId > 0) { + const seqExists = await redis.exists(seqKey(config, streamId)) + if (seqExists === 0) { + return { status: 'pruned', earliestEventId: undefined } + } + } + return { + status: 'ok', + events: raw + .map((entry) => { + try { + return JSON.parse(entry) as E + } catch { + return null + } + }) + .filter((entry): entry is E => entry !== null), + } + } catch (error) { + const message = toError(error).message + logger.warn('readEventsSince failed', { streamId, error: message }) + return { status: 'unavailable', error: message } + } +} + +/** Test-only: clear the in-memory streams between cases. */ +export function resetEventLogMemoryForTesting(): void { + memoryStreams.clear() +} diff --git a/apps/sim/lib/realtime/event-stream-route.ts b/apps/sim/lib/realtime/event-stream-route.ts new file mode 100644 index 00000000000..ad4fd2e8a82 --- /dev/null +++ b/apps/sim/lib/realtime/event-stream-route.ts @@ -0,0 +1,160 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { NextResponse } from 'next/server' +import { SSE_HEADERS } from '@/lib/core/utils/sse' +import type { EventLogEntry, EventLogReadResult } from '@/lib/realtime/event-log' + +const logger = createLogger('EventStreamRoute') + +const POLL_INTERVAL_MS = 500 +const HEARTBEAT_INTERVAL_MS = 15_000 +/** Defensive ceiling; the client reconnects (resuming from lastEventId) past this. */ +const MAX_STREAM_DURATION_MS = 4 * 60 * 60 * 1000 + +export interface EventStreamResponseOptions { + requestId: string + /** The durable-log stream id (e.g. a tableId). */ + streamId: string + /** Replay cursor from `?from=`; `undefined` tails from the latest event id. */ + fromEventId: number | undefined + getLatestEventId: (streamId: string) => Promise + readEventsSince: (streamId: string, afterEventId: number) => Promise> + /** Extra response headers (e.g. `{ 'X-Table-Id': id }`). */ + extraHeaders?: Record + /** Short label for logs (e.g. 'table'). */ + label: string +} + +/** + * Shared SSE stream for any durable event log (`@/lib/realtime/event-log`). Handles + * replay-on-reconnect (`?from=`), tail-from-latest on a fresh mount, chunked poll + + * forward, heartbeats, graceful `pruned`/`rotate` close, and error propagation. + * + * Auth and contract parsing stay in the route (they are domain-specific); this + * owns only the streaming mechanics, so every durable-log surface streams + * identically. The poll loop mirrors the execution stream; pub/sub wakeups are an + * optimization that can replace the 500ms poll later without changing this shape. + */ +export function createEventStreamResponse( + options: EventStreamResponseOptions +): NextResponse { + const { requestId, streamId, fromEventId, getLatestEventId, readEventsSince, label } = options + + logger.info(`[${requestId}] ${label} event stream opened`, { streamId, fromEventId }) + + const encoder = new TextEncoder() + let closed = false + + const stream = new ReadableStream({ + async start(controller) { + let lastEventId = fromEventId ?? 0 + const deadline = Date.now() + MAX_STREAM_DURATION_MS + let nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS + + const enqueue = (text: string) => { + if (closed) return + try { + controller.enqueue(encoder.encode(text)) + } catch { + closed = true + } + } + + const sendEvents = (events: E[]) => { + for (const entry of events) { + if (closed) return + enqueue(`data: ${JSON.stringify(entry)}\n\n`) + lastEventId = entry.eventId + } + } + + const sendPrunedAndClose = (earliestEventId: number | undefined) => { + enqueue( + `event: pruned\ndata: ${JSON.stringify({ earliestEventId: earliestEventId ?? null })}\n\n` + ) + if (!closed) { + closed = true + try { + controller.close() + } catch {} + } + } + + const sendHeartbeat = () => { + // SSE comment line — keeps proxies (ALB default 60s idle) from closing + // the connection during quiet periods. + enqueue(`: ping ${Date.now()}\n\n`) + } + + try { + // No replay cursor → tail from the latest event id. Resolved inside the + // try so a Redis failure errors the stream (client reconnects with + // backoff) rather than silently replaying the whole buffer. + if (fromEventId === undefined) { + lastEventId = await getLatestEventId(streamId) + } + + const initial = await readEventsSince(streamId, lastEventId) + if (initial.status === 'pruned') { + sendPrunedAndClose(initial.earliestEventId) + return + } + if (initial.status === 'unavailable') { + throw new Error(`${label} event buffer unavailable: ${initial.error}`) + } + sendEvents(initial.events) + + while (!closed && Date.now() < deadline) { + await sleep(POLL_INTERVAL_MS) + if (closed) return + + const result = await readEventsSince(streamId, lastEventId) + if (result.status === 'pruned') { + sendPrunedAndClose(result.earliestEventId) + return + } + if (result.status === 'unavailable') { + throw new Error(`${label} event buffer unavailable: ${result.error}`) + } + if (result.events.length > 0) { + sendEvents(result.events) + } + + if (Date.now() >= nextHeartbeatAt) { + sendHeartbeat() + nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS + } + } + + // Reached the defensive duration ceiling — close cleanly so the client + // reconnects with the latest lastEventId. + if (!closed) { + enqueue(`event: rotate\ndata: {}\n\n`) + closed = true + try { + controller.close() + } catch {} + } + } catch (error) { + logger.error(`[${requestId}] ${label} event stream error`, { + streamId, + error: toError(error).message, + }) + if (!closed) { + try { + controller.error(error) + } catch {} + } + } + }, + cancel() { + closed = true + logger.info(`[${requestId}] Client disconnected from ${label} event stream`, { streamId }) + }, + }) + + return new NextResponse(stream, { + headers: { ...SSE_HEADERS, ...(options.extraHeaders ?? {}) }, + }) +} diff --git a/apps/sim/lib/realtime/notify.test.ts b/apps/sim/lib/realtime/notify.test.ts new file mode 100644 index 00000000000..7e780e76130 --- /dev/null +++ b/apps/sim/lib/realtime/notify.test.ts @@ -0,0 +1,41 @@ +/** + * @vitest-environment node + */ +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/utils/urls', () => ({ getSocketServerUrl: () => 'http://realtime' })) +vi.mock('@/lib/core/config/env', () => ({ env: { INTERNAL_API_SECRET: 'secret' } })) + +import { mergeEditIntoLiveFileDoc } from './notify' + +describe('mergeEditIntoLiveFileDoc', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('POSTs the edit to the realtime apply-edit endpoint with the api key', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true }) + vi.stubGlobal('fetch', fetchMock) + + await mergeEditIntoLiveFileDoc('file-1', '# hello') + + expect(fetchMock).toHaveBeenCalledWith( + 'http://realtime/api/file-doc/apply-edit', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ 'x-api-key': 'secret' }), + body: JSON.stringify({ fileId: 'file-1', markdown: '# hello' }), + }) + ) + }) + + it('never throws when the realtime call fails (best-effort)', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('socket pod down'))) + await expect(mergeEditIntoLiveFileDoc('file-1', '# hello')).resolves.toBeUndefined() + }) + + it('never throws on a non-2xx response', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, status: 503 })) + await expect(mergeEditIntoLiveFileDoc('file-1', '# hello')).resolves.toBeUndefined() + }) +}) diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts new file mode 100644 index 00000000000..f8aedd6f011 --- /dev/null +++ b/apps/sim/lib/realtime/notify.ts @@ -0,0 +1,141 @@ +import { createLogger } from '@sim/logger' +import { FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' +import { getErrorMessage } from '@sim/utils/errors' +import type { FolderResourceType } from '@/lib/api/contracts/folders' +import { env } from '@/lib/core/config/env' +import { getSocketServerUrl } from '@/lib/core/utils/urls' + +const logger = createLogger('RealtimeNotify') + +/** Bound the wait on the realtime server so a slow/hung socket pod can't stall a file mutation. */ +const NOTIFY_TIMEOUT_MS = 2000 + +/** + * Bound the wait on the live-doc merge. This OUTER call wraps the relay's inner relay→app `/merge` + * request (`FILE_DOC_TIMEOUTS.mergeRequestMs`), so it must stay comfortably ABOVE that — the shared + * constant + its test enforce the ordering. It leaves the inner merge plus the two network hops. + */ +const APPLY_EDIT_TIMEOUT_MS = FILE_DOC_TIMEOUTS.applyEditMs + +/** + * Best-effort fan-out to the realtime server that a workspace's file tree changed, + * so every browser currently viewing that workspace's files refetches. File + * mutations happen over the HTTP API (not the socket); this is a lossy liveness + * signal — a dropped notification only degrades to stale-until-refetch. + * + * Never throws. Callers `await` it (rather than fire-and-forget) so the fetch is + * guaranteed to dispatch before a Node route handler returns — a floating promise + * can be dropped after the response is sent. It is a normally-sub-millisecond + * local call and is hard-bounded to {@link NOTIFY_TIMEOUT_MS}, so it adds that + * latency only when the socket pod is unreachable. + */ +export async function notifyWorkspaceFilesChanged(workspaceId: string): Promise { + try { + const response = await fetch(`${getSocketServerUrl()}/api/workspace-files-changed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ workspaceId }), + signal: AbortSignal.timeout(NOTIFY_TIMEOUT_MS), + }) + if (!response.ok) { + logger.warn('workspace-files-changed notify failed', { + workspaceId, + status: response.status, + }) + } + } catch (error) { + logger.warn('workspace-files-changed notify error', { + workspaceId, + error: getErrorMessage(error), + }) + } +} + +/** + * Best-effort fan-out to the realtime server that a workspace's table list changed (a table was + * created, renamed, moved, deleted, or restored), so every browser currently viewing that + * workspace's tables refetches. The list-level counterpart to {@link notifyWorkspaceFilesChanged}; + * table mutations happen server-side (HTTP routes AND copilot), so this fires from the shared table + * service, not a socket. Lossy — a dropped notification only degrades to stale-until-refetch. + * + * Never throws. Callers `await` it so the fetch is guaranteed to dispatch before the mutation + * returns; hard-bounded to {@link NOTIFY_TIMEOUT_MS}, so it adds that latency only when the socket + * pod is unreachable. + */ +export async function notifyWorkspaceTablesChanged(workspaceId: string): Promise { + try { + const response = await fetch(`${getSocketServerUrl()}/api/workspace-tables-changed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ workspaceId }), + signal: AbortSignal.timeout(NOTIFY_TIMEOUT_MS), + }) + if (!response.ok) { + logger.warn('workspace-tables-changed notify failed', { + workspaceId, + status: response.status, + }) + } + } catch (error) { + logger.warn('workspace-tables-changed notify error', { + workspaceId, + error: getErrorMessage(error), + }) + } +} + +/** + * Folder resource types whose list is kept live by a workspace invalidation room: a folder mutation + * (create/rename/move/delete/restore) for one of these must fan out the same list-changed signal as a + * direct resource mutation, because a new/renamed/removed folder changes what that resource's browser + * shows. Extend this map as more resource lists adopt an invalidation room — `file` and + * `knowledge_base` currently refetch through their own paths, and `workflow` has no such list room. + */ +const FOLDER_RESOURCE_NOTIFIERS: Partial< + Record Promise> +> = { + table: notifyWorkspaceTablesChanged, +} + +/** + * Fan out the workspace live-list signal for a folder mutation, dispatched on the folder's resource + * type. A no-op for resource types without an invalidation room. Never throws (the underlying notify + * is best-effort). Callers `await` it so the dispatch is guaranteed before the mutation returns. + */ +export async function notifyFolderResourceChanged( + resourceType: FolderResourceType, + workspaceId: string +): Promise { + await FOLDER_RESOURCE_NOTIFIERS[resourceType]?.(workspaceId) +} + +/** + * Best-effort: ask the realtime relay to merge a copilot edit into a file's LIVE collaborative + * document, so open editors see it stream in as a CRDT merge (Stage C) rather than the file changing + * underneath them. No-op when no doc is (or was recently) live (the relay reports `applied: false`). + * The file itself is written durably by the caller regardless — this only drives the live view. + * Never throws. + * + * The former clobber gap — an open editor's autosave dropping this edit — is now closed: a + * collaborative editor no longer client-autosaves (the relay persists the shared doc to markdown + * server-side), and the relay applies this merge THROUGH the shared Redis stream, so it reaches the + * live doc on whichever task holds it and can't go stale relative to this direct write. + * + * Awaited (not fire-and-forget) so the fetch dispatches before the route handler returns; bounded to + * {@link APPLY_EDIT_TIMEOUT_MS}, so it adds latency only when the socket pod is unreachable. + */ +export async function mergeEditIntoLiveFileDoc(fileId: string, markdown: string): Promise { + try { + const response = await fetch(`${getSocketServerUrl()}/api/file-doc/apply-edit`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ fileId, markdown }), + signal: AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS), + }) + if (!response.ok) { + logger.warn('file-doc apply-edit failed', { fileId, status: response.status }) + } + } catch (error) { + logger.warn('file-doc apply-edit error', { fileId, error: getErrorMessage(error) }) + } +} diff --git a/apps/sim/lib/table/events.ts b/apps/sim/lib/table/events.ts index c088a28793d..0fda87f6d1f 100644 --- a/apps/sim/lib/table/events.ts +++ b/apps/sim/lib/table/events.ts @@ -1,69 +1,38 @@ /** * Per-table event buffer for live cell-state updates. * - * The grid subscribes to a per-table SSE stream and patches its React Query - * cache as events arrive. This buffer is the durable mid-tier between the - * cell-write paths (`writeWorkflowGroupState`, `cancelWorkflowGroupRuns`) and - * the SSE consumers — every status transition appends here with a monotonic - * eventId; SSE clients resume on reconnect via `?from=` and the - * server replays from this buffer. + * The grid subscribes to a per-table SSE stream and patches its React Query cache + * as events arrive. This is a thin domain adapter over the generic durable event + * log (`@/lib/realtime/event-log`): it owns the Redis key prefix (`table:stream:`) + * and the entry wire shape (`{ eventId, tableId, event }`), and gets append/read/ + * tail + replay + prune semantics from the core. Every status transition appends + * here with a monotonic eventId; SSE clients resume on reconnect via + * `?from=` and the server replays from this buffer. * - * Modeled after `apps/sim/lib/execution/event-buffer.ts` but stripped of - * complexity tables don't need: no per-execution lifecycle, no id reservation - * batching, no write-queue serialization. Tables are always-on; cell writes - * are sparse and independent. + * The `table:stream:` prefix and the entry shape are a wire contract — renaming + * the prefix resets the seq counter and silently strands connected clients (their + * in-memory `lastEventId` no longer matches), so both are intentionally fixed here. */ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { env } from '@/lib/core/config/env' -import { getRedisClient } from '@/lib/core/config/redis' +import { + appendEvent, + type EventLogConfig, + type EventLogReadResult, + getLatestEventId, + readEventsSince, +} from '@/lib/realtime/event-log' -const logger = createLogger('TableEventBuffer') - -const REDIS_PREFIX = 'table:stream:' export const TABLE_EVENT_TTL_SECONDS = 60 * 60 // 1 hour export const TABLE_EVENT_CAP = 5000 /** Max events returned by a single read; the SSE route drains in chunks. */ export const TABLE_EVENT_READ_CHUNK = 500 -/** - * Atomic append: INCR the seq counter to mint a new eventId, build the entry - * JSON inline, ZADD it, refresh TTL on events + seq + meta, trim to cap, then - * write the resulting earliestEventId to meta. Single round-trip per event. - * Without atomicity a slow reader could observe the trim before the meta - * update and miss the prune signal. - * - * KEYS: [events, seq, meta] - * ARGV: [ttlSec, cap, updatedAtIso, entryPrefix, entrySuffix] - * The new eventId is spliced between prefix/suffix to form the entry JSON. - * Returns the new eventId. - */ -const APPEND_EVENT_SCRIPT = ` -local eventId = redis.call('INCR', KEYS[2]) -local entry = ARGV[4] .. eventId .. ARGV[5] -redis.call('ZADD', KEYS[1], eventId, entry) -redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1])) -redis.call('EXPIRE', KEYS[2], tonumber(ARGV[1])) -redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -tonumber(ARGV[2]) - 1) -local oldest = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES') -if oldest[2] then - redis.call('HSET', KEYS[3], 'earliestEventId', tostring(math.floor(tonumber(oldest[2]))), 'updatedAt', ARGV[3]) - redis.call('EXPIRE', KEYS[3], tonumber(ARGV[1])) -end -return eventId -` - -function getEventsKey(tableId: string) { - return `${REDIS_PREFIX}${tableId}:events` -} - -function getSeqKey(tableId: string) { - return `${REDIS_PREFIX}${tableId}:seq` -} - -function getMetaKey(tableId: string) { - return `${REDIS_PREFIX}${tableId}:meta` +/** Wire contract — see the file header; the prefix must never change. */ +const TABLE_EVENT_LOG: EventLogConfig = { + prefix: 'table:stream:', + ttlSeconds: TABLE_EVENT_TTL_SECONDS, + cap: TABLE_EVENT_CAP, + readChunk: TABLE_EVENT_READ_CHUNK, } export type TableCellStatus = 'pending' | 'queued' | 'running' | 'completed' | 'cancelled' | 'error' @@ -141,14 +110,42 @@ export type TableEvent = dispatchId?: string message: string } + | { + /** A user changed row data manually (a cell edit, or an added/deleted row) — + * not an execution. Signals collaborators to refetch the rows so the change + * shows live; last-write-wins is simply the DB's committed order. Carries no + * value: peers refetch in their own wire format, avoiding auth-specific value + * translation on the wire. */ + kind: 'edit' + tableId: string + } + | { + /** A user changed the table's structure (added/updated/deleted a column, or + * renamed the table). Signals collaborators to refetch the table definition + * and rows, since a schema change reshapes how rows render. Value-less, same + * refetch-in-own-format rationale as {@link kind} `edit`. */ + kind: 'schema' + tableId: string + } + | { + /** A user changed the table's UI metadata (column width, pin, or order). Signals + * collaborators to re-apply the new layout live. Lighter than {@link kind} + * `schema`: only the definition carries metadata, so peers refetch the definition + * alone — no rows/run-state refetch. Value-less, same refetch-in-own-format + * rationale as {@link kind} `edit`. */ + kind: 'metadata' + tableId: string + } | { /** The table definition changed in a way that isn't row/cell data — a - * lock toggle or a schema mutation. Carries no payload; the client just - * invalidates the table-detail query so every open viewer re-reads the - * fresh locks/schema (otherwise an idle grid stays stale and writes 423). */ + * lock toggle. Carries no payload; the client just invalidates the + * table-detail query so every open viewer re-reads the fresh locks + * (otherwise an idle grid stays stale and writes 423). Emitted only by + * `updateTableLocks` in the service; schema and metadata changes use the + * dedicated `schema`/`metadata` kinds above. */ kind: 'definition' tableId: string - reason: 'locks' | 'schema' + reason: 'locks' } export interface TableEventEntry { @@ -157,220 +154,72 @@ export interface TableEventEntry { event: TableEvent } -export type TableEventsReadResult = - | { status: 'ok'; events: TableEventEntry[] } - | { status: 'pruned'; earliestEventId: number | undefined } - | { status: 'unavailable'; error: string } - -/** In-memory fallback for dev/tests when Redis isn't configured. */ -interface MemoryTableStream { - events: TableEventEntry[] - earliestEventId?: number - nextEventId: number - expiresAt: number -} - -const memoryTableStreams = new Map() +export type TableEventsReadResult = EventLogReadResult -function canUseMemoryBuffer(): boolean { - return typeof window === 'undefined' && !env.REDIS_URL -} - -function pruneExpiredMemoryStreams(now = Date.now()): void { - for (const [tableId, stream] of memoryTableStreams) { - if (stream.expiresAt <= now) { - memoryTableStreams.delete(tableId) - } - } +/** + * Append an event to the table's buffer. Fire-and-forget — never throws, returns + * null on failure (a Redis blip must not fail a cell-write). The Redis (Lua splice) + * and in-memory paths are built to produce byte-identical entries. + */ +export async function appendTableEvent(event: TableEvent): Promise { + return appendEvent(TABLE_EVENT_LOG, event.tableId, { + entryPrefix: '{"eventId":', + entrySuffix: `,"tableId":${JSON.stringify(event.tableId)},"event":${JSON.stringify(event)}}`, + buildEntry: (eventId) => ({ eventId, tableId: event.tableId, event }), + }) } -function getMemoryStream(tableId: string): MemoryTableStream { - pruneExpiredMemoryStreams() - let stream = memoryTableStreams.get(tableId) - if (!stream) { - stream = { - events: [], - nextEventId: 1, - expiresAt: Date.now() + TABLE_EVENT_TTL_SECONDS * 1000, - } - memoryTableStreams.set(tableId, stream) - } - return stream -} +// The mutating client receives its own signal too (the stream carries no originator id) +// and self-refetches. Data-correct — signals fire after the write commits, so the refetch +// returns the committed state, and in-flight edits are protected by the update/delete +// hooks' cancelQueries. The one caveat is an own row-CREATE on a scrolled, multi-page +// table, which can briefly reshuffle loaded pages (the create hook otherwise skips that +// refetch); if that ever proves visible, stamp an originator id so the actor ignores its +// own signal. -function appendMemory(event: TableEvent): TableEventEntry { - const stream = getMemoryStream(event.tableId) - const entry: TableEventEntry = { - eventId: stream.nextEventId++, - tableId: event.tableId, - event, - } - stream.events.push(entry) - if (stream.events.length > TABLE_EVENT_CAP) { - stream.events = stream.events.slice(-TABLE_EVENT_CAP) - stream.earliestEventId = stream.events[0]?.eventId - } - stream.expiresAt = Date.now() + TABLE_EVENT_TTL_SECONDS * 1000 - return entry +/** + * Signal collaborators that a user changed row data so they refetch the rows live. + * Fire-and-forget — a Redis blip must never fail the write that triggered it. + */ +export function signalTableRowsChanged(tableId: string): void { + void appendTableEvent({ kind: 'edit', tableId }) } -function readMemory(tableId: string, afterEventId: number): TableEventsReadResult { - pruneExpiredMemoryStreams() - const stream = memoryTableStreams.get(tableId) - if (!stream) { - // Mirror the Redis path: a non-zero afterEventId with no buffer at all - // means TTL expired or the stream never existed; either way the caller's - // cursor is stale. - if (afterEventId > 0) return { status: 'pruned', earliestEventId: undefined } - return { status: 'ok', events: [] } - } - if (stream.earliestEventId !== undefined && afterEventId + 1 < stream.earliestEventId) { - return { status: 'pruned', earliestEventId: stream.earliestEventId } - } - return { - status: 'ok', - events: stream.events - .filter((entry) => entry.eventId > afterEventId) - .slice(0, TABLE_EVENT_READ_CHUNK), - } +/** + * Signal collaborators that a user changed the table structure so they refetch the + * definition + rows live. Fire-and-forget for the same reason as + * {@link signalTableRowsChanged}. + */ +export function signalTableSchemaChanged(tableId: string): void { + void appendTableEvent({ kind: 'schema', tableId }) } /** - * Append an event to the table's buffer. Fire-and-forget from the caller — - * this never throws, returns null on failure. A Redis blip must not fail a - * cell-write. + * Signal collaborators that a user changed the table's UI metadata (column width, pin, + * or order) so they re-apply the new layout live. Fire-and-forget for the same reason as + * {@link signalTableRowsChanged}. */ -export async function appendTableEvent(event: TableEvent): Promise { - const redis = getRedisClient() - if (!redis) { - if (canUseMemoryBuffer()) { - try { - return appendMemory(event) - } catch (error) { - logger.warn('appendTableEvent: memory append failed', { - tableId: event.tableId, - error: toError(error).message, - }) - return null - } - } - return null - } - try { - // Build the entry JSON in two halves so Lua can splice the new eventId - // between them without us needing a round-trip just to mint the id first. - const tail = `,"tableId":${JSON.stringify(event.tableId)},"event":${JSON.stringify(event)}}` - const head = `{"eventId":` - const result = await redis.eval( - APPEND_EVENT_SCRIPT, - 3, - getEventsKey(event.tableId), - getSeqKey(event.tableId), - getMetaKey(event.tableId), - TABLE_EVENT_TTL_SECONDS, - TABLE_EVENT_CAP, - new Date().toISOString(), - head, - tail - ) - const eventId = typeof result === 'number' ? result : Number(result) - if (!Number.isFinite(eventId)) return null - return { eventId, tableId: event.tableId, event } - } catch (error) { - logger.warn('appendTableEvent: Redis append failed', { - tableId: event.tableId, - error: toError(error).message, - }) - return null - } +export function signalTableMetadataChanged(tableId: string): void { + void appendTableEvent({ kind: 'metadata', tableId }) } /** * The latest eventId assigned for a table, or 0 when the buffer is empty or * expired. Used by the stream route to tail from "now" when a client connects - * without a replay cursor (fresh mount — its caches were just fetched from - * the DB, so replaying history would only rewind them). - * - * Redis errors propagate: silently falling back to 0 would replay the whole - * buffer over fresh state — the exact churn tail-from-latest exists to avoid. - * The stream route errors the stream instead and the client reconnects with - * backoff. + * without a replay cursor. */ -export async function getLatestTableEventId(tableId: string): Promise { - const redis = getRedisClient() - if (!redis) { - if (canUseMemoryBuffer()) { - // Pure read — getMemoryStream() would allocate a stream as a side effect. - const stream = memoryTableStreams.get(tableId) - return stream ? stream.nextEventId - 1 : 0 - } - return 0 - } - const raw = await redis.get(getSeqKey(tableId)) - if (!raw) return 0 - const parsed = Number.parseInt(raw, 10) - return Number.isFinite(parsed) && parsed > 0 ? parsed : 0 +export function getLatestTableEventId(tableId: string): Promise { + return getLatestEventId(TABLE_EVENT_LOG, tableId) } /** - * Read events for a table where eventId > afterEventId. Returns 'pruned' if - * the caller has fallen off the back of the buffer (TTL expired or cap rolled - * past their lastEventId). Caller should respond by full-refetching from DB - * and resuming streaming from the new earliestEventId. + * Read events for a table where eventId > afterEventId. Returns 'pruned' if the + * caller has fallen off the back of the buffer (TTL expired or cap rolled past + * their lastEventId). */ -export async function readTableEventsSince( +export function readTableEventsSince( tableId: string, afterEventId: number ): Promise { - const redis = getRedisClient() - if (!redis) { - if (canUseMemoryBuffer()) { - return readMemory(tableId, afterEventId) - } - return { status: 'unavailable', error: 'Redis client unavailable' } - } - try { - const meta = await redis.hgetall(getMetaKey(tableId)) - const earliestEventId = - meta?.earliestEventId !== undefined ? Number(meta.earliestEventId) : undefined - if (earliestEventId !== undefined && afterEventId + 1 < earliestEventId) { - return { status: 'pruned', earliestEventId } - } - // Read in capped chunks so a 5000-event backlog doesn't materialize as one - // multi-MB Redis reply + JSON parse + SSE flush. The route loop drains - // chunks across ticks. - const raw = await redis.zrangebyscore( - getEventsKey(tableId), - afterEventId + 1, - '+inf', - 'LIMIT', - 0, - TABLE_EVENT_READ_CHUNK - ) - if (raw.length === 0 && afterEventId > 0) { - // Total TTL expiry: events + meta both gone. The seq counter has the - // same TTL — its absence means the buffer was wiped and the caller's - // `afterEventId` is stale. Signal pruned so the client refetches. - const seqExists = await redis.exists(getSeqKey(tableId)) - if (seqExists === 0) { - return { status: 'pruned', earliestEventId: undefined } - } - } - return { - status: 'ok', - events: raw - .map((entry) => { - try { - return JSON.parse(entry) as TableEventEntry - } catch { - return null - } - }) - .filter((entry): entry is TableEventEntry => Boolean(entry)), - } - } catch (error) { - const message = toError(error).message - logger.warn('readTableEventsSince failed', { tableId, error: message }) - return { status: 'unavailable', error: message } - } + return readEventsSince(TABLE_EVENT_LOG, tableId, afterEventId) } diff --git a/apps/sim/lib/table/service.ts b/apps/sim/lib/table/service.ts index 5d23ec56931..2f5baf0f46c 100644 --- a/apps/sim/lib/table/service.ts +++ b/apps/sim/lib/table/service.ts @@ -17,6 +17,7 @@ import { and, count, eq, isNull, sql } from 'drizzle-orm' import { generateRestoreName } from '@/lib/core/utils/restore-name' import type { DbOrTx } from '@/lib/db/types' import { resolveRestoredFolderId } from '@/lib/folders/queries' +import { notifyWorkspaceTablesChanged } from '@/lib/realtime/notify' import { assertRowCapacity, notifyTableRowUsage } from '@/lib/table/billing' import { generateColumnId, getColumnId, withGeneratedColumnIds } from '@/lib/table/column-keys' import { COLUMN_TYPES, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants' @@ -424,6 +425,9 @@ export async function createTable( logger.info(`[${requestId}] Created table ${tableId} in workspace ${data.workspaceId}`) + // Live tables list: tell everyone viewing this workspace's tables to refetch. + await notifyWorkspaceTablesChanged(data.workspaceId) + return { id: newTable.id, name: newTable.name, @@ -612,6 +616,10 @@ export async function renameTable( } logger.info(`[${requestId}] Renamed table ${tableId} to "${newName}"`) + + // Live tables list: a rename changes the list result, so everyone viewing refetches. + if (workspaceId) await notifyWorkspaceTablesChanged(workspaceId) + return { id: tableId, name: newName } } catch (error: unknown) { if (getPostgresErrorCode(error) === '23505') { @@ -687,6 +695,9 @@ export async function moveTableToFolder( } logger.info(`[${requestId}] Moved table ${tableId} to folder ${folderId ?? 'root'}`) + + // Live tables list: a move changes each table's folder placement in the list result. + await notifyWorkspaceTablesChanged(workspaceId) } /** @@ -759,13 +770,13 @@ export async function updateTableLocks( * @param tableId - Table ID to update * @param metadata - Partial metadata object (merged with existing) * @param existingMetadata - Existing metadata from a prior fetch (avoids redundant DB read) - * @returns Updated metadata + * @returns The merged metadata plus whether the reorder also mutated the schema */ export async function updateTableMetadata( tableId: string, metadata: TableMetadata, existingMetadata?: TableMetadata | null -): Promise { +): Promise<{ metadata: TableMetadata; schemaChanged: boolean }> { const merged: TableMetadata = { ...(existingMetadata ?? {}), ...metadata } // When `columnOrder` is in the patch, scrub any workflow-group dependency @@ -814,7 +825,7 @@ export async function updateTableMetadata( .set(nextSchema ? { metadata: merged, schema: nextSchema } : { metadata: merged }) .where(eq(userTableDefinitions.id, tableId)) - return merged + return { metadata: merged, schemaChanged: nextSchema !== null } } /** @@ -831,7 +842,7 @@ export async function deleteTable( tableId: string, requestId: string, actingUserId?: string, - options?: { archivedAt?: Date } + options?: { archivedAt?: Date; skipNotify?: boolean } ): Promise { const now = options?.archivedAt ?? new Date() // Archiving destroys access to every row, so it is gated on the delete lock. @@ -889,6 +900,12 @@ export async function deleteTable( } logger.info(`[${requestId}] Archived table ${tableId}`) + + // Live tables list: only on a genuine archive (a no-op/already-archived delete changes nothing). + // Skipped under a folder cascade — deleteFolder fires one folder-level notify for the whole subtree, + // so we don't spam one relay call per archived table. + if (deleted?.workspaceId && !options?.skipNotify) + await notifyWorkspaceTablesChanged(deleted.workspaceId) } /** @@ -903,7 +920,7 @@ export async function deleteTable( export async function restoreTable( tableId: string, requestId: string, - options?: { restoringFolderIds?: ReadonlySet } + options?: { restoringFolderIds?: ReadonlySet; skipNotify?: boolean } ): Promise { const table = await getTableById(tableId, { includeArchived: true }) if (!table) { @@ -988,4 +1005,9 @@ export async function restoreTable( } logger.info(`[${requestId}] Restored table ${tableId} as "${attemptedRestoreName}"`) + + // Live tables list: a restore re-adds the table to the active list. Skipped under a folder cascade — + // restoreFolder fires one folder-level notify for the whole subtree (see deleteTable). + if (table.workspaceId && !options?.skipNotify) + await notifyWorkspaceTablesChanged(table.workspaceId) } diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 483ad7f4616..0eac611b3f8 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -22,6 +22,7 @@ import { canonicalWorkspaceFilePath, decodeVfsPathSegments } from '@/lib/copilot import { generateRequestId } from '@/lib/core/utils/request' import { generateRestoreName } from '@/lib/core/utils/restore-name' import type { DbOrTx } from '@/lib/db/types' +import { mergeEditIntoLiveFileDoc, notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { getServePathPrefix } from '@/lib/uploads' import { deleteFile, @@ -31,6 +32,7 @@ import { uploadFile, } from '@/lib/uploads/core/storage-service' import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' +import { isMarkdownFile } from '@/lib/uploads/utils/file-utils' import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' import { isUuid, sanitizeFileName } from '@/executor/constants' import type { UserFile } from '@/executor/types' @@ -378,6 +380,11 @@ export async function uploadWorkspaceFile( const pathPrefix = getServePathPrefix() const serveUrl = `${pathPrefix}${encodeURIComponent(uploadResult.key)}?context=workspace` + // Fan out the live-tree signal for the direct-upload paths (multipart + // fallback, copilot create, /api/files/upload, v1 files) — the presigned + // path already notifies from its register route. + await notifyWorkspaceFilesChanged(workspaceId) + return { id: fileId, name: uniqueName, @@ -905,12 +912,18 @@ export async function resolveWorkspaceFileReference( } /** - * Get a specific workspace file + * Get a specific workspace file. + * + * By default a DB error is logged and swallowed to `null` — for most callers "couldn't load it" + * and "doesn't exist" are handled the same way. Pass `{ throwOnError: true }` when the caller must + * distinguish a genuinely-absent file (`null`) from a transient read failure (throws): the + * collaborative-doc seed builder relies on this so a DB blip never looks like an empty file and gets + * seeded as blank content over the real document. */ export async function getWorkspaceFile( workspaceId: string, fileId: string, - options?: { includeDeleted?: boolean } + options?: { includeDeleted?: boolean; throwOnError?: boolean } ): Promise { try { const { includeDeleted = false } = options ?? {} @@ -938,6 +951,7 @@ export async function getWorkspaceFile( return mapSingleWorkspaceFileRecord(files[0], workspaceId) } catch (error) { logger.error(`Failed to get workspace file ${fileId}:`, error) + if (options?.throwOnError) throw error return null } } @@ -1011,7 +1025,18 @@ export async function updateWorkspaceFileContent( fileId: string, userId: string, content: Buffer, - contentType?: string + contentType?: string, + options?: { + /** + * Whether to stream this write into any open collaborative editor as a live CRDT merge. Defaults + * to `true`, so EVERY external write path (copilot tools, the file tool, the content route) reaches + * an open editor through this one chokepoint — no per-writer wiring to forget. Pass `false` only + * for a write that must NOT touch the live doc: the relay's own project-to-markdown persist (which + * would otherwise merge the doc back into itself in a loop), and empty-shell creates whose real + * content arrives via a subsequent write. + */ + syncLiveDoc?: boolean + } ): Promise { logger.info(`Updating workspace file content: ${fileId} for workspace ${workspaceId}`) @@ -1130,6 +1155,18 @@ export async function updateWorkspaceFileContent( await cleanupWorkspaceStorageObject(finalized.oldKey, 'version replacement') } + // Stream this write into any open collaborative editor as a CRDT merge, so a copilot/tool edit + // shows up live instead of the file silently changing underneath the reader. Gated to markdown (the + // only format the collaborative editor renders) and best-effort (a no-op when nobody has the file + // open; never throws). This is the single chokepoint every external writer shares — the relay's own + // persist and empty-shell creates pass `syncLiveDoc: false` to stay out of it. + if ( + options?.syncLiveDoc !== false && + isMarkdownFile({ type: nextContentType, name: finalized.file.originalName }) + ) { + await mergeEditIntoLiveFileDoc(fileId, content.toString('utf-8')) + } + const pathPrefix = getServePathPrefix() const currentFolderPath = finalized.file.folderId === fileRecord.folderId ? fileRecord.folderPath : null diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts index 9ac40d5587f..138bb8e1293 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts @@ -12,6 +12,8 @@ const { mockHeadObject, mockIncrementStorageUsageForBillingContextInTx, mockMaybeNotifyStorageLimitForBillingContext, + mockMergeEditIntoLiveFileDoc, + mockNotifyWorkspaceFilesChanged, mockResolveStorageBillingContext, mockUploadFile, } = vi.hoisted(() => ({ @@ -22,10 +24,17 @@ const { mockHeadObject: vi.fn(), mockIncrementStorageUsageForBillingContextInTx: vi.fn(), mockMaybeNotifyStorageLimitForBillingContext: vi.fn(), + mockMergeEditIntoLiveFileDoc: vi.fn(), + mockNotifyWorkspaceFilesChanged: vi.fn(), mockResolveStorageBillingContext: vi.fn(), mockUploadFile: vi.fn(), })) +vi.mock('@/lib/realtime/notify', () => ({ + mergeEditIntoLiveFileDoc: mockMergeEditIntoLiveFileDoc, + notifyWorkspaceFilesChanged: mockNotifyWorkspaceFilesChanged, +})) + vi.mock('@/lib/billing/storage', () => ({ decrementStorageUsageForBillingContextInTx: mockDecrementStorageUsageForBillingContextInTx, incrementStorageUsageForBillingContextInTx: mockIncrementStorageUsageForBillingContextInTx, @@ -105,6 +114,8 @@ describe('workspace file metadata and storage accounting', () => { mockDecrementStorageUsageForBillingContextInTx.mockResolvedValue(undefined) mockMaybeNotifyStorageLimitForBillingContext.mockResolvedValue(undefined) mockDeleteFile.mockResolvedValue(undefined) + mockMergeEditIntoLiveFileDoc.mockResolvedValue(undefined) + mockNotifyWorkspaceFilesChanged.mockResolvedValue(undefined) }) it('cleans up a newly uploaded object when atomic metadata finalization rolls back', async () => { @@ -328,4 +339,57 @@ describe('workspace file metadata and storage accounting', () => { expect(mockDeleteFile).toHaveBeenCalledTimes(1) expect(mockDeleteFile).toHaveBeenCalledWith({ key: replacementKey, context: 'workspace' }) }) + + const MD_ROW = { ...FILE_ROW, originalName: 'note.md', contentType: 'text/markdown' } + + it('streams a markdown overwrite into any open collaborative editor (the shared merge chokepoint)', async () => { + const updatedFile = { ...MD_ROW, size: 12 } + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: MD_ROW.key }) + + await updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.from('# new content', 'utf-8') + ) + + expect(mockMergeEditIntoLiveFileDoc).toHaveBeenCalledWith(MD_ROW.id, '# new content') + }) + + it('does NOT merge when syncLiveDoc is false (the relay persist / empty-shell opt-out)', async () => { + const updatedFile = { ...MD_ROW, size: 12 } + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: MD_ROW.key }) + + await updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.from('# new content', 'utf-8'), + undefined, + { syncLiveDoc: false } + ) + + expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + }) + + it('does NOT merge a non-markdown write (the collaborative editor only renders markdown)', async () => { + const updatedFile = { ...FILE_ROW, size: 10 } + dbChainMockFns.limit.mockResolvedValueOnce([FILE_ROW]).mockResolvedValueOnce([FILE_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: FILE_ROW.key }) + + await updateWorkspaceFileContent( + FILE_ROW.workspaceId, + FILE_ROW.id, + FILE_ROW.userId, + Buffer.alloc(10), + 'application/octet-stream' + ) + + expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/uploads/utils/file-utils.test.ts b/apps/sim/lib/uploads/utils/file-utils.test.ts index 7c9e62d9504..c771d7e29fe 100644 --- a/apps/sim/lib/uploads/utils/file-utils.test.ts +++ b/apps/sim/lib/uploads/utils/file-utils.test.ts @@ -8,6 +8,7 @@ import { inferContextFromKey, isAbortError, isInternalFileUrl, + isMarkdownFile, isNetworkError, processSingleFileToUserFile, resolveTrustedFileContext, @@ -15,6 +16,26 @@ import { const logger = createLogger('FileUtilsTest') +describe('isMarkdownFile', () => { + it('is true for .md and .markdown (case-insensitive)', () => { + expect(isMarkdownFile({ name: 'notes.md' })).toBe(true) + expect(isMarkdownFile({ name: 'README.MD' })).toBe(true) + expect(isMarkdownFile({ name: 'doc.markdown' })).toBe(true) + }) + + it('is true for a text/markdown MIME even without a .md name', () => { + expect(isMarkdownFile({ type: 'text/markdown', name: 'notes' })).toBe(true) + expect(isMarkdownFile({ type: 'text/markdown', name: 'doc.txt' })).toBe(true) + }) + + it('is false for non-markdown files', () => { + expect(isMarkdownFile({ type: 'text/javascript', name: 'script.js' })).toBe(false) + expect(isMarkdownFile({ name: 'report.docx' })).toBe(false) + expect(isMarkdownFile({ type: 'text/plain', name: 'notes.txt' })).toBe(false) + expect(isMarkdownFile({ name: 'noext' })).toBe(false) + }) +}) + describe('extractStorageKey', () => { it('strips every provider serve prefix', () => { expect(extractStorageKey('/api/files/serve/s3/workspace%2Fws-1%2Ffile.txt')).toBe( diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index cd36052fdbe..9e331c10854 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -210,6 +210,19 @@ export function getFileExtension(filename: string): string { return lastDot !== -1 ? filename.slice(lastDot + 1).toLowerCase() : '' } +/** + * Whether a file renders in the collaborative rich markdown editor. Server-safe counterpart to the + * client's `isMarkdownFile` (which uses `resolvePreviewType`): the editor treats a file as markdown by + * its `text/markdown` MIME *or* a `.md`/`.markdown` extension — MIME first, matching the client — so a + * `text/markdown` file with a non-`.md` name still counts. Used to gate server work (e.g. the live-doc + * merge) to exactly the files that can be open in that editor. + */ +export function isMarkdownFile(file: { type?: string | null; name: string }): boolean { + if (file.type === 'text/markdown') return true + const ext = getFileExtension(file.name) + return ext === 'md' || ext === 'markdown' +} + /** * Extensions whose stored bytes may be a generation source that renders to a larger * binary. Everything else stores exactly what it serves, so its declared size is diff --git a/apps/sim/lib/workspace-files/orchestration/file-folder-lifecycle.ts b/apps/sim/lib/workspace-files/orchestration/file-folder-lifecycle.ts index ce11157ccb7..4abfd980bcf 100644 --- a/apps/sim/lib/workspace-files/orchestration/file-folder-lifecycle.ts +++ b/apps/sim/lib/workspace-files/orchestration/file-folder-lifecycle.ts @@ -1,6 +1,7 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getPostgresErrorCode, toError } from '@sim/utils/errors' +import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { bulkArchiveWorkspaceFileItems, createWorkspaceFileFolder, @@ -194,6 +195,7 @@ export async function performDeleteWorkspaceFileItems( }) } + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, deletedItems } } catch (error) { logger.error('Failed to delete workspace file items', { error }) @@ -254,6 +256,7 @@ export async function performMoveWorkspaceFileItems( }) } + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, movedItems } } catch (error) { logger.error('Failed to move workspace file items', { error }) @@ -298,6 +301,7 @@ export async function performRenameWorkspaceFile( description: `Renamed file to "${file.name}"`, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, file } } catch (error) { logger.error('Failed to rename workspace file', { error }) @@ -361,6 +365,7 @@ export async function performMoveRenameWorkspaceFile( }) } + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, file } } catch (error) { logger.error('Failed to move/rename workspace file', { error }) @@ -394,6 +399,7 @@ export async function performRestoreWorkspaceFile( description: `Restored workspace file ${fileId}`, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true } } catch (error) { logger.error('Failed to restore workspace file', { error }) @@ -424,6 +430,7 @@ export async function performCreateWorkspaceFileFolder( description: `Created file folder "${folder.name}"`, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, folder } } catch (error) { logger.error('Failed to create workspace file folder', { error }) @@ -463,6 +470,7 @@ export async function performUpdateWorkspaceFileFolder( description: `Updated file folder "${folder.name}"`, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, folder } } catch (error) { logger.error('Failed to update workspace file folder', { error }) @@ -509,6 +517,7 @@ export async function performRestoreWorkspaceFileFolder( }, }) + await notifyWorkspaceFilesChanged(workspaceId) return { success: true, folder, restoredItems } } catch (error) { logger.error('Failed to restore workspace file folder', { error }) diff --git a/apps/sim/lib/workspaces/colors.ts b/apps/sim/lib/workspaces/colors.ts index 61401b68ad1..87b1fa34be8 100644 --- a/apps/sim/lib/workspaces/colors.ts +++ b/apps/sim/lib/workspaces/colors.ts @@ -17,15 +17,6 @@ export function getRandomWorkspaceColor(): string { return randomItem(WORKSPACE_COLORS) } -const APP_COLORS = [ - { from: '#4F46E5', to: '#7C3AED' }, // indigo to purple - { from: '#7C3AED', to: '#C026D3' }, // purple to fuchsia - { from: '#EC4899', to: '#F97316' }, // pink to orange - { from: '#14B8A6', to: '#10B981' }, // teal to emerald - { from: '#6366F1', to: '#8B5CF6' }, // indigo to violet - { from: '#F59E0B', to: '#F97316' }, // amber to orange -] - /** * User color palette matching terminal.tsx RUN_ID_COLORS * These colors are used consistently across cursors, avatars, and terminal run IDs @@ -39,12 +30,6 @@ export const USER_COLORS = [ '#FCD34D', // Yellow ] as const -interface PresenceColorPalette { - gradient: string - accentColor: string - baseColor: string -} - const HEX_COLOR_REGEX = /^#(?:[0-9a-fA-F]{3}){1,2}$/ function hashIdentifier(identifier: string | number): number { @@ -59,7 +44,7 @@ function hashIdentifier(identifier: string | number): number { return 0 } -function withAlpha(hexColor: string, alpha: number): string { +export function withAlpha(hexColor: string, alpha: number): string { if (!HEX_COLOR_REGEX.test(hexColor)) { return hexColor } @@ -68,39 +53,6 @@ function withAlpha(hexColor: string, alpha: number): string { return `rgba(${r}, ${g}, ${b}, ${Math.min(Math.max(alpha, 0), 1)})` } -function buildGradient(fromColor: string, toColor: string, rotationSeed: number): string { - const rotation = (rotationSeed * 25) % 360 - return `linear-gradient(${rotation}deg, ${fromColor}, ${toColor})` -} - -export function getPresenceColors( - identifier: string | number, - explicitColor?: string -): PresenceColorPalette { - const paletteIndex = hashIdentifier(identifier) - - if (explicitColor) { - const normalizedColor = explicitColor.trim() - const lighterShade = HEX_COLOR_REGEX.test(normalizedColor) - ? withAlpha(normalizedColor, 0.85) - : normalizedColor - - return { - gradient: buildGradient(lighterShade, normalizedColor, paletteIndex), - accentColor: normalizedColor, - baseColor: lighterShade, - } - } - - const colorPair = APP_COLORS[paletteIndex % APP_COLORS.length] - - return { - gradient: buildGradient(colorPair.from, colorPair.to, paletteIndex), - accentColor: colorPair.to, - baseColor: colorPair.from, - } -} - /** * Gets a consistent color for a user based on their ID. * The same user will always get the same color across cursors, avatars, and terminal. @@ -112,23 +64,3 @@ export function getUserColor(userId: string): string { const hash = hashIdentifier(userId) return USER_COLORS[hash % USER_COLORS.length] } - -/** - * Creates a stable mapping of user IDs to color indices for a list of users. - * Useful when you need to maintain consistent color assignments across renders. - * - * @param userIds - Array of user IDs to map - * @returns Map of user ID to color index - */ -export function createUserColorMap(userIds: string[]): Map { - const colorMap = new Map() - let colorIndex = 0 - - for (const userId of userIds) { - if (!colorMap.has(userId)) { - colorMap.set(userId, colorIndex++) - } - } - - return colorMap -} diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index eba51b37fd7..1e5ed4245e6 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -132,9 +132,41 @@ const nextConfig: NextConfig = { '@daytona/sdk', '@earendil-works/pi-ai', '@earendil-works/pi-coding-agent', + // The collab-doc seed converter lazily `require`s jsdom for a headless TipTap editor. Keep it + // external so webpack doesn't try to bundle jsdom's dynamic internal requires. + 'jsdom', + // The collab-doc converter runs TipTap + Yjs headlessly server-side. Two reasons these must be + // external (native Node require), not bundled: (1) the server bundler gives bundled TipTap a + // `window` that does NOT read `globalThis`, so `elementFromString` throws "no window object" even + // after the converter installs a jsdom window; (2) bundling would load a SECOND copy of `yjs`, so + // `@tiptap/y-tiptap`'s `item instanceof Y.XmlElement` checks — against the external `yjs` — would + // fail on nodes the app created with the bundled `yjs` ("Unexpected case"). One external copy fixes + // both. Server-only — the client editor bundles its own copies for the browser. + 'yjs', + 'y-protocols', + 'lib0', + '@tiptap/core', + '@tiptap/pm', + '@tiptap/markdown', + '@tiptap/y-tiptap', + '@tiptap/starter-kit', + '@tiptap/extension-code', + '@tiptap/extension-code-block', + '@tiptap/extension-image', + '@tiptap/extension-list', + '@tiptap/extension-paragraph', + '@tiptap/extension-table', + '@tiptap/extension-highlight', ], outputFileTracingIncludes: { '/api/tools/stagehand/*': ['./node_modules/ws/**/*'], + // The seed, merge, and persist endpoints all lazily `require('jsdom')` (via the collab-doc + // converter), which is invisible to the standalone file tracer, so force jsdom (and its transitive + // deps, followed from its static requires) into the trace — otherwise a Docker/standalone build + // omits it and the endpoint 500s with MODULE_NOT_FOUND. + '/api/internal/file-doc/seed': ['./node_modules/jsdom/**/*'], + '/api/internal/file-doc/merge': ['./node_modules/jsdom/**/*'], + '/api/internal/file-doc/persist': ['./node_modules/jsdom/**/*'], '/*': [ './node_modules/sharp/**/*', './node_modules/@img/**/*', diff --git a/apps/sim/package.json b/apps/sim/package.json index d2f851743a2..ce019f0ab5e 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -124,6 +124,8 @@ "@tanstack/react-virtual": "3.13.24", "@tiptap/core": "3.26.1", "@tiptap/extension-code-block": "3.26.1", + "@tiptap/extension-collaboration": "3.26.1", + "@tiptap/extension-collaboration-caret": "3.26.1", "@tiptap/extension-image": "3.26.1", "@tiptap/extension-list": "3.26.1", "@tiptap/extension-placeholder": "3.26.1", @@ -133,6 +135,7 @@ "@tiptap/react": "3.26.1", "@tiptap/starter-kit": "3.26.1", "@tiptap/suggestion": "3.26.1", + "@tiptap/y-tiptap": "3.0.7", "@trigger.dev/sdk": "4.4.3", "@typescript/typescript6": "^6.0.2", "@xterm/addon-fit": "0.11.0", @@ -178,7 +181,9 @@ "jose": "6.0.11", "js-tiktoken": "1.0.21", "js-yaml": "4.3.0", + "jsdom": "^26.0.0", "jszip": "3.10.1", + "lib0": "0.2.117", "lru-cache": "11.3.6", "lucide-react": "^0.511.0", "mammoth": "^1.9.0", @@ -230,6 +235,8 @@ "undici": "7.28.0", "unpdf": "1.4.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", + "y-protocols": "1.0.7", + "yjs": "13.6.31", "zod": "4.3.6", "zustand": "^5.0.13" }, @@ -256,7 +263,6 @@ "@typescript/native-preview": "7.0.0-dev.20260707.2", "@vitejs/plugin-react": "^4.3.4", "@vitest/coverage-v8": "^4.1.0", - "jsdom": "^26.0.0", "postcss": "^8", "react-email": "6.9.0", "tailwindcss": "^3.4.1", diff --git a/bun.lock b/bun.lock index 3322e254811..04fd870670a 100644 --- a/bun.lock +++ b/bun.lock @@ -121,9 +121,12 @@ "@sim/workflow-types": "workspace:*", "@socket.io/redis-adapter": "8.3.0", "drizzle-orm": "^0.45.2", + "lib0": "0.2.117", "postgres": "^3.4.5", "redis": "5.10.0", "socket.io": "^4.8.1", + "y-protocols": "1.0.7", + "yjs": "13.6.31", "zod": "4.3.6", }, "devDependencies": { @@ -228,6 +231,8 @@ "@tanstack/react-virtual": "3.13.24", "@tiptap/core": "3.26.1", "@tiptap/extension-code-block": "3.26.1", + "@tiptap/extension-collaboration": "3.26.1", + "@tiptap/extension-collaboration-caret": "3.26.1", "@tiptap/extension-image": "3.26.1", "@tiptap/extension-list": "3.26.1", "@tiptap/extension-placeholder": "3.26.1", @@ -237,6 +242,7 @@ "@tiptap/react": "3.26.1", "@tiptap/starter-kit": "3.26.1", "@tiptap/suggestion": "3.26.1", + "@tiptap/y-tiptap": "3.0.7", "@trigger.dev/sdk": "4.4.3", "@typescript/typescript6": "^6.0.2", "@xterm/addon-fit": "0.11.0", @@ -282,7 +288,9 @@ "jose": "6.0.11", "js-tiktoken": "1.0.21", "js-yaml": "4.3.0", + "jsdom": "^26.0.0", "jszip": "3.10.1", + "lib0": "0.2.117", "lru-cache": "11.3.6", "lucide-react": "^0.511.0", "mammoth": "^1.9.0", @@ -334,6 +342,8 @@ "undici": "7.28.0", "unpdf": "1.4.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", + "y-protocols": "1.0.7", + "yjs": "13.6.31", "zod": "4.3.6", "zustand": "^5.0.13", }, @@ -360,7 +370,6 @@ "@typescript/native-preview": "7.0.0-dev.20260707.2", "@vitejs/plugin-react": "^4.3.4", "@vitest/coverage-v8": "^4.1.0", - "jsdom": "^26.0.0", "postcss": "^8", "react-email": "6.9.0", "tailwindcss": "^3.4.1", @@ -533,6 +542,7 @@ "version": "0.1.0", "dependencies": { "@sim/db": "workspace:*", + "@sim/realtime-protocol": "workspace:*", "drizzle-orm": "^0.45.2", }, "devDependencies": { @@ -550,6 +560,7 @@ "devDependencies": { "@sim/tsconfig": "workspace:*", "typescript": "^7.0.2", + "vitest": "^4.1.0", }, }, "packages/runtime-secrets": { @@ -1931,6 +1942,10 @@ "@tiptap/extension-code-block": ["@tiptap/extension-code-block@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1", "@tiptap/pm": "3.26.1" } }, "sha512-NY7SYqcrqDVYTSWyaNGdSfCims6pOHoRQ2Rh4DEFb/rb8gLVkqbLZhcHzQCVfinlPqgV3xWF6cYMORwmnlBkXQ=="], + "@tiptap/extension-collaboration": ["@tiptap/extension-collaboration@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1", "@tiptap/pm": "3.26.1", "@tiptap/y-tiptap": "^3.0.5", "yjs": "^13" } }, "sha512-NLF3tWPla1bg9VAaICTrheEjDi9ZFGk1HhoALfVHWSwIqnUpVxSubyBxw/PFWyHYZNQrxaGvypf457NHHINolA=="], + + "@tiptap/extension-collaboration-caret": ["@tiptap/extension-collaboration-caret@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1", "@tiptap/pm": "3.26.1", "@tiptap/y-tiptap": "^3.0.5" } }, "sha512-fJpIO8eXYksiTyxck3e7vS2Fsddo+sJNaQdNHXuDykVZzH4LPXAHyGGdSRrLM4aMUsK1IZh8CPyti1CHwbz7vA=="], + "@tiptap/extension-document": ["@tiptap/extension-document@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1" } }, "sha512-6W2vZjvi0Mv+4xEtwMDGhWwo7FotWR6eKfmntmduvehWevFpMxOKcTtyotjLigfZv738y50YWmvbaPuAPJG3BA=="], "@tiptap/extension-dropcursor": ["@tiptap/extension-dropcursor@3.26.1", "", { "peerDependencies": { "@tiptap/extensions": "3.26.1" } }, "sha512-eVq3BvFIa3YD+pBIlj1i72vYEixlegGVKHnSYiVF2ovkQOSAH9sca7pkq6WgV1sMTCyWCU8e+WznTqtydvHUWA=="], @@ -1983,6 +1998,8 @@ "@tiptap/suggestion": ["@tiptap/suggestion@3.26.1", "", { "peerDependencies": { "@tiptap/core": "3.26.1", "@tiptap/pm": "3.26.1" } }, "sha512-Bg8IyuDC92InSPzcHvCT3+ZDCJSMJIEINdFg513RPQzwZTw1dsrU0K00XYcDT6lOhZwLM2IVTiE6sZl2GY25Rg=="], + "@tiptap/y-tiptap": ["@tiptap/y-tiptap@3.0.7", "", { "dependencies": { "lib0": "^0.2.100" }, "peerDependencies": { "prosemirror-model": "^1.7.1", "prosemirror-state": "^1.2.3", "prosemirror-view": "^1.9.10", "y-protocols": "^1.0.1", "yjs": "^13.5.38" } }, "sha512-3VG01F7i2JDghWsBZSBKi7ypMiN5UVVSyD18IwoXx7ilm0ZynrTS+XNpmgxfuiSBjdauWk7tUlP04xfM8Bw4Vw=="], + "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], "@tootallnate/once": ["@tootallnate/once@2.0.1", "", {}, "sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ=="], @@ -3245,6 +3262,8 @@ "isomorphic-ws": ["isomorphic-ws@5.0.0", "", { "peerDependencies": { "ws": "*" } }, "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw=="], + "isomorphic.js": ["isomorphic.js@0.2.5", "", {}, "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw=="], + "istanbul-lib-coverage": ["istanbul-lib-coverage@3.2.2", "", {}, "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg=="], "istanbul-lib-report": ["istanbul-lib-report@3.0.1", "", { "dependencies": { "istanbul-lib-coverage": "^3.0.0", "make-dir": "^4.0.0", "supports-color": "^7.1.0" } }, "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw=="], @@ -3321,6 +3340,8 @@ "leac": ["leac@0.6.0", "", {}, "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg=="], + "lib0": ["lib0@0.2.117", "", { "dependencies": { "isomorphic.js": "^0.2.4" }, "bin": { "0serve": "bin/0serve.js", "0gentesthtml": "bin/gentesthtml.js", "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js" } }, "sha512-DeXj9X5xDCjgKLU/7RR+/HQEVzuuEUiwldwOGsHK/sfAfELGWEyTcf0x+uOvCvK3O2zPmZePXWL85vtia6GyZw=="], + "libbase64": ["libbase64@1.3.0", "", {}, "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg=="], "libmime": ["libmime@5.3.7", "", { "dependencies": { "encoding-japanese": "2.2.0", "iconv-lite": "0.6.3", "libbase64": "1.3.0", "libqp": "2.1.1" } }, "sha512-FlDb3Wtha8P01kTL3P9M+ZDNDWPKPmKHWaU/cG/lg5pfuAwdflVpZE+wm9m7pKmC5ww6s+zTxBKS1p6yl3KpSw=="], @@ -4551,6 +4572,8 @@ "xpath": ["xpath@0.0.34", "", {}, "sha512-FxF6+rkr1rNSQrhUNYrAFJpRXNzlDoMxeXN5qI84939ylEv3qqPFKa85Oxr6tDaJKqwW6KKyo2v26TSv3k6LeA=="], + "y-protocols": ["y-protocols@1.0.7", "", { "dependencies": { "lib0": "^0.2.85" }, "peerDependencies": { "yjs": "^13.0.0" } }, "sha512-YSVsLoXxO67J6eE/nV4AtFtT3QEotZf5sK5BHxFBXso7VDUT3Tx07IfA6hsu5Q5OmBdMkQVmFZ9QOA7fikWvnw=="], + "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], "yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="], @@ -4563,6 +4586,8 @@ "yauzl": ["yauzl@3.4.0", "", { "dependencies": { "pend": "~1.2.0" } }, "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw=="], + "yjs": ["yjs@13.6.31", "", { "dependencies": { "lib0": "^0.2.99" } }, "sha512-Eq+5BRfbeGyqGVrTJL3bEcr8gKkxPuyuoHmAwpk52fDb8kOVMrfVSTRPd6yiGgX5Fskb96qCRjzjbRjrL4YEnw=="], + "yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], "yoctocolors": ["yoctocolors@2.1.2", "", {}, "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug=="], diff --git a/packages/db/migrations/0277_workspace_file_collab_state.sql b/packages/db/migrations/0277_workspace_file_collab_state.sql new file mode 100644 index 00000000000..a40748242cc --- /dev/null +++ b/packages/db/migrations/0277_workspace_file_collab_state.sql @@ -0,0 +1,8 @@ +CREATE TABLE "workspace_file_collab_state" ( + "file_id" text PRIMARY KEY NOT NULL, + "doc_state" "bytea" NOT NULL, + "source_hash" text NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "workspace_file_collab_state" ADD CONSTRAINT "workspace_file_collab_state_file_id_workspace_files_id_fk" FOREIGN KEY ("file_id") REFERENCES "public"."workspace_files"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/db/migrations/meta/0277_snapshot.json b/packages/db/migrations/meta/0277_snapshot.json new file mode 100644 index 00000000000..66b90a36714 --- /dev/null +++ b/packages/db/migrations/meta/0277_snapshot.json @@ -0,0 +1,18028 @@ +{ + "id": "27607581-9ffb-4b7d-89a5-eaebe9f7d0d1", + "prevId": "100afbc7-1a7f-4f05-9950-b2dbd6996c15", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.academy_certificate": { + "name": "academy_certificate", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "course_id": { + "name": "course_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "academy_cert_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "certificate_number": { + "name": "certificate_number", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "academy_certificate_user_id_idx": { + "name": "academy_certificate_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_course_id_idx": { + "name": "academy_certificate_course_id_idx", + "columns": [ + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_user_course_unique": { + "name": "academy_certificate_user_course_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_number_idx": { + "name": "academy_certificate_number_idx", + "columns": [ + { + "expression": "certificate_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_status_idx": { + "name": "academy_certificate_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "academy_certificate_user_id_user_id_fk": { + "name": "academy_certificate_user_id_user_id_fk", + "tableFrom": "academy_certificate", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "academy_certificate_certificate_number_unique": { + "name": "academy_certificate_certificate_number_unique", + "nullsNotDistinct": false, + "columns": ["certificate_number"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_account_on_account_id_provider_id": { + "name": "idx_account_on_account_id_provider_id", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_key": { + "name": "api_key", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'personal'" + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_key_workspace_type_idx": { + "name": "api_key_workspace_type_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_user_type_idx": { + "name": "api_key_user_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_key_hash_idx": { + "name": "api_key_key_hash_idx", + "columns": [ + { + "expression": "key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_key_user_id_user_id_fk": { + "name": "api_key_user_id_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_workspace_id_workspace_id_fk": { + "name": "api_key_workspace_id_workspace_id_fk", + "tableFrom": "api_key", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_created_by_user_id_fk": { + "name": "api_key_created_by_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_key_key_unique": { + "name": "api_key_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": { + "workspace_type_check": { + "name": "workspace_type_check", + "value": "(type = 'workspace' AND workspace_id IS NOT NULL) OR (type = 'personal' AND workspace_id IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.async_jobs": { + "name": "async_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "run_at": { + "name": "run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output": { + "name": "output", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "async_jobs_status_started_at_idx": { + "name": "async_jobs_status_started_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_status_completed_at_idx": { + "name": "async_jobs_status_completed_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "completed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_pending_run_at_idx": { + "name": "async_jobs_schedule_pending_run_at_idx", + "columns": [ + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_processing_started_at_idx": { + "name": "async_jobs_schedule_processing_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'processing'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_name": { + "name": "actor_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_email": { + "name": "actor_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource_name": { + "name": "resource_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_log_workspace_created_idx": { + "name": "audit_log_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_workspace_created_at_id_idx": { + "name": "audit_log_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_actor_created_idx": { + "name": "audit_log_actor_created_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_resource_idx": { + "name": "audit_log_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_action_idx": { + "name": "audit_log_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "audit_log_workspace_id_workspace_id_fk": { + "name": "audit_log_workspace_id_workspace_id_fk", + "tableFrom": "audit_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "audit_log_actor_id_user_id_fk": { + "name": "audit_log_actor_id_user_id_fk", + "tableFrom": "audit_log", + "tableTo": "user", + "columnsFrom": ["actor_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.background_work_status": { + "name": "background_work_status", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "background_work_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "background_work_status_value", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "background_work_status_workspace_status_idx": { + "name": "background_work_status_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_workflow_status_idx": { + "name": "background_work_status_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_child_ws_idx": { + "name": "background_work_status_meta_child_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'childWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_other_ws_idx": { + "name": "background_work_status_meta_other_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'otherWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "background_work_status_workspace_id_workspace_id_fk": { + "name": "background_work_status_workspace_id_workspace_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "background_work_status_workflow_id_workflow_id_fk": { + "name": "background_work_status_workflow_id_workflow_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat": { + "name": "chat", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "customizations": { + "name": "customizations", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "output_configs": { + "name": "output_configs", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "include_thinking": { + "name": "include_thinking", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "include_tool_calls": { + "name": "include_tool_calls", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "identifier_idx": { + "name": "identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"chat\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_archived_at_partial_idx": { + "name": "chat_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"chat\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_chat_on_workflow_id_archived_at": { + "name": "idx_chat_on_workflow_id_archived_at", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_workflow_id_workflow_id_fk": { + "name": "chat_workflow_id_workflow_id_fk", + "tableFrom": "chat", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_user_id_user_id_fk": { + "name": "chat_user_id_user_id_fk", + "tableFrom": "chat", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_async_tool_calls": { + "name": "copilot_async_tool_calls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "checkpoint_id": { + "name": "checkpoint_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "args": { + "name": "args", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "status": { + "name": "status", + "type": "copilot_async_tool_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "permission_decision": { + "name": "permission_decision", + "type": "copilot_tool_permission_decision", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "permission_decided_at": { + "name": "permission_decided_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_async_tool_calls_run_id_idx": { + "name": "copilot_async_tool_calls_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_checkpoint_id_idx": { + "name": "copilot_async_tool_calls_checkpoint_id_idx", + "columns": [ + { + "expression": "checkpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_tool_call_id_idx": { + "name": "copilot_async_tool_calls_tool_call_id_idx", + "columns": [ + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_status_idx": { + "name": "copilot_async_tool_calls_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_run_status_idx": { + "name": "copilot_async_tool_calls_run_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_tool_call_id_unique": { + "name": "copilot_async_tool_calls_tool_call_id_unique", + "columns": [ + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_async_tool_calls_run_id_copilot_runs_id_fk": { + "name": "copilot_async_tool_calls_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk": { + "name": "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_run_checkpoints", + "columnsFrom": ["checkpoint_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_chats": { + "name": "copilot_chats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "chat_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'copilot'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'claude-3-7-sonnet-latest'" + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preview_yaml": { + "name": "preview_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan_artifact": { + "name": "plan_artifact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "auto_allowed_tools": { + "name": "auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "pinned": { + "name": "pinned", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_chats_user_id_idx": { + "name": "copilot_chats_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workflow_id_idx": { + "name": "copilot_chats_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workflow_idx": { + "name": "copilot_chats_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_idx": { + "name": "copilot_chats_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_created_at_idx": { + "name": "copilot_chats_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_updated_at_idx": { + "name": "copilot_chats_updated_at_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workspace_created_at_id_idx": { + "name": "copilot_chats_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_deleted_partial_idx": { + "name": "copilot_chats_user_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_chats\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_chats_user_id_user_id_fk": { + "name": "copilot_chats_user_id_user_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workflow_id_workflow_id_fk": { + "name": "copilot_chats_workflow_id_workflow_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workspace_id_workspace_id_fk": { + "name": "copilot_chats_workspace_id_workspace_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_feedback": { + "name": "copilot_feedback", + "schema": "", + "columns": { + "feedback_id": { + "name": "feedback_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_query": { + "name": "user_query", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_response": { + "name": "agent_response", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_positive": { + "name": "is_positive", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "feedback": { + "name": "feedback", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_yaml": { + "name": "workflow_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_feedback_user_id_idx": { + "name": "copilot_feedback_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_chat_id_idx": { + "name": "copilot_feedback_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_user_chat_idx": { + "name": "copilot_feedback_user_chat_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_is_positive_idx": { + "name": "copilot_feedback_is_positive_idx", + "columns": [ + { + "expression": "is_positive", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_created_at_idx": { + "name": "copilot_feedback_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_feedback_user_id_user_id_fk": { + "name": "copilot_feedback_user_id_user_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_feedback_chat_id_copilot_chats_id_fk": { + "name": "copilot_feedback_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_messages": { + "name": "copilot_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parent_message_id": { + "name": "parent_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens_in": { + "name": "tokens_in", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "tokens_out": { + "name": "tokens_out", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_messages_chat_message_unique": { + "name": "copilot_messages_chat_message_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_created_at_idx": { + "name": "copilot_messages_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_seq_idx": { + "name": "copilot_messages_chat_seq_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_stream_idx": { + "name": "copilot_messages_chat_stream_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"stream_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_user_created_at_idx": { + "name": "copilot_messages_user_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"role\" = 'user' AND \"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_messages_chat_id_copilot_chats_id_fk": { + "name": "copilot_messages_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_messages", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_run_checkpoints": { + "name": "copilot_run_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "pending_tool_call_id": { + "name": "pending_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conversation_snapshot": { + "name": "conversation_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "agent_state": { + "name": "agent_state", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "provider_request": { + "name": "provider_request", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_run_checkpoints_run_id_idx": { + "name": "copilot_run_checkpoints_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_pending_tool_call_id_idx": { + "name": "copilot_run_checkpoints_pending_tool_call_id_idx", + "columns": [ + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_run_pending_tool_unique": { + "name": "copilot_run_checkpoints_run_pending_tool_unique", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_run_checkpoints_run_id_copilot_runs_id_fk": { + "name": "copilot_run_checkpoints_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_run_checkpoints", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_runs": { + "name": "copilot_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_run_id": { + "name": "parent_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent": { + "name": "agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "copilot_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "request_context": { + "name": "request_context", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "copilot_runs_execution_id_idx": { + "name": "copilot_runs_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_parent_run_id_idx": { + "name": "copilot_runs_parent_run_id_idx", + "columns": [ + { + "expression": "parent_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_id_idx": { + "name": "copilot_runs_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_user_id_idx": { + "name": "copilot_runs_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workflow_id_idx": { + "name": "copilot_runs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_id_idx": { + "name": "copilot_runs_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_status_idx": { + "name": "copilot_runs_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_execution_idx": { + "name": "copilot_runs_chat_execution_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_execution_started_at_idx": { + "name": "copilot_runs_execution_started_at_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_completed_at_id_idx": { + "name": "copilot_runs_workspace_completed_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"completed_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_stream_id_unique": { + "name": "copilot_runs_stream_id_unique", + "columns": [ + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_runs_chat_id_copilot_chats_id_fk": { + "name": "copilot_runs_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_user_id_user_id_fk": { + "name": "copilot_runs_user_id_user_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workflow_id_workflow_id_fk": { + "name": "copilot_runs_workflow_id_workflow_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workspace_id_workspace_id_fk": { + "name": "copilot_runs_workspace_id_workspace_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_workflow_read_hashes": { + "name": "copilot_workflow_read_hashes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_workflow_read_hashes_chat_id_idx": { + "name": "copilot_workflow_read_hashes_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_workflow_id_idx": { + "name": "copilot_workflow_read_hashes_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_chat_workflow_unique": { + "name": "copilot_workflow_read_hashes_chat_workflow_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk": { + "name": "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_workflow_read_hashes_workflow_id_workflow_id_fk": { + "name": "copilot_workflow_read_hashes_workflow_id_workflow_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credential": { + "name": "credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "credential_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_key": { + "name": "env_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_owner_user_id": { + "name": "env_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_service_account_key": { + "name": "encrypted_service_account_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_workspace_id_idx": { + "name": "credential_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_type_idx": { + "name": "credential_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_provider_id_idx": { + "name": "credential_provider_id_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_account_id_idx": { + "name": "credential_account_id_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_env_owner_user_id_idx": { + "name": "credential_env_owner_user_id_idx", + "columns": [ + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_account_unique": { + "name": "credential_workspace_account_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "account_id IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_env_unique": { + "name": "credential_workspace_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_workspace'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_personal_env_unique": { + "name": "credential_workspace_personal_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_personal'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_workspace_id_workspace_id_fk": { + "name": "credential_workspace_id_workspace_id_fk", + "tableFrom": "credential", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_account_id_account_id_fk": { + "name": "credential_account_id_account_id_fk", + "tableFrom": "credential", + "tableTo": "account", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_env_owner_user_id_user_id_fk": { + "name": "credential_env_owner_user_id_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["env_owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_created_by_user_id_fk": { + "name": "credential_created_by_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "credential_oauth_source_check": { + "name": "credential_oauth_source_check", + "value": "(type <> 'oauth') OR (account_id IS NOT NULL AND provider_id IS NOT NULL)" + }, + "credential_workspace_env_source_check": { + "name": "credential_workspace_env_source_check", + "value": "(type <> 'env_workspace') OR (env_key IS NOT NULL AND env_owner_user_id IS NULL)" + }, + "credential_personal_env_source_check": { + "name": "credential_personal_env_source_check", + "value": "(type <> 'env_personal') OR (env_key IS NOT NULL AND env_owner_user_id IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.credential_member": { + "name": "credential_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "credential_member_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "status": { + "name": "status", + "type": "credential_member_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "joined_at": { + "name": "joined_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_member_user_id_idx": { + "name": "credential_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_role_idx": { + "name": "credential_member_role_idx", + "columns": [ + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_status_idx": { + "name": "credential_member_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_unique": { + "name": "credential_member_unique", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_member_credential_id_credential_id_fk": { + "name": "credential_member_credential_id_credential_id_fk", + "tableFrom": "credential_member", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_user_id_user_id_fk": { + "name": "credential_member_user_id_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_invited_by_user_id_fk": { + "name": "credential_member_invited_by_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_block": { + "name": "custom_block", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inputs": { + "name": "inputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "outputs": { + "name": "outputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_block_organization_id_idx": { + "name": "custom_block_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_workflow_id_idx": { + "name": "custom_block_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_organization_type_unique": { + "name": "custom_block_organization_type_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_block_organization_id_organization_id_fk": { + "name": "custom_block_organization_id_organization_id_fk", + "tableFrom": "custom_block", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_workflow_id_workflow_id_fk": { + "name": "custom_block_workflow_id_workflow_id_fk", + "tableFrom": "custom_block", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_created_by_user_id_fk": { + "name": "custom_block_created_by_user_id_fk", + "tableFrom": "custom_block", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_tools": { + "name": "custom_tools", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_tools_workspace_id_idx": { + "name": "custom_tools_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_tools_workspace_title_unique": { + "name": "custom_tools_workspace_title_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "title", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_tools_workspace_id_workspace_id_fk": { + "name": "custom_tools_workspace_id_workspace_id_fk", + "tableFrom": "custom_tools", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_tools_user_id_user_id_fk": { + "name": "custom_tools_user_id_user_id_fk", + "tableFrom": "custom_tools", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drain_runs": { + "name": "data_drain_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "drain_id": { + "name": "drain_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "data_drain_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "trigger": { + "name": "trigger", + "type": "data_drain_run_trigger", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rows_exported": { + "name": "rows_exported", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bytes_written": { + "name": "bytes_written", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "cursor_before": { + "name": "cursor_before", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cursor_after": { + "name": "cursor_after", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locators": { + "name": "locators", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + } + }, + "indexes": { + "data_drain_runs_drain_started_idx": { + "name": "data_drain_runs_drain_started_idx", + "columns": [ + { + "expression": "drain_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drain_runs_drain_id_data_drains_id_fk": { + "name": "data_drain_runs_drain_id_data_drains_id_fk", + "tableFrom": "data_drain_runs", + "tableTo": "data_drains", + "columnsFrom": ["drain_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drains": { + "name": "data_drains", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "data_drain_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_type": { + "name": "destination_type", + "type": "data_drain_destination", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_config": { + "name": "destination_config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "destination_credentials": { + "name": "destination_credentials", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schedule_cadence": { + "name": "schedule_cadence", + "type": "data_drain_cadence", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_drains_org_idx": { + "name": "data_drains_org_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_due_idx": { + "name": "data_drains_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_org_name_unique": { + "name": "data_drains_org_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drains_organization_id_organization_id_fk": { + "name": "data_drains_organization_id_organization_id_fk", + "tableFrom": "data_drains", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_drains_created_by_user_id_fk": { + "name": "data_drains_created_by_user_id_fk", + "tableFrom": "data_drains", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.docs_embeddings": { + "name": "docs_embeddings", + "schema": "", + "columns": { + "chunk_id": { + "name": "chunk_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chunk_text": { + "name": "chunk_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_document": { + "name": "source_document", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_link": { + "name": "source_link", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_text": { + "name": "header_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_level": { + "name": "header_level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": true + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "chunk_text_tsv": { + "name": "chunk_text_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"docs_embeddings\".\"chunk_text\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "docs_emb_source_document_idx": { + "name": "docs_emb_source_document_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_header_level_idx": { + "name": "docs_emb_header_level_idx", + "columns": [ + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_source_header_idx": { + "name": "docs_emb_source_header_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_model_idx": { + "name": "docs_emb_model_idx", + "columns": [ + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_created_at_idx": { + "name": "docs_emb_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_embedding_vector_hnsw_idx": { + "name": "docs_embedding_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "docs_emb_metadata_gin_idx": { + "name": "docs_emb_metadata_gin_idx", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "docs_emb_chunk_text_fts_idx": { + "name": "docs_emb_chunk_text_fts_idx", + "columns": [ + { + "expression": "chunk_text_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "docs_embedding_not_null_check": { + "name": "docs_embedding_not_null_check", + "value": "\"embedding\" IS NOT NULL" + }, + "docs_header_level_check": { + "name": "docs_header_level_check", + "value": "\"header_level\" >= 1 AND \"header_level\" <= 6" + } + }, + "isRLSEnabled": false + }, + "public.document": { + "name": "document", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_url": { + "name": "file_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "character_count": { + "name": "character_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "processing_status": { + "name": "processing_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_completed_at": { + "name": "processing_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "user_excluded": { + "name": "user_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "doc_kb_id_idx": { + "name": "doc_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_filename_idx": { + "name": "doc_filename_idx", + "columns": [ + { + "expression": "filename", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_processing_status_idx": { + "name": "doc_processing_status_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "processing_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_external_id_idx": { + "name": "doc_connector_external_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_id_idx": { + "name": "doc_connector_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_storage_key_idx": { + "name": "doc_storage_key_idx", + "columns": [ + { + "expression": "storage_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"storage_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_archived_at_partial_idx": { + "name": "doc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_deleted_at_partial_idx": { + "name": "doc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag1_idx": { + "name": "doc_tag1_idx", + "columns": [ + { + "expression": "tag1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag2_idx": { + "name": "doc_tag2_idx", + "columns": [ + { + "expression": "tag2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag3_idx": { + "name": "doc_tag3_idx", + "columns": [ + { + "expression": "tag3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag4_idx": { + "name": "doc_tag4_idx", + "columns": [ + { + "expression": "tag4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag5_idx": { + "name": "doc_tag5_idx", + "columns": [ + { + "expression": "tag5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag6_idx": { + "name": "doc_tag6_idx", + "columns": [ + { + "expression": "tag6", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag7_idx": { + "name": "doc_tag7_idx", + "columns": [ + { + "expression": "tag7", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number1_idx": { + "name": "doc_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number2_idx": { + "name": "doc_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number3_idx": { + "name": "doc_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number4_idx": { + "name": "doc_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number5_idx": { + "name": "doc_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date1_idx": { + "name": "doc_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date2_idx": { + "name": "doc_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean1_idx": { + "name": "doc_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean2_idx": { + "name": "doc_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean3_idx": { + "name": "doc_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_knowledge_base_id_knowledge_base_id_fk": { + "name": "document_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "document_connector_id_knowledge_connector_id_fk": { + "name": "document_connector_id_knowledge_connector_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "document_uploaded_by_user_id_fk": { + "name": "document_uploaded_by_user_id_fk", + "tableFrom": "document", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.embedding": { + "name": "embedding", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "chunk_hash": { + "name": "chunk_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_length": { + "name": "content_length", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "content_tsv": { + "name": "content_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"embedding\".\"content\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "emb_kb_id_idx": { + "name": "emb_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_id_idx": { + "name": "emb_doc_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_chunk_idx": { + "name": "emb_doc_chunk_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chunk_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_model_idx": { + "name": "emb_kb_model_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_enabled_idx": { + "name": "emb_kb_enabled_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_enabled_idx": { + "name": "emb_doc_enabled_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "embedding_vector_hnsw_idx": { + "name": "embedding_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "emb_tag1_idx": { + "name": "emb_tag1_idx", + "columns": [ + { + "expression": "tag1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag2_idx": { + "name": "emb_tag2_idx", + "columns": [ + { + "expression": "tag2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag3_idx": { + "name": "emb_tag3_idx", + "columns": [ + { + "expression": "tag3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag4_idx": { + "name": "emb_tag4_idx", + "columns": [ + { + "expression": "tag4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag5_idx": { + "name": "emb_tag5_idx", + "columns": [ + { + "expression": "tag5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag6_idx": { + "name": "emb_tag6_idx", + "columns": [ + { + "expression": "tag6", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag7_idx": { + "name": "emb_tag7_idx", + "columns": [ + { + "expression": "tag7", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number1_idx": { + "name": "emb_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number2_idx": { + "name": "emb_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number3_idx": { + "name": "emb_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number4_idx": { + "name": "emb_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number5_idx": { + "name": "emb_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date1_idx": { + "name": "emb_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date2_idx": { + "name": "emb_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean1_idx": { + "name": "emb_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean2_idx": { + "name": "emb_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean3_idx": { + "name": "emb_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_content_fts_idx": { + "name": "emb_content_fts_idx", + "columns": [ + { + "expression": "content_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "embedding_knowledge_base_id_knowledge_base_id_fk": { + "name": "embedding_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "embedding", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embedding_document_id_document_id_fk": { + "name": "embedding_document_id_document_id_fk", + "tableFrom": "embedding", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "embedding_not_null_check": { + "name": "embedding_not_null_check", + "value": "\"embedding\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.environment": { + "name": "environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "environment_user_id_user_id_fk": { + "name": "environment_user_id_user_id_fk", + "tableFrom": "environment", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "environment_user_id_unique": { + "name": "environment_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_dependencies": { + "name": "execution_large_value_dependencies", + "schema": "", + "columns": { + "parent_key": { + "name": "parent_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_key": { + "name": "child_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_dependencies_workspace_parent_key_idx": { + "name": "execution_large_value_dependencies_workspace_parent_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_value_dependencies_workspace_child_key_idx": { + "name": "execution_large_value_dependencies_workspace_child_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_dependencies_workspace_id_workspace_id_fk": { + "name": "execution_large_value_dependencies_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_dependencies", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_dependencies_parent_key_child_key_pk": { + "name": "execution_large_value_dependencies_parent_key_child_key_pk", + "columns": ["parent_key", "child_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_references": { + "name": "execution_large_value_references", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "execution_large_value_reference_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_references_workspace_execution_source_idx": { + "name": "execution_large_value_references_workspace_execution_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_references_workspace_id_workspace_id_fk": { + "name": "execution_large_value_references_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_value_references_workflow_id_workflow_id_fk": { + "name": "execution_large_value_references_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_references_key_execution_id_source_pk": { + "name": "execution_large_value_references_key_execution_id_source_pk", + "columns": ["key", "execution_id", "source"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_values": { + "name": "execution_large_values", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_execution_id": { + "name": "owner_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "execution_large_values_owner_execution_id_idx": { + "name": "execution_large_values_owner_execution_id_idx", + "columns": [ + { + "expression": "owner_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_cleanup_idx": { + "name": "execution_large_values_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_tombstone_cleanup_idx": { + "name": "execution_large_values_tombstone_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_values_workspace_id_workspace_id_fk": { + "name": "execution_large_values_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_values_workflow_id_workflow_id_fk": { + "name": "execution_large_values_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.folder": { + "name": "folder", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "folder_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "folder_user_idx": { + "name": "folder_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_idx": { + "name": "folder_workspace_resource_parent_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_parent_sort_idx": { + "name": "folder_parent_sort_idx", + "columns": [ + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_deleted_at_idx": { + "name": "folder_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_deleted_partial_idx": { + "name": "folder_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"folder\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_name_active_unique": { + "name": "folder_workspace_resource_parent_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"parent_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"folder\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "folder_user_id_user_id_fk": { + "name": "folder_user_id_user_id_fk", + "tableFrom": "folder", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_workspace_id_workspace_id_fk": { + "name": "folder_workspace_id_workspace_id_fk", + "tableFrom": "folder", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_parent_id_folder_id_fk": { + "name": "folder_parent_id_folder_id_fk", + "tableFrom": "folder", + "tableTo": "folder", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.idempotency_key": { + "name": "idempotency_key", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idempotency_key_created_at_idx": { + "name": "idempotency_key_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "invitation_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'organization'" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "membership_intent": { + "name": "membership_intent", + "type": "invitation_membership_intent", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'internal'" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "invitation_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_status_idx": { + "name": "invitation_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_pending_email_org_unique": { + "name": "invitation_pending_email_org_unique", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"invitation\".\"status\" = 'pending' AND \"invitation\".\"organization_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "invitation_token_unique": { + "name": "invitation_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation_workspace_grant": { + "name": "invitation_workspace_grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "invitation_id": { + "name": "invitation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission": { + "name": "permission", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_workspace_grant_unique": { + "name": "invitation_workspace_grant_unique", + "columns": [ + { + "expression": "invitation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_workspace_grant_workspace_id_idx": { + "name": "invitation_workspace_grant_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_workspace_grant_invitation_id_invitation_id_fk": { + "name": "invitation_workspace_grant_invitation_id_invitation_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "invitation", + "columnsFrom": ["invitation_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_workspace_grant_workspace_id_workspace_id_fk": { + "name": "invitation_workspace_grant_workspace_id_workspace_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.job_execution_logs": { + "name": "job_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schedule_id": { + "name": "schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost": { + "name": "cost", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "job_execution_logs_schedule_id_idx": { + "name": "job_execution_logs_schedule_id_idx", + "columns": [ + { + "expression": "schedule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_started_at_idx": { + "name": "job_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_ended_at_id_idx": { + "name": "job_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_execution_id_unique": { + "name": "job_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_trigger_idx": { + "name": "job_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "job_execution_logs_schedule_id_workflow_schedule_id_fk": { + "name": "job_execution_logs_schedule_id_workflow_schedule_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workflow_schedule", + "columnsFrom": ["schedule_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "job_execution_logs_workspace_id_workspace_id_fk": { + "name": "job_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_base": { + "name": "knowledge_base", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "embedding_dimension": { + "name": "embedding_dimension", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1536 + }, + "chunking_config": { + "name": "chunking_config", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{\"maxSize\": 1024, \"minSize\": 1, \"overlap\": 200}'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_user_id_idx": { + "name": "kb_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_id_idx": { + "name": "kb_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_user_workspace_idx": { + "name": "kb_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_folder_id_idx": { + "name": "kb_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_deleted_at_idx": { + "name": "kb_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_deleted_partial_idx": { + "name": "kb_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_base\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_name_active_unique": { + "name": "kb_workspace_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_user_id_user_id_fk": { + "name": "knowledge_base_user_id_user_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_workspace_id_workspace_id_fk": { + "name": "knowledge_base_workspace_id_workspace_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_folder_id_folder_id_fk": { + "name": "knowledge_base_folder_id_folder_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_base_tag_definitions": { + "name": "knowledge_base_tag_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_slot": { + "name": "tag_slot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "field_type": { + "name": "field_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_tag_definitions_kb_slot_idx": { + "name": "kb_tag_definitions_kb_slot_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_slot", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_display_name_idx": { + "name": "kb_tag_definitions_kb_display_name_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_id_idx": { + "name": "kb_tag_definitions_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_base_tag_definitions", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_connector": { + "name": "knowledge_connector", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connector_type": { + "name": "connector_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_config": { + "name": "source_config", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "sync_mode": { + "name": "sync_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'full'" + }, + "sync_interval_minutes": { + "name": "sync_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1440 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_sync_doc_count": { + "name": "last_sync_doc_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "next_sync_at": { + "name": "next_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kc_knowledge_base_id_idx": { + "name": "kc_knowledge_base_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_status_next_sync_idx": { + "name": "kc_status_next_sync_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_sync_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_archived_at_partial_idx": { + "name": "kc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_deleted_at_partial_idx": { + "name": "kc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_connector_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_connector", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_connector_sync_log": { + "name": "knowledge_connector_sync_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "docs_added": { + "name": "docs_added", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_updated": { + "name": "docs_updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_deleted": { + "name": "docs_deleted", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_unchanged": { + "name": "docs_unchanged", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_failed": { + "name": "docs_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kcsl_connector_id_idx": { + "name": "kcsl_connector_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_sync_log", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_server_oauth": { + "name": "mcp_server_oauth", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_server_id": { + "name": "mcp_server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_information": { + "name": "client_information", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_created_at": { + "name": "state_created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_refreshed_at": { + "name": "last_refreshed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_server_oauth_server_unique": { + "name": "mcp_server_oauth_server_unique", + "columns": [ + { + "expression": "mcp_server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_server_oauth_state_idx": { + "name": "mcp_server_oauth_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk": { + "name": "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "mcp_servers", + "columnsFrom": ["mcp_server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_server_oauth_user_id_user_id_fk": { + "name": "mcp_server_oauth_user_id_user_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "mcp_server_oauth_workspace_id_workspace_id_fk": { + "name": "mcp_server_oauth_workspace_id_workspace_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "transport": { + "name": "transport", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'headers'" + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_secret": { + "name": "oauth_client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "headers": { + "name": "headers", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "timeout": { + "name": "timeout", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30000 + }, + "retries": { + "name": "retries", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 3 + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_connected": { + "name": "last_connected", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "connection_status": { + "name": "connection_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'disconnected'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status_config": { + "name": "status_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "tool_count": { + "name": "tool_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_tools_refresh": { + "name": "last_tools_refresh", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_servers_workspace_enabled_idx": { + "name": "mcp_servers_workspace_enabled_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_workspace_deleted_partial_idx": { + "name": "mcp_servers_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"mcp_servers\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_servers_workspace_id_workspace_id_fk": { + "name": "mcp_servers_workspace_id_workspace_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_servers_created_by_user_id_fk": { + "name": "mcp_servers_created_by_user_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "member_user_id_unique": { + "name": "member_user_id_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory": { + "name": "memory", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_key_idx": { + "name": "memory_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_idx": { + "name": "memory_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_key_idx": { + "name": "memory_workspace_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_deleted_partial_idx": { + "name": "memory_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"memory\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_workspace_id_workspace_id_fk": { + "name": "memory_workspace_id_workspace_id_fk", + "tableFrom": "memory", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_allowed_sender": { + "name": "mothership_inbox_allowed_sender", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inbox_sender_ws_email_idx": { + "name": "inbox_sender_ws_email_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_allowed_sender_added_by_user_id_fk": { + "name": "mothership_inbox_allowed_sender_added_by_user_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "user", + "columnsFrom": ["added_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_task": { + "name": "mothership_inbox_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_email": { + "name": "from_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_name": { + "name": "from_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body_preview": { + "name": "body_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_text": { + "name": "body_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_html": { + "name": "body_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_message_id": { + "name": "email_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "in_reply_to": { + "name": "in_reply_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "response_message_id": { + "name": "response_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agentmail_message_id": { + "name": "agentmail_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "trigger_job_id": { + "name": "trigger_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result_summary": { + "name": "result_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rejection_reason": { + "name": "rejection_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "has_attachments": { + "name": "has_attachments", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cc_recipients": { + "name": "cc_recipients", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "inbox_task_ws_created_at_idx": { + "name": "inbox_task_ws_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_ws_status_idx": { + "name": "inbox_task_ws_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_response_msg_id_idx": { + "name": "inbox_task_response_msg_id_idx", + "columns": [ + { + "expression": "response_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_email_msg_id_idx": { + "name": "inbox_task_email_msg_id_idx", + "columns": [ + { + "expression": "email_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_task_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_task_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_task_chat_id_copilot_chats_id_fk": { + "name": "mothership_inbox_task_chat_id_copilot_chats_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_webhook": { + "name": "mothership_inbox_webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mothership_inbox_webhook_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_webhook_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_webhook", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mothership_inbox_webhook_workspace_id_unique": { + "name": "mothership_inbox_webhook_workspace_id_unique", + "nullsNotDistinct": false, + "columns": ["workspace_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_settings": { + "name": "mothership_settings", + "schema": "", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_tool_refs": { + "name": "mcp_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "custom_tool_refs": { + "name": "custom_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "skill_refs": { + "name": "skill_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mothership_settings_workspace_id_idx": { + "name": "mothership_settings_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_settings_workspace_id_workspace_id_fk": { + "name": "mothership_settings_workspace_id_workspace_id_fk", + "tableFrom": "mothership_settings", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "session_policy_settings": { + "name": "session_policy_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "security_policy_version": { + "name": "security_policy_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "whitelabel_settings": { + "name": "whitelabel_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "data_retention_settings": { + "name": "data_retention_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "org_usage_limit": { + "name": "org_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "departed_member_usage": { + "name": "departed_member_usage", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_member_usage_limit": { + "name": "organization_member_usage_limit", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "usage_limit": { + "name": "usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "set_by": { + "name": "set_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_member_usage_limit_org_user_unique": { + "name": "org_member_usage_limit_org_user_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_member_usage_limit_organization_id_idx": { + "name": "org_member_usage_limit_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_member_usage_limit_organization_id_organization_id_fk": { + "name": "organization_member_usage_limit_organization_id_organization_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_user_id_user_id_fk": { + "name": "organization_member_usage_limit_user_id_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_set_by_user_id_fk": { + "name": "organization_member_usage_limit_set_by_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["set_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.outbox_event": { + "name": "outbox_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10 + }, + "available_at": { + "name": "available_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "locked_at": { + "name": "locked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "outbox_event_status_available_idx": { + "name": "outbox_event_status_available_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "available_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_locked_at_idx": { + "name": "outbox_event_locked_at_idx", + "columns": [ + { + "expression": "locked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_type_created_idx": { + "name": "outbox_event_type_created_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.paused_executions": { + "name": "paused_executions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_snapshot": { + "name": "execution_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "pause_points": { + "name": "pause_points", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "total_pause_count": { + "name": "total_pause_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "resumed_count": { + "name": "resumed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "automatic_resume_retry_count": { + "name": "automatic_resume_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paused'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "paused_at": { + "name": "paused_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "next_resume_at": { + "name": "next_resume_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "paused_executions_workflow_id_idx": { + "name": "paused_executions_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_status_idx": { + "name": "paused_executions_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_execution_id_unique": { + "name": "paused_executions_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_next_resume_at_idx": { + "name": "paused_executions_next_resume_at_idx", + "columns": [ + { + "expression": "next_resume_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'paused' AND next_resume_at IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "paused_executions_workflow_id_workflow_id_fk": { + "name": "paused_executions_workflow_id_workflow_id_fk", + "tableFrom": "paused_executions", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pending_credential_draft": { + "name": "pending_credential_draft", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pending_draft_user_provider_ws": { + "name": "pending_draft_user_provider_ws", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pending_credential_draft_user_id_user_id_fk": { + "name": "pending_credential_draft_user_id_user_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_workspace_id_workspace_id_fk": { + "name": "pending_credential_draft_workspace_id_workspace_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_credential_id_credential_id_fk": { + "name": "pending_credential_draft_credential_id_credential_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group": { + "name": "permission_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "permission_group_created_by_idx": { + "name": "permission_group_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_name_unique": { + "name": "permission_group_organization_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_default_unique": { + "name": "permission_group_organization_default_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_organization_id_organization_id_fk": { + "name": "permission_group_organization_id_organization_id_fk", + "tableFrom": "permission_group", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_created_by_user_id_fk": { + "name": "permission_group_created_by_user_id_fk", + "tableFrom": "permission_group", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_member": { + "name": "permission_group_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "assigned_by": { + "name": "assigned_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assigned_at": { + "name": "assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_member_group_id_idx": { + "name": "permission_group_member_group_id_idx", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_group_user_unique": { + "name": "permission_group_member_group_user_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_organization_user_idx": { + "name": "permission_group_member_organization_user_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_member_permission_group_id_permission_group_id_fk": { + "name": "permission_group_member_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_organization_id_organization_id_fk": { + "name": "permission_group_member_organization_id_organization_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_user_id_user_id_fk": { + "name": "permission_group_member_user_id_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_assigned_by_user_id_fk": { + "name": "permission_group_member_assigned_by_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["assigned_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_workspace": { + "name": "permission_group_workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_workspace_workspace_id_idx": { + "name": "permission_group_workspace_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_workspace_group_workspace_unique": { + "name": "permission_group_workspace_group_workspace_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_workspace_permission_group_id_permission_group_id_fk": { + "name": "permission_group_workspace_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_workspace_id_workspace_id_fk": { + "name": "permission_group_workspace_workspace_id_workspace_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_organization_id_organization_id_fk": { + "name": "permission_group_workspace_organization_id_organization_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permissions": { + "name": "permissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission_type": { + "name": "permission_type", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permissions_user_id_idx": { + "name": "permissions_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_entity_idx": { + "name": "permissions_entity_idx", + "columns": [ + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_type_idx": { + "name": "permissions_user_entity_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_permission_idx": { + "name": "permissions_user_entity_permission_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "permission_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_idx": { + "name": "permissions_user_entity_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_unique_constraint": { + "name": "permissions_unique_constraint", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permissions_user_id_user_id_fk": { + "name": "permissions_user_id_user_id_fk", + "tableFrom": "permissions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pinned_item": { + "name": "pinned_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pinned_item_user_workspace_idx": { + "name": "pinned_item_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_resource_idx": { + "name": "pinned_item_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_user_resource_unique": { + "name": "pinned_item_user_resource_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pinned_item_user_id_user_id_fk": { + "name": "pinned_item_user_id_user_id_fk", + "tableFrom": "pinned_item", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pinned_item_workspace_id_workspace_id_fk": { + "name": "pinned_item_workspace_id_workspace_id_fk", + "tableFrom": "pinned_item", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.public_share": { + "name": "public_share", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "public_share_token_unique": { + "name": "public_share_token_unique", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_unique": { + "name": "public_share_resource_unique", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_id_idx": { + "name": "public_share_resource_id_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_workspace_id_idx": { + "name": "public_share_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "public_share_workspace_id_workspace_id_fk": { + "name": "public_share_workspace_id_workspace_id_fk", + "tableFrom": "public_share", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "public_share_created_by_user_id_fk": { + "name": "public_share_created_by_user_id_fk", + "tableFrom": "public_share", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit_bucket": { + "name": "rate_limit_bucket", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tokens": { + "name": "tokens", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.resume_queue": { + "name": "resume_queue", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "paused_execution_id": { + "name": "paused_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_execution_id": { + "name": "parent_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "new_execution_id": { + "name": "new_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context_id": { + "name": "context_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resume_input": { + "name": "resume_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "resume_queue_parent_status_idx": { + "name": "resume_queue_parent_status_idx", + "columns": [ + { + "expression": "parent_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resume_queue_new_execution_idx": { + "name": "resume_queue_new_execution_idx", + "columns": [ + { + "expression": "new_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "resume_queue_paused_execution_id_paused_executions_id_fk": { + "name": "resume_queue_paused_execution_id_paused_executions_id_fk", + "tableFrom": "resume_queue", + "tableTo": "paused_executions", + "columnsFrom": ["paused_execution_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "session_token_idx": { + "name": "session_token_idx", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_active_organization_id_organization_id_fk": { + "name": "session_active_organization_id_organization_id_fk", + "tableFrom": "session", + "tableTo": "organization", + "columnsFrom": ["active_organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.settings": { + "name": "settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "theme": { + "name": "theme", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'system'" + }, + "auto_connect": { + "name": "auto_connect", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "telemetry_enabled": { + "name": "telemetry_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "email_preferences": { + "name": "email_preferences", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "billing_usage_notifications_enabled": { + "name": "billing_usage_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "show_training_controls": { + "name": "show_training_controls", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "super_user_mode_enabled": { + "name": "super_user_mode_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "mothership_environment": { + "name": "mothership_environment", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "error_notifications_enabled": { + "name": "error_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "snap_to_grid_size": { + "name": "snap_to_grid_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "show_action_bar": { + "name": "show_action_bar", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "copilot_enabled_models": { + "name": "copilot_enabled_models", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "copilot_auto_allowed_tools": { + "name": "copilot_auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_active_workspace_id": { + "name": "last_active_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "settings_user_id_user_id_fk": { + "name": "settings_user_id_user_id_fk", + "tableFrom": "settings", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "settings_user_id_unique": { + "name": "settings_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sim_trigger_state": { + "name": "sim_trigger_state", + "schema": "", + "columns": { + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope_key": { + "name": "scope_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "last_fired_at": { + "name": "last_fired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sim_trigger_state_workflow_id_workflow_id_fk": { + "name": "sim_trigger_state_workflow_id_workflow_id_fk", + "tableFrom": "sim_trigger_state", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "sim_trigger_state_workflow_id_block_id_scope_key_pk": { + "name": "sim_trigger_state_workflow_id_block_id_scope_key_pk", + "columns": ["workflow_id", "block_id", "scope_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill": { + "name": "skill", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_workspace_name_unique": { + "name": "skill_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_workspace_id_workspace_id_fk": { + "name": "skill_workspace_id_workspace_id_fk", + "tableFrom": "skill", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_user_id_user_id_fk": { + "name": "skill_user_id_user_id_fk", + "tableFrom": "skill", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_member": { + "name": "skill_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_member_user_id_idx": { + "name": "skill_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skill_member_unique": { + "name": "skill_member_unique", + "columns": [ + { + "expression": "skill_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_member_skill_id_skill_id_fk": { + "name": "skill_member_skill_id_skill_id_fk", + "tableFrom": "skill_member", + "tableTo": "skill", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_user_id_user_id_fk": { + "name": "skill_member_user_id_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_invited_by_user_id_fk": { + "name": "skill_member_invited_by_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_domain": { + "name": "sso_domain", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "verification_token": { + "name": "verification_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sso_domain_organization_id_idx": { + "name": "sso_domain_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_domain_idx": { + "name": "sso_domain_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_org_domain_unique": { + "name": "sso_domain_org_domain_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_verified_unique": { + "name": "sso_domain_verified_unique", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "status = 'verified'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_domain_organization_id_organization_id_fk": { + "name": "sso_domain_organization_id_organization_id_fk", + "tableFrom": "sso_domain", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_domain_created_by_user_id_fk": { + "name": "sso_domain_created_by_user_id_fk", + "tableFrom": "sso_domain", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_provider": { + "name": "sso_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sso_provider_provider_id_idx": { + "name": "sso_provider_provider_id_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_domain_idx": { + "name": "sso_provider_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_user_id_idx": { + "name": "sso_provider_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_organization_id_idx": { + "name": "sso_provider_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_provider_user_id_user_id_fk": { + "name": "sso_provider_user_id_user_id_fk", + "tableFrom": "sso_provider", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_provider_organization_id_organization_id_fk": { + "name": "sso_provider_organization_id_organization_id_fk", + "tableFrom": "sso_provider", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subscription": { + "name": "subscription", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_subscription_id": { + "name": "stripe_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "period_start": { + "name": "period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "period_end": { + "name": "period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancel_at_period_end": { + "name": "cancel_at_period_end", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cancel_at": { + "name": "cancel_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "canceled_at": { + "name": "canceled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "seats": { + "name": "seats", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "trial_start": { + "name": "trial_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trial_end": { + "name": "trial_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_interval": { + "name": "billing_interval", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_schedule_id": { + "name": "stripe_schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "subscription_reference_status_idx": { + "name": "subscription_reference_status_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "check_enterprise_metadata": { + "name": "check_enterprise_metadata", + "value": "plan != 'enterprise' OR metadata IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.table_jobs": { + "name": "table_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rows_processed": { + "name": "rows_processed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_jobs_one_active_per_table": { + "name": "table_jobs_one_active_per_table", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"table_jobs\".\"status\" = 'running' AND \"table_jobs\".\"type\" <> 'export'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_watchdog_idx": { + "name": "table_jobs_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_table_started_idx": { + "name": "table_jobs_table_started_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_jobs_table_id_user_table_definitions_id_fk": { + "name": "table_jobs_table_id_user_table_definitions_id_fk", + "tableFrom": "table_jobs", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_jobs_workspace_id_workspace_id_fk": { + "name": "table_jobs_workspace_id_workspace_id_fk", + "tableFrom": "table_jobs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_row_executions": { + "name": "table_row_executions", + "schema": "", + "columns": { + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "running_block_ids": { + "name": "running_block_ids", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "block_errors": { + "name": "block_errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "enrichment_details": { + "name": "enrichment_details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "table_row_executions_table_status_idx": { + "name": "table_row_executions_table_status_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"status\" IN ('queued', 'running', 'pending')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_execution_id_idx": { + "name": "table_row_executions_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"execution_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_table_group_idx": { + "name": "table_row_executions_table_group_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_row_executions_table_id_user_table_definitions_id_fk": { + "name": "table_row_executions_table_id_user_table_definitions_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_row_executions_row_id_user_table_rows_id_fk": { + "name": "table_row_executions_row_id_user_table_rows_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_rows", + "columnsFrom": ["row_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "table_row_executions_row_id_group_id_pk": { + "name": "table_row_executions_row_id_group_id_pk", + "columns": ["row_id", "group_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_run_dispatches": { + "name": "table_run_dispatches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "cursor": { + "name": "cursor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit": { + "name": "limit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "processed_count": { + "name": "processed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_manual_run": { + "name": "is_manual_run", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "triggered_by_user_id": { + "name": "triggered_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_run_dispatches_active_idx": { + "name": "table_run_dispatches_active_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_run_dispatches_watchdog_idx": { + "name": "table_run_dispatches_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_run_dispatches_table_id_user_table_definitions_id_fk": { + "name": "table_run_dispatches_table_id_user_table_definitions_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_workspace_id_workspace_id_fk": { + "name": "table_run_dispatches_workspace_id_workspace_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_triggered_by_user_id_user_id_fk": { + "name": "table_run_dispatches_triggered_by_user_id_user_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user", + "columnsFrom": ["triggered_by_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_views": { + "name": "table_views", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "table_views_table_created_idx": { + "name": "table_views_table_created_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_views_table_default_unique": { + "name": "table_views_table_default_unique", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_views_table_id_user_table_definitions_id_fk": { + "name": "table_views_table_id_user_table_definitions_id_fk", + "tableFrom": "table_views", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_views_workspace_id_workspace_id_fk": { + "name": "table_views_workspace_id_workspace_id_fk", + "tableFrom": "table_views", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_views_created_by_user_id_fk": { + "name": "table_views_created_by_user_id_fk", + "tableFrom": "table_views", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_log": { + "name": "usage_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "usage_log_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "usage_log_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "cost": { + "name": "cost", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_entity_type": { + "name": "billing_entity_type", + "type": "billing_entity_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "billing_entity_id": { + "name": "billing_entity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_period_start": { + "name": "billing_period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_period_end": { + "name": "billing_period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "usage_log_user_created_at_idx": { + "name": "usage_log_user_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_source_idx": { + "name": "usage_log_source_idx", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_id_idx": { + "name": "usage_log_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workflow_id_idx": { + "name": "usage_log_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_event_key_unique": { + "name": "usage_log_event_key_unique", + "columns": [ + { + "expression": "event_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"usage_log\".\"event_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_entity_period_idx": { + "name": "usage_log_billing_entity_period_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_period_cost_idx": { + "name": "usage_log_billing_period_cost_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_created_at_idx": { + "name": "usage_log_workspace_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_execution_id_idx": { + "name": "usage_log_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_log_user_id_user_id_fk": { + "name": "usage_log_user_id_user_id_fk", + "tableFrom": "usage_log", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "usage_log_workspace_id_workspace_id_fk": { + "name": "usage_log_workspace_id_workspace_id_fk", + "tableFrom": "usage_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "usage_log_workflow_id_workflow_id_fk": { + "name": "usage_log_workflow_id_workflow_id_fk", + "tableFrom": "usage_log", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_log_billing_scope_all_or_none": { + "name": "usage_log_billing_scope_all_or_none", + "value": "(\n (\"usage_log\".\"billing_entity_type\" IS NULL AND \"usage_log\".\"billing_entity_id\" IS NULL AND \"usage_log\".\"billing_period_start\" IS NULL AND \"usage_log\".\"billing_period_end\" IS NULL)\n OR\n (\"usage_log\".\"billing_entity_type\" IS NOT NULL AND \"usage_log\".\"billing_entity_id\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" IS NOT NULL AND \"usage_log\".\"billing_period_end\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" < \"usage_log\".\"billing_period_end\")\n )" + } + }, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": ["normalized_email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_stats": { + "name": "user_stats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "total_manual_executions": { + "name": "total_manual_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_api_calls": { + "name": "total_api_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_webhook_triggers": { + "name": "total_webhook_triggers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_scheduled_executions": { + "name": "total_scheduled_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_chat_executions": { + "name": "total_chat_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_executions": { + "name": "total_mcp_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_tokens_used": { + "name": "total_tokens_used", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_cost": { + "name": "total_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_usage_limit": { + "name": "current_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'5'" + }, + "usage_limit_updated_at": { + "name": "usage_limit_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "current_period_cost": { + "name": "current_period_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_period_cost": { + "name": "last_period_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "billed_overage_this_period": { + "name": "billed_overage_this_period", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "pro_period_cost_snapshot": { + "name": "pro_period_cost_snapshot", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "pro_period_cost_snapshot_at": { + "name": "pro_period_cost_snapshot_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "total_copilot_cost": { + "name": "total_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_period_copilot_cost": { + "name": "current_period_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_period_copilot_cost": { + "name": "last_period_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "total_copilot_tokens": { + "name": "total_copilot_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_copilot_calls": { + "name": "total_copilot_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_copilot_calls": { + "name": "total_mcp_copilot_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_copilot_cost": { + "name": "total_mcp_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_period_mcp_copilot_cost": { + "name": "current_period_mcp_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_active": { + "name": "last_active", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "billing_blocked": { + "name": "billing_blocked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "billing_blocked_reason": { + "name": "billing_blocked_reason", + "type": "billing_blocked_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + } + }, + "indexes": {}, + "foreignKeys": { + "user_stats_user_id_user_id_fk": { + "name": "user_stats_user_id_user_id_fk", + "tableFrom": "user_stats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_stats_user_id_unique": { + "name": "user_stats_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_definitions": { + "name": "user_table_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "max_rows": { + "name": "max_rows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "row_count": { + "name": "row_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "rows_version": { + "name": "rows_version", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "schema_locked": { + "name": "schema_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "insert_locked": { + "name": "insert_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "update_locked": { + "name": "update_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "delete_locked": { + "name": "delete_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "user_table_def_workspace_id_idx": { + "name": "user_table_def_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_folder_id_idx": { + "name": "user_table_def_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_name_unique": { + "name": "user_table_def_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"user_table_definitions\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_archived_at_idx": { + "name": "user_table_def_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_archived_partial_idx": { + "name": "user_table_def_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"user_table_definitions\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_definitions_workspace_id_workspace_id_fk": { + "name": "user_table_definitions_workspace_id_workspace_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_definitions_folder_id_folder_id_fk": { + "name": "user_table_definitions_folder_id_folder_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "user_table_definitions_created_by_user_id_fk": { + "name": "user_table_definitions_created_by_user_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_rows": { + "name": "user_table_rows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "user_table_rows_tenant_data_gin_idx": { + "name": "user_table_rows_tenant_data_gin_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"data\" jsonb_path_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "user_table_rows_workspace_table_idx": { + "name": "user_table_rows_workspace_table_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_position_idx": { + "name": "user_table_rows_table_position_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "position", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_order_key_idx": { + "name": "user_table_rows_table_order_key_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "order_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_id_id_idx": { + "name": "user_table_rows_table_id_id_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_rows_table_id_user_table_definitions_id_fk": { + "name": "user_table_rows_table_id_user_table_definitions_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_workspace_id_workspace_id_fk": { + "name": "user_table_rows_workspace_id_workspace_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_created_by_user_id_fk": { + "name": "user_table_rows_created_by_user_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "verification_expires_at_idx": { + "name": "verification_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.waitlist": { + "name": "waitlist", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "waitlist_email_unique": { + "name": "waitlist_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook": { + "name": "webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_status": { + "name": "registration_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_generation": { + "name": "registration_generation", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "config_fingerprint": { + "name": "config_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prepared_at": { + "name": "prepared_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "routing_key": { + "name": "routing_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_config": { + "name": "provider_config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "path_deployment_unique": { + "name": "path_deployment_unique", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_workflow_deployment_idx": { + "name": "webhook_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_routing_key_active_idx": { + "name": "webhook_routing_key_active_idx", + "columns": [ + { + "expression": "routing_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NULL AND \"webhook\".\"routing_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_archived_at_partial_idx": { + "name": "webhook_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468": { + "name": "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_tiktok_credential_id_idx": { + "name": "webhook_tiktok_credential_id_idx", + "columns": [ + { + "expression": "((\"provider_config\")::jsonb ->> 'credentialId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"provider\" = 'tiktok' AND \"webhook\".\"is_active\" = true AND \"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_workflow_id_block_id_updated_at_desc": { + "name": "idx_webhook_on_workflow_id_block_id_updated_at_desc", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_active_registration_unique": { + "name": "webhook_active_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'active' AND \"webhook\".\"block_id\" IS NOT NULL AND \"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_candidate_registration_unique": { + "name": "webhook_candidate_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'candidate' AND \"webhook\".\"block_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_registration_status_generation_idx": { + "name": "webhook_registration_status_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_workflow_id_workflow_id_fk": { + "name": "webhook_workflow_id_workflow_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "webhook_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_registration_status_check": { + "name": "webhook_registration_status_check", + "value": "\"webhook\".\"registration_status\" IS NULL OR \"webhook\".\"registration_status\" IN ('active', 'candidate', 'retired', 'orphaned')" + }, + "webhook_registration_generation_check": { + "name": "webhook_registration_generation_check", + "value": "\"webhook\".\"registration_generation\" IS NULL OR \"webhook\".\"registration_generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.webhook_path_claim": { + "name": "webhook_path_claim", + "schema": "", + "columns": { + "path": { + "name": "path", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "webhook_path_claim_workflow_idx": { + "name": "webhook_path_claim_workflow_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_path_claim_workflow_id_workflow_id_fk": { + "name": "webhook_path_claim_workflow_id_workflow_id_fk", + "tableFrom": "webhook_path_claim", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_path_claim_generation_check": { + "name": "webhook_path_claim_generation_check", + "value": "\"webhook_path_claim\".\"generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow": { + "name": "workflow", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_synced": { + "name": "last_synced", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "is_deployed": { + "name": "is_deployed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deployed_at": { + "name": "deployed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "is_public_api": { + "name": "is_public_api", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "fork_sync_excluded": { + "name": "fork_sync_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_user_id_idx": { + "name": "workflow_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_id_idx": { + "name": "workflow_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_user_workspace_idx": { + "name": "workflow_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_folder_name_active_unique": { + "name": "workflow_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_folder_sort_idx": { + "name": "workflow_folder_sort_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_archived_at_idx": { + "name": "workflow_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_archived_partial_idx": { + "name": "workflow_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_user_id_user_id_fk": { + "name": "workflow_user_id_user_id_fk", + "tableFrom": "workflow", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_workspace_id_workspace_id_fk": { + "name": "workflow_workspace_id_workspace_id_fk", + "tableFrom": "workflow", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_folder_id_folder_id_fk": { + "name": "workflow_folder_id_folder_id_fk", + "tableFrom": "workflow", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_blocks": { + "name": "workflow_blocks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position_x": { + "name": "position_x", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "position_y": { + "name": "position_y", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "horizontal_handles": { + "name": "horizontal_handles", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_wide": { + "name": "is_wide", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "advanced_mode": { + "name": "advanced_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "trigger_mode": { + "name": "trigger_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "height": { + "name": "height", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "sub_blocks": { + "name": "sub_blocks", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "outputs": { + "name": "outputs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_blocks_workflow_id_idx": { + "name": "workflow_blocks_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_blocks_type_idx": { + "name": "workflow_blocks_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_blocks_workflow_id_workflow_id_fk": { + "name": "workflow_blocks_workflow_id_workflow_id_fk", + "tableFrom": "workflow_blocks", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_checkpoints": { + "name": "workflow_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_state": { + "name": "workflow_state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_checkpoints_user_id_idx": { + "name": "workflow_checkpoints_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_id_idx": { + "name": "workflow_checkpoints_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_id_idx": { + "name": "workflow_checkpoints_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_message_id_idx": { + "name": "workflow_checkpoints_message_id_idx", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_user_workflow_idx": { + "name": "workflow_checkpoints_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_chat_idx": { + "name": "workflow_checkpoints_workflow_chat_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_created_at_idx": { + "name": "workflow_checkpoints_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_created_at_idx": { + "name": "workflow_checkpoints_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_checkpoints_user_id_user_id_fk": { + "name": "workflow_checkpoints_user_id_user_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_workflow_id_workflow_id_fk": { + "name": "workflow_checkpoints_workflow_id_workflow_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_chat_id_copilot_chats_id_fk": { + "name": "workflow_checkpoints_chat_id_copilot_chats_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_deployment_operation": { + "name": "workflow_deployment_operation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "previous_active_version_id": { + "name": "previous_active_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "protocol_version": { + "name": "protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'preparing'" + }, + "component_readiness": { + "name": "component_readiness", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_hash": { + "name": "request_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_deployment_operation_workflow_generation_unique": { + "name": "workflow_deployment_operation_workflow_generation_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_idempotency_unique": { + "name": "workflow_deployment_operation_workflow_idempotency_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"idempotency_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_in_flight_unique": { + "name": "workflow_deployment_operation_workflow_in_flight_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_status_idx": { + "name": "workflow_deployment_operation_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_deployment_version_idx": { + "name": "workflow_deployment_operation_deployment_version_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_version_generation_idx": { + "name": "workflow_deployment_operation_workflow_version_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_operation_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_operation_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["previous_active_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_deployment_operation_action_check": { + "name": "workflow_deployment_operation_action_check", + "value": "\"workflow_deployment_operation\".\"action\" IN ('deploy', 'activate')" + }, + "workflow_deployment_operation_status_check": { + "name": "workflow_deployment_operation_status_check", + "value": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating', 'active', 'failed', 'superseded')" + }, + "workflow_deployment_operation_generation_check": { + "name": "workflow_deployment_operation_generation_check", + "value": "\"workflow_deployment_operation\".\"generation\" > 0" + }, + "workflow_deployment_operation_protocol_version_check": { + "name": "workflow_deployment_operation_protocol_version_check", + "value": "\"workflow_deployment_operation\".\"protocol_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow_deployment_version": { + "name": "workflow_deployment_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_deployment_version_workflow_version_unique": { + "name": "workflow_deployment_version_workflow_version_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_workflow_active_idx": { + "name": "workflow_deployment_version_workflow_active_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_created_at_idx": { + "name": "workflow_deployment_version_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_version_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_version_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_version", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_edges": { + "name": "workflow_edges", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_block_id": { + "name": "source_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_handle": { + "name": "source_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_handle": { + "name": "target_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_edges_workflow_id_idx": { + "name": "workflow_edges_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_source_idx": { + "name": "workflow_edges_workflow_source_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_target_idx": { + "name": "workflow_edges_workflow_target_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_edges_workflow_id_workflow_id_fk": { + "name": "workflow_edges_workflow_id_workflow_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_source_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_source_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["source_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_target_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_target_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["target_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_logs": { + "name": "workflow_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_snapshot_id": { + "name": "state_snapshot_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost": { + "name": "cost", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "cost_total": { + "name": "cost_total", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "models_used": { + "name": "models_used", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "files": { + "name": "files", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_execution_logs_workflow_id_idx": { + "name": "workflow_execution_logs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_state_snapshot_id_idx": { + "name": "workflow_execution_logs_state_snapshot_id_idx", + "columns": [ + { + "expression": "state_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_deployment_version_id_idx": { + "name": "workflow_execution_logs_deployment_version_id_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_trigger_idx": { + "name": "workflow_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_level_idx": { + "name": "workflow_execution_logs_level_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_started_at_idx": { + "name": "workflow_execution_logs_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_execution_id_unique": { + "name": "workflow_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workflow_started_at_idx": { + "name": "workflow_execution_logs_workflow_started_at_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_idx": { + "name": "workflow_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_id_desc_idx": { + "name": "workflow_execution_logs_workspace_started_at_id_desc_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"started_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "\"id\" DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_cost_total_idx": { + "name": "workflow_execution_logs_workspace_cost_total_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_total", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_models_used_idx": { + "name": "workflow_execution_logs_models_used_idx", + "columns": [ + { + "expression": "models_used", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "workflow_execution_logs_workspace_ended_at_id_idx": { + "name": "workflow_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_running_started_at_idx": { + "name": "workflow_execution_logs_running_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_completed_ended_at_idx": { + "name": "workflow_execution_logs_completed_ended_at_idx", + "columns": [ + { + "expression": "ended_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_execution_logs\".\"status\" = 'completed' AND \"workflow_execution_logs\".\"level\" = 'info' AND \"workflow_execution_logs\".\"ended_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_logs_workflow_id_workflow_id_fk": { + "name": "workflow_execution_logs_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_execution_logs_workspace_id_workspace_id_fk": { + "name": "workflow_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk": { + "name": "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_execution_snapshots", + "columnsFrom": ["state_snapshot_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_snapshots": { + "name": "workflow_execution_snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_hash": { + "name": "state_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_data": { + "name": "state_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_snapshots_workflow_id_idx": { + "name": "workflow_snapshots_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_hash_idx": { + "name": "workflow_snapshots_hash_idx", + "columns": [ + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_workflow_hash_idx": { + "name": "workflow_snapshots_workflow_hash_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_created_at_idx": { + "name": "workflow_snapshots_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_snapshots_workflow_id_workflow_id_fk": { + "name": "workflow_execution_snapshots_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_snapshots", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_server": { + "name": "workflow_mcp_server", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_server_workspace_id_idx": { + "name": "workflow_mcp_server_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_created_by_idx": { + "name": "workflow_mcp_server_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_deleted_at_idx": { + "name": "workflow_mcp_server_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_workspace_deleted_partial_idx": { + "name": "workflow_mcp_server_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_server\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_server_workspace_id_workspace_id_fk": { + "name": "workflow_mcp_server_workspace_id_workspace_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_server_created_by_user_id_fk": { + "name": "workflow_mcp_server_created_by_user_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_tool": { + "name": "workflow_mcp_tool", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_description": { + "name": "tool_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parameter_schema": { + "name": "parameter_schema", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "parameter_description_overrides": { + "name": "parameter_description_overrides", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'::json" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_tool_server_id_idx": { + "name": "workflow_mcp_tool_server_id_idx", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_workflow_id_idx": { + "name": "workflow_mcp_tool_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_server_workflow_unique": { + "name": "workflow_mcp_tool_server_workflow_unique", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_archived_at_partial_idx": { + "name": "workflow_mcp_tool_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk": { + "name": "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow_mcp_server", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_tool_workflow_id_workflow_id_fk": { + "name": "workflow_mcp_tool_workflow_id_workflow_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_schedule": { + "name": "workflow_schedule", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_operation_id": { + "name": "deployment_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cron_expression": { + "name": "cron_expression", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_ran_at": { + "name": "last_ran_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_queued_at": { + "name": "last_queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trigger_type": { + "name": "trigger_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "infra_retry_count": { + "name": "infra_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'workflow'" + }, + "job_title": { + "name": "job_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lifecycle": { + "name": "lifecycle", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'persistent'" + }, + "success_condition": { + "name": "success_condition", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "max_runs": { + "name": "max_runs", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "source_chat_id": { + "name": "source_chat_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_task_name": { + "name": "source_task_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_user_id": { + "name": "source_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "job_history": { + "name": "job_history", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "contexts": { + "name": "contexts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "excluded_dates": { + "name": "excluded_dates", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ends_at": { + "name": "ends_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_schedule_workflow_block_deployment_unique": { + "name": "workflow_schedule_workflow_block_deployment_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_workflow_deployment_idx": { + "name": "workflow_schedule_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_archived_at_partial_idx": { + "name": "workflow_schedule_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6": { + "name": "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6", + "columns": [ + { + "expression": "source_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_workflow_idx": { + "name": "workflow_schedule_due_workflow_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND (\"workflow_schedule\".\"source_type\" = 'workflow' OR \"workflow_schedule\".\"source_type\" IS NULL)", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_job_idx": { + "name": "workflow_schedule_due_job_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND \"workflow_schedule\".\"source_type\" = 'job'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_schedule_workflow_id_workflow_id_fk": { + "name": "workflow_schedule_workflow_id_workflow_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk": { + "name": "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_operation", + "columnsFrom": ["deployment_operation_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_schedule_source_user_id_user_id_fk": { + "name": "workflow_schedule_source_user_id_user_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "user", + "columnsFrom": ["source_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_source_workspace_id_workspace_id_fk": { + "name": "workflow_schedule_source_workspace_id_workspace_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workspace", + "columnsFrom": ["source_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_subflows": { + "name": "workflow_subflows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_subflows_workflow_id_idx": { + "name": "workflow_subflows_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_subflows_workflow_type_idx": { + "name": "workflow_subflows_workflow_type_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_subflows_workflow_id_workflow_id_fk": { + "name": "workflow_subflows_workflow_id_workflow_id_fk", + "tableFrom": "workflow_subflows", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace": { + "name": "workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'#33C482'" + }, + "logo_url": { + "name": "logo_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_mode": { + "name": "workspace_mode", + "type": "workspace_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'grandfathered_shared'" + }, + "billed_account_user_id": { + "name": "billed_account_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "allow_personal_api_keys": { + "name": "allow_personal_api_keys", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "inbox_enabled": { + "name": "inbox_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "inbox_address": { + "name": "inbox_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inbox_provider_id": { + "name": "inbox_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "organization_assigned_at": { + "name": "organization_assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "forked_from_workspace_id": { + "name": "forked_from_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_owner_id_idx": { + "name": "workspace_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_organization_id_idx": { + "name": "workspace_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_mode_idx": { + "name": "workspace_mode_idx", + "columns": [ + { + "expression": "workspace_mode", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_forked_from_workspace_id_idx": { + "name": "workspace_forked_from_workspace_id_idx", + "columns": [ + { + "expression": "forked_from_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_owner_id_user_id_fk": { + "name": "workspace_owner_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_organization_id_organization_id_fk": { + "name": "workspace_organization_id_organization_id_fk", + "tableFrom": "workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workspace_billed_account_user_id_user_id_fk": { + "name": "workspace_billed_account_user_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["billed_account_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workspace_forked_from_workspace_id_workspace_id_fk": { + "name": "workspace_forked_from_workspace_id_workspace_id_fk", + "tableFrom": "workspace", + "tableTo": "workspace", + "columnsFrom": ["forked_from_workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_storage_used_bytes_non_negative": { + "name": "workspace_storage_used_bytes_non_negative", + "value": "\"workspace\".\"storage_used_bytes\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workspace_byok_keys": { + "name": "workspace_byok_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_byok_workspace_provider_idx": { + "name": "workspace_byok_workspace_provider_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_byok_keys_workspace_id_workspace_id_fk": { + "name": "workspace_byok_keys_workspace_id_workspace_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_byok_keys_created_by_user_id_fk": { + "name": "workspace_byok_keys_created_by_user_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_environment": { + "name": "workspace_environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_environment_workspace_unique": { + "name": "workspace_environment_workspace_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_environment_workspace_id_workspace_id_fk": { + "name": "workspace_environment_workspace_id_workspace_id_fk", + "tableFrom": "workspace_environment", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file": { + "name": "workspace_file", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_workspace_id_idx": { + "name": "workspace_file_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_key_idx": { + "name": "workspace_file_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_deleted_at_idx": { + "name": "workspace_file_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_workspace_deleted_partial_idx": { + "name": "workspace_file_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_workspace_id_workspace_id_fk": { + "name": "workspace_file_workspace_id_workspace_id_fk", + "tableFrom": "workspace_file", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_uploaded_by_user_id_fk": { + "name": "workspace_file_uploaded_by_user_id_fk", + "tableFrom": "workspace_file", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_file_key_unique": { + "name": "workspace_file_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file_collab_state": { + "name": "workspace_file_collab_state", + "schema": "", + "columns": { + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "doc_state": { + "name": "doc_state", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "source_hash": { + "name": "source_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_file_collab_state_file_id_workspace_files_id_fk": { + "name": "workspace_file_collab_state_file_id_workspace_files_id_fk", + "tableFrom": "workspace_file_collab_state", + "tableTo": "workspace_files", + "columnsFrom": ["file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_files": { + "name": "workspace_files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context": { + "name": "context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_files_key_active_unique": { + "name": "workspace_files_key_active_unique", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_folder_name_active_unique": { + "name": "workspace_files_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "original_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL AND \"workspace_files\".\"context\" = 'workspace' AND \"workspace_files\".\"workspace_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_chat_display_name_unique": { + "name": "workspace_files_chat_display_name_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"context\" = 'mothership' AND \"workspace_files\".\"chat_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_key_idx": { + "name": "workspace_files_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_user_id_idx": { + "name": "workspace_files_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_id_idx": { + "name": "workspace_files_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_folder_id_idx": { + "name": "workspace_files_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_context_idx": { + "name": "workspace_files_context_idx", + "columns": [ + { + "expression": "context", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_chat_id_idx": { + "name": "workspace_files_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_deleted_at_idx": { + "name": "workspace_files_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_deleted_partial_idx": { + "name": "workspace_files_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_files\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_files_user_id_user_id_fk": { + "name": "workspace_files_user_id_user_id_fk", + "tableFrom": "workspace_files", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_workspace_id_workspace_id_fk": { + "name": "workspace_files_workspace_id_workspace_id_fk", + "tableFrom": "workspace_files", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_folder_id_folder_id_fk": { + "name": "workspace_files_folder_id_folder_id_fk", + "tableFrom": "workspace_files", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workspace_files_chat_id_copilot_chats_id_fk": { + "name": "workspace_files_chat_id_copilot_chats_id_fk", + "tableFrom": "workspace_files", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_block_map": { + "name": "workspace_fork_block_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_workflow_id": { + "name": "parent_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_block_id": { + "name": "parent_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_workflow_id": { + "name": "child_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_block_id": { + "name": "child_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_block_map_child_ws_parent_unique": { + "name": "workspace_fork_block_map_child_ws_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_unique": { + "name": "workspace_fork_block_map_child_ws_child_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_parent_wf_idx": { + "name": "workspace_fork_block_map_child_ws_parent_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_wf_idx": { + "name": "workspace_fork_block_map_child_ws_child_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_block_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_block_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_block_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_dependent_value": { + "name": "workspace_fork_dependent_value", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workflow_id": { + "name": "target_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_block_key": { + "name": "sub_block_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_dependent_value_child_ws_wf_idx": { + "name": "workspace_fork_dependent_value_child_ws_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_dependent_value_field_unique": { + "name": "workspace_fork_dependent_value_field_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sub_block_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_dependent_value", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_promote_run": { + "name": "workspace_fork_promote_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workspace_id": { + "name": "target_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "workspace_fork_promote_direction", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_promote_run_child_ws_target_unique": { + "name": "workspace_fork_promote_run_child_ws_target_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_promote_run_target_ws_idx": { + "name": "workspace_fork_promote_run_target_ws_idx", + "columns": [ + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_promote_run_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_promote_run_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_promote_run_created_by_user_id_fk": { + "name": "workspace_fork_promote_run_created_by_user_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_resource_map": { + "name": "workspace_fork_resource_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "workspace_fork_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_resource_id": { + "name": "parent_resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_resource_id": { + "name": "child_resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_resource_map_child_ws_idx": { + "name": "workspace_fork_resource_map_child_ws_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_ws_type_idx": { + "name": "workspace_fork_resource_map_child_ws_type_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_type_parent_unique": { + "name": "workspace_fork_resource_map_child_type_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_resource_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_resource_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_resource_map_created_by_user_id_fk": { + "name": "workspace_fork_resource_map_created_by_user_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.academy_cert_status": { + "name": "academy_cert_status", + "schema": "public", + "values": ["active", "revoked", "expired"] + }, + "public.background_work_kind": { + "name": "background_work_kind", + "schema": "public", + "values": ["deployment_side_effects", "fork_content_copy", "fork_sync", "fork_rollback"] + }, + "public.background_work_status_value": { + "name": "background_work_status_value", + "schema": "public", + "values": ["pending", "processing", "completed", "completed_with_warnings", "failed"] + }, + "public.billing_blocked_reason": { + "name": "billing_blocked_reason", + "schema": "public", + "values": ["payment_failed", "dispute"] + }, + "public.billing_entity_type": { + "name": "billing_entity_type", + "schema": "public", + "values": ["user", "organization"] + }, + "public.chat_type": { + "name": "chat_type", + "schema": "public", + "values": ["mothership", "copilot"] + }, + "public.copilot_async_tool_status": { + "name": "copilot_async_tool_status", + "schema": "public", + "values": ["pending", "running", "completed", "failed", "cancelled", "delivered"] + }, + "public.copilot_run_status": { + "name": "copilot_run_status", + "schema": "public", + "values": ["active", "paused_waiting_for_tool", "resuming", "complete", "error", "cancelled"] + }, + "public.copilot_tool_permission_decision": { + "name": "copilot_tool_permission_decision", + "schema": "public", + "values": ["allow", "allow_chat", "always_allow", "skip"] + }, + "public.credential_member_role": { + "name": "credential_member_role", + "schema": "public", + "values": ["admin", "member"] + }, + "public.credential_member_status": { + "name": "credential_member_status", + "schema": "public", + "values": ["active", "pending", "revoked"] + }, + "public.credential_type": { + "name": "credential_type", + "schema": "public", + "values": ["oauth", "env_workspace", "env_personal", "service_account"] + }, + "public.data_drain_cadence": { + "name": "data_drain_cadence", + "schema": "public", + "values": ["hourly", "daily"] + }, + "public.data_drain_destination": { + "name": "data_drain_destination", + "schema": "public", + "values": ["s3", "gcs", "azure_blob", "datadog", "bigquery", "snowflake", "webhook"] + }, + "public.data_drain_run_status": { + "name": "data_drain_run_status", + "schema": "public", + "values": ["running", "success", "failed"] + }, + "public.data_drain_run_trigger": { + "name": "data_drain_run_trigger", + "schema": "public", + "values": ["cron", "manual"] + }, + "public.data_drain_source": { + "name": "data_drain_source", + "schema": "public", + "values": ["workflow_logs", "job_logs", "audit_logs", "copilot_chats", "copilot_runs"] + }, + "public.execution_large_value_reference_source": { + "name": "execution_large_value_reference_source", + "schema": "public", + "values": ["execution_log", "paused_snapshot"] + }, + "public.folder_resource_type": { + "name": "folder_resource_type", + "schema": "public", + "values": ["workflow", "file", "knowledge_base", "table"] + }, + "public.invitation_kind": { + "name": "invitation_kind", + "schema": "public", + "values": ["organization", "workspace"] + }, + "public.invitation_membership_intent": { + "name": "invitation_membership_intent", + "schema": "public", + "values": ["internal", "external"] + }, + "public.invitation_status": { + "name": "invitation_status", + "schema": "public", + "values": ["pending", "accepted", "rejected", "cancelled", "expired"] + }, + "public.permission_type": { + "name": "permission_type", + "schema": "public", + "values": ["admin", "write", "read"] + }, + "public.usage_log_category": { + "name": "usage_log_category", + "schema": "public", + "values": ["model", "fixed", "tool"] + }, + "public.usage_log_source": { + "name": "usage_log_source", + "schema": "public", + "values": [ + "workflow", + "wand", + "copilot", + "workspace-chat", + "mcp_copilot", + "mothership_block", + "knowledge-base", + "voice-input", + "enrichment" + ] + }, + "public.workspace_fork_promote_direction": { + "name": "workspace_fork_promote_direction", + "schema": "public", + "values": ["push", "pull"] + }, + "public.workspace_fork_resource_type": { + "name": "workspace_fork_resource_type", + "schema": "public", + "values": [ + "workflow", + "oauth_credential", + "service_account_credential", + "env_var", + "table", + "knowledge_base", + "knowledge_document", + "file", + "mcp_server", + "workflow_mcp_server", + "custom_tool", + "skill" + ] + }, + "public.workspace_mode": { + "name": "workspace_mode", + "schema": "public", + "values": ["personal", "organization", "grandfathered_shared"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index 43d813aafee..a0b2463aa2e 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -1933,6 +1933,13 @@ "when": 1785352177983, "tag": "0276_drop_legacy_folder_tables", "breakpoints": true + }, + { + "idx": 277, + "version": "7", + "when": 1785366320079, + "tag": "0277_workspace_file_collab_state", + "breakpoints": true } ] } diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 778f1aa8420..39eb0c2e7e6 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -31,6 +31,16 @@ export const tsvector = customType<{ }, }) +/** Raw binary column. Postgres `bytea` ↔ Node `Buffer` (the pg driver handles the encoding). */ +export const bytea = customType<{ + data: Buffer + driverData: Buffer +}>({ + dataType() { + return 'bytea' + }, +}) + export const user = pgTable('user', { id: text('id').primaryKey(), name: text('name').notNull(), @@ -1932,6 +1942,26 @@ export const workspaceFiles = pgTable( }) ) +/** + * Cached collaborative-document state for a workspace markdown file: the last-persisted Yjs binary and + * a hash of the markdown it was derived from. On a cold room open the seed loads this binary directly + * (the Hocuspocus load-document pattern) rather than re-converting markdown → Yjs — which avoids the + * "recreate the CRDT from a non-binary format" anti-pattern (fresh client ids / content duplication on + * reconnect) and the server-side headless-editor conversion. The row is STALE, and the seed re-converts + * from markdown, when `sourceHash` no longer matches the file's current markdown (edited externally by a + * copilot write or a direct save). One row per file; dropped by FK cascade when the file is deleted. + */ +export const workspaceFileCollabState = pgTable('workspace_file_collab_state', { + fileId: text('file_id') + .primaryKey() + .references(() => workspaceFiles.id, { onDelete: 'cascade' }), + /** `Y.encodeStateAsUpdate` of the collaborative doc at last persist — apply with `Y.applyUpdate`. */ + docState: bytea('doc_state').notNull(), + /** sha256 (hex) of the markdown `docState` was derived from — the freshness tag for cold-start. */ + sourceHash: text('source_hash').notNull(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), +}) + /** * Public share links for workspace resources. Polymorphic on `resourceType` so a * single mechanism serves files now and folders later. One row per resource diff --git a/packages/platform-authz/package.json b/packages/platform-authz/package.json index fd989954ec1..d8d0bca5b03 100644 --- a/packages/platform-authz/package.json +++ b/packages/platform-authz/package.json @@ -21,6 +21,10 @@ "./workflow": { "types": "./src/workflow.ts", "default": "./src/workflow.ts" + }, + "./rooms": { + "types": "./src/rooms.ts", + "default": "./src/rooms.ts" } }, "scripts": { @@ -32,6 +36,7 @@ }, "dependencies": { "@sim/db": "workspace:*", + "@sim/realtime-protocol": "workspace:*", "drizzle-orm": "^0.45.2" }, "devDependencies": { diff --git a/packages/platform-authz/src/rooms.ts b/packages/platform-authz/src/rooms.ts new file mode 100644 index 00000000000..6d76fdceb4f --- /dev/null +++ b/packages/platform-authz/src/rooms.ts @@ -0,0 +1,169 @@ +import { db, userTableDefinitions, workspace, workspaceFiles } from '@sim/db' +import { ROOM_TYPES, type RoomRef, type RoomType } from '@sim/realtime-protocol/rooms' +import { and, eq, isNull } from 'drizzle-orm' +import { + type PermissionType, + permissionSatisfies, + resolveEffectiveWorkspacePermission, +} from './workspace' + +export type { PermissionType, RoomRef, RoomType } + +/** + * The owning workspace of a room, plus the org that owns that workspace — the + * exact inputs {@link resolveEffectiveWorkspacePermission} needs. + */ +export interface RoomWorkspace { + workspaceId: string + workspaceOrganizationId: string | null +} + +/** + * Resolves a room's owning workspace from its {@link RoomRef.id}. Returns `null` + * when the underlying resource is missing/archived (→ a 404 authorization + * result). One resolver per workspace-scoped {@link RoomType}; this is the single + * place a new such room type declares its resource→workspace lookup. + */ +export type RoomWorkspaceResolver = (roomId: string) => Promise + +async function resolveWorkspaceRoomWorkspace(workspaceId: string): Promise { + const [row] = await db + .select({ id: workspace.id, organizationId: workspace.organizationId }) + .from(workspace) + .where(and(eq(workspace.id, workspaceId), isNull(workspace.archivedAt))) + .limit(1) + + return row ? { workspaceId: row.id, workspaceOrganizationId: row.organizationId } : null +} + +/** + * Resolves a collaborative file-document room to its owning workspace. The room + * id is the file id; look up its (active) workspace file, then reuse the + * workspace resolver so archival is honored uniformly. Returns `null` when the + * file is missing/soft-deleted or is not workspace-scoped (copilot/chat uploads + * carry a null `workspaceId` and have no collaborative editor). + */ +async function resolveFileDocWorkspace(fileId: string): Promise { + const [file] = await db + .select({ workspaceId: workspaceFiles.workspaceId }) + .from(workspaceFiles) + .where(and(eq(workspaceFiles.id, fileId), isNull(workspaceFiles.deletedAt))) + .limit(1) + + if (!file?.workspaceId) return null + return resolveWorkspaceRoomWorkspace(file.workspaceId) +} + +/** + * Resolves a table presence room to its owning workspace. The room id is the + * table id; look up its (non-archived) definition, then reuse the workspace + * resolver so archival is honored uniformly. Returns `null` when the table is + * missing or archived. + */ +async function resolveTableWorkspace(tableId: string): Promise { + const [table] = await db + .select({ workspaceId: userTableDefinitions.workspaceId }) + .from(userTableDefinitions) + .where(and(eq(userTableDefinitions.id, tableId), isNull(userTableDefinitions.archivedAt))) + .limit(1) + + if (!table?.workspaceId) return null + return resolveWorkspaceRoomWorkspace(table.workspaceId) +} + +/** + * Maps each workspace-scoped room type to its resource→workspace lookup. These + * rooms authorize uniformly through {@link authorizeRoom}: resolve *which* + * workspace the room belongs to, then gate on effective workspace permission. + * + * Workflow is intentionally absent: it authorizes through its own dedicated path + * (`authorizeWorkflowByWorkspacePermission` + the realtime role cache in + * `middleware/permissions`) and never flows through {@link authorizeRoom}. A + * workflow ref reaching here is therefore an unknown type (400) by design. + * Adding a *workspace-scoped* room type = adding one entry here. + */ +const ROOM_WORKSPACE_RESOLVERS: Partial> = { + // A workspace-files room is addressed directly by its workspace id. + [ROOM_TYPES.WORKSPACE_FILES]: resolveWorkspaceRoomWorkspace, + // A workspace-tables room is addressed directly by its workspace id. + [ROOM_TYPES.WORKSPACE_TABLES]: resolveWorkspaceRoomWorkspace, + // A file-doc room is addressed by file id; resolve it to its workspace. + [ROOM_TYPES.WORKSPACE_FILE_DOC]: resolveFileDocWorkspace, + // A table room is addressed by table id; resolve it to its workspace. + [ROOM_TYPES.TABLE]: resolveTableWorkspace, +} + +export interface RoomAuthorizationResult { + allowed: boolean + status: number + message?: string + workspaceId: string | null + workspacePermission: PermissionType | null +} + +/** + * Authorizes a user against a workspace-scoped realtime room (workspace-files, + * file-doc, table). Mirrors `authorizeWorkflowByWorkspacePermission` (the + * exemplary workflow authorizer) but generalized over room type: resolve the + * room's workspace, then gate on the user's effective workspace permission under + * the read < write < admin ordering. Workflow rooms use their own authorizer and + * do not pass through here (see {@link ROOM_WORKSPACE_RESOLVERS}). + * + * Returns a denial (never throws) for unknown room type (400), missing/archived + * resource (404), and insufficient permission (403), so realtime handlers and + * SSE routes can map the `status` to a wire error uniformly. + */ +export async function authorizeRoom(params: { + userId: string + room: RoomRef + action?: PermissionType +}): Promise { + const { userId, room, action = 'read' } = params + + const resolver = ROOM_WORKSPACE_RESOLVERS[room.type] + if (!resolver) { + // Either an unknown type or one deliberately outside this authorizer (workflow authorizes via + // its own path); both are not-authorizable-here → 400. + return { + allowed: false, + status: 400, + message: `Room type not authorizable here: ${room.type}`, + workspaceId: null, + workspacePermission: null, + } + } + + const roomWorkspace = await resolver(room.id) + if (!roomWorkspace) { + return { + allowed: false, + status: 404, + message: 'Room not found', + workspaceId: null, + workspacePermission: null, + } + } + + const workspacePermission = await resolveEffectiveWorkspacePermission( + userId, + roomWorkspace.workspaceId, + roomWorkspace.workspaceOrganizationId + ) + + if (!permissionSatisfies(workspacePermission, action)) { + return { + allowed: false, + status: 403, + message: `Access denied to ${action} this room`, + workspaceId: roomWorkspace.workspaceId, + workspacePermission, + } + } + + return { + allowed: true, + status: 200, + workspaceId: roomWorkspace.workspaceId, + workspacePermission, + } +} diff --git a/packages/realtime-protocol/package.json b/packages/realtime-protocol/package.json index a4248404adc..e1995aae687 100644 --- a/packages/realtime-protocol/package.json +++ b/packages/realtime-protocol/package.json @@ -21,10 +21,24 @@ "./schemas": { "types": "./src/schemas.ts", "default": "./src/schemas.ts" + }, + "./rooms": { + "types": "./src/rooms.ts", + "default": "./src/rooms.ts" + }, + "./file-doc": { + "types": "./src/file-doc.ts", + "default": "./src/file-doc.ts" + }, + "./table-presence": { + "types": "./src/table-presence.ts", + "default": "./src/table-presence.ts" } }, "scripts": { "type-check": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest", "lint": "biome check --write --unsafe .", "lint:check": "biome check .", "format": "biome format --write .", @@ -35,6 +49,7 @@ }, "devDependencies": { "@sim/tsconfig": "workspace:*", - "typescript": "^7.0.2" + "typescript": "^7.0.2", + "vitest": "^4.1.0" } } diff --git a/packages/realtime-protocol/src/file-doc.test.ts b/packages/realtime-protocol/src/file-doc.test.ts new file mode 100644 index 00000000000..48fd7db5b8a --- /dev/null +++ b/packages/realtime-protocol/src/file-doc.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from 'vitest' +import { FILE_DOC_TIMEOUTS } from './file-doc' + +describe('FILE_DOC_TIMEOUTS ordering invariants', () => { + it('bounds every nested call below the one that wraps it', () => { + // The relay's inner `/merge` fetch must finish before the app's outer `apply-edit` call gives up, + // or the relay applies a merge after the caller has already returned (racing a follow-on edit). + expect(FILE_DOC_TIMEOUTS.mergeRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.applyEditMs) + // The relay's `/seed` fetch must finish before the client's readiness deadline lapses into its + // read-only fallback, or a late-but-successful seed can never reach the client. + expect(FILE_DOC_TIMEOUTS.seedRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.readinessDeadlineMs) + }) +}) diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts new file mode 100644 index 00000000000..e58ceb477a3 --- /dev/null +++ b/packages/realtime-protocol/src/file-doc.ts @@ -0,0 +1,169 @@ +/** + * Wire protocol for the collaborative file-document room + * ({@link ROOM_TYPES.WORKSPACE_FILE_DOC}). Live carets and text selection ride + * Yjs document sync + awareness over the shared Socket.IO connection. These are + * the event names, binary message tags, and join payloads that the server + * (`apps/realtime/src/handlers/file-doc.ts`) and the client provider + * (`apps/sim/.../file-doc`) must agree on exactly — the single source of truth + * for both sides so they can never drift. + * + * The binary channel uses the standard Yjs "websocket" framing: every + * {@link FILE_DOC_EVENTS.MESSAGE} payload is a `Uint8Array` whose first varUint + * is a {@link FILE_DOC_MESSAGE_TYPE} tag (sync protocol vs awareness protocol), + * so the client provider can reuse `y-protocols` verbatim. + */ + +/** Socket.IO event names for the file-document collaboration channel. */ +export const FILE_DOC_EVENTS = { + /** Client → server: join a file's collaborative session ({@link JoinFileDocPayload}). */ + JOIN: 'join-file-doc', + /** Server → client: join accepted ({@link JoinFileDocSuccess}). */ + JOIN_SUCCESS: 'join-file-doc-success', + /** Server → client: join rejected ({@link JoinFileDocError}). */ + JOIN_ERROR: 'join-file-doc-error', + /** Client → server: leave the session ({@link LeaveFileDocPayload}). */ + LEAVE: 'leave-file-doc', + /** Both directions: a framed Yjs message (binary), tagged by {@link FILE_DOC_MESSAGE_TYPE}. */ + MESSAGE: 'file-doc-message', + /** + * Server → client: the roster of collaborators currently in the document + * ({@link FileDocPresence}), for the avatar stack. Identity is server-authenticated (from + * each socket's session), so — unlike the client-set awareness `user` field — a peer + * cannot spoof or suppress another user's entry. Re-sent on every join and leave. + */ + PRESENCE: 'file-doc-presence', +} as const + +/** + * The tag carried in the first varUint of a {@link FILE_DOC_EVENTS.MESSAGE} + * payload — the standard Yjs websocket framing distinguishing a document-sync + * message from an awareness (cursor/selection) message. + */ +export const FILE_DOC_MESSAGE_TYPE = { + SYNC: 0, + AWARENESS: 1, +} as const + +export type FileDocMessageType = (typeof FILE_DOC_MESSAGE_TYPE)[keyof typeof FILE_DOC_MESSAGE_TYPE] + +/** + * Where "the document's initial content has been seeded" is recorded, stored inside the Yjs + * document as `doc.getMap(configMap).get(flag) === true`. It lives in the CRDT so it merges across + * clients; the client reads it as half of its readiness gate (`synced && initialContentLoaded`). + * + * The server seeds authoritatively: it builds the document from the file's stored markdown and sets + * this flag in the SAME update, so a seeded document arrives content-and-flag together and the + * client needs no seed handshake. `configMap` is a reserved top-level Y.Map name; the editor must + * not use a top-level type of the same name (TipTap uses `getXmlFragment('default')`, no collision). + * + * Consumer (editor hook) contract: + * 1. **Never overwrite content with an unseeded doc.** The markdown-mirror autosave MUST be gated on + * the document being both synced AND seeded — otherwise an empty/still-syncing doc could be saved + * over the real file (the one true data-loss path). + * 2. **One provider per socket.** Destroy the previous provider before creating the next (document + * switch), so a stale provider's binary-frame listener can't apply another document's updates. + * 3. **Treat a fatal (`retryable: false`) join error as terminal.** Latch it and fall back to a + * read-only view of the file's stored content — do not keep rejoining. The server auto-reclaims a + * same-user client-id collision silently (the reconnecting socket succeeds), so `CLIENT_ID_IN_USE` + * surfaces to a client only for the rare case of a *different* user colliding on the same random + * Yjs `clientID`; a page reload mints a new id and recovers. + */ +export const FILE_DOC_SEED = { + configMap: 'config', + flag: 'initialContentLoaded', + /** + * The file's raw YAML frontmatter string, stored in the config map so it lives in the CRDT and + * merges across clients. The collaborative document body never contains frontmatter (it is stripped + * on seed); the editor re-attaches THIS value on autosave. Keeping it in the doc — rather than + * locked at open time — is what lets a server-side edit (e.g. copilot) that changes the frontmatter + * be reflected instead of reverted by an open editor's autosave. + */ + frontmatterKey: 'frontmatter', +} as const + +/** + * The timeouts for the file-doc conversion round-trips, in ONE place because two of the pairs form an + * ordering invariant that would otherwise live only in prose comments across two apps. A nested call's + * bound must exceed the bound of the call it wraps, or the outer aborts while the inner is still + * running — orphaning work that lands late (see the assertion in the accompanying test): + * + * - `applyEditMs` (app → relay `apply-edit`, in `apps/sim`) wraps `mergeRequestMs` (relay → app + * `/merge`, in `apps/realtime`), so `mergeRequestMs < applyEditMs`. + * - `readinessDeadlineMs` (the client's give-up-and-fall-back-read-only deadline) must outlast + * `seedRequestMs` (relay → app `/seed`), so `seedRequestMs < readinessDeadlineMs`. + * + * The seed request gets more headroom than the merge because it reads a (possibly cold) blob before + * converting; the merge is a pure in-memory conversion the caller fully supplies. + * + * `persistRequestMs` (relay → app `/persist`) stands alone — no client waits on it (the relay flushes + * the live doc to durable markdown debounced during editing and on the last collaborator leaving), so + * it forms no ordering invariant. It gets seed-level headroom because, like the seed, it crosses a + * durable blob write (Yjs → markdown → storage), not just an in-memory conversion. + */ +export const FILE_DOC_TIMEOUTS = { + seedRequestMs: 8_000, + mergeRequestMs: 3_000, + applyEditMs: 6_000, + readinessDeadlineMs: 12_000, + persistRequestMs: 8_000, +} as const + +/** Client → server join request. `fileId` is the `workspace_files.id`. */ +export interface JoinFileDocPayload { + fileId: string + /** + * The joining Yjs document's `clientID`. The server binds it to this socket so + * a client can only publish/remove awareness (cursor/selection) for its own + * client — an authenticated peer cannot forge or clear another's presence. + */ + clientId: number +} + +/** Server → client acceptance of a {@link FILE_DOC_EVENTS.JOIN}. */ +export interface JoinFileDocSuccess { + fileId: string +} + +/** Server → client rejection of a {@link FILE_DOC_EVENTS.JOIN}. */ +export interface JoinFileDocError { + fileId: string + error: string + code: string + retryable?: boolean +} + +/** Client → server leave request. */ +export interface LeaveFileDocPayload { + fileId: string +} + +/** One collaborator session in a {@link FileDocPresence} roster — server-authenticated identity. + * Keyed per socket (session), not per user: the client excludes its OWN `socketId` and then + * dedupes the rest per user for the avatar stack, so a second tab of the same account still + * registers as present (mirroring the canvas presence model). */ +export interface FileDocPresenceUser { + /** The collaborator's socket id, so a client can exclude its own session (not every session + * that happens to share its userId). */ + socketId: string + userId: string + userName: string + avatarUrl: string | null +} + +/** Server → client roster of who is in the document ({@link FILE_DOC_EVENTS.PRESENCE}). */ +export interface FileDocPresence { + fileId: string + users: FileDocPresenceUser[] +} + +/** + * Coerce a Socket.IO binary payload to a `Uint8Array`, or `null` if it is + * neither. Shared by the server relay and the client provider so the two agree + * on how an inbound {@link FILE_DOC_EVENTS.MESSAGE} frame is read (Socket.IO may + * deliver a `Uint8Array`/`Buffer` or an `ArrayBuffer` depending on runtime). + */ +export function toFileDocBytes(data: unknown): Uint8Array | null { + if (data instanceof Uint8Array) return data + if (data instanceof ArrayBuffer) return new Uint8Array(data) + return null +} diff --git a/packages/realtime-protocol/src/rooms.test.ts b/packages/realtime-protocol/src/rooms.test.ts new file mode 100644 index 00000000000..bb3651e9978 --- /dev/null +++ b/packages/realtime-protocol/src/rooms.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest' +import { + ALL_ROOM_TYPES, + isRoomType, + parseRoomName, + presenceEventName, + ROOM_TYPES, + type RoomRef, + roomName, +} from './rooms' + +describe('roomName', () => { + it('maps a workflow room to its bare id (backward compatibility)', () => { + expect(roomName({ type: ROOM_TYPES.WORKFLOW, id: 'wf-123' })).toBe('wf-123') + }) + + it('namespaces every non-workflow room type', () => { + expect(roomName({ type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-123' })).toBe( + 'workspace-files:ws-123' + ) + expect(roomName({ type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: 'file-123' })).toBe( + 'workspace-file-doc:file-123' + ) + }) + + it('keeps the file-doc and file-browser namespaces distinct for the same id', () => { + // `workspace-file-doc` must not be parsed as the `workspace-files` browser + // room (or vice versa): the prefix match is the whole segment before `:`. + const id = 'a1b2c3d4-uuid' + const doc = roomName({ type: ROOM_TYPES.WORKSPACE_FILE_DOC, id }) + const browser = roomName({ type: ROOM_TYPES.WORKSPACE_FILES, id }) + expect(doc).not.toBe(browser) + expect(parseRoomName(doc)).toEqual({ type: ROOM_TYPES.WORKSPACE_FILE_DOC, id }) + expect(parseRoomName(browser)).toEqual({ type: ROOM_TYPES.WORKSPACE_FILES, id }) + }) + + it('never collides a namespaced room with a bare workflow id for real ids', () => { + // Room ids in Sim are opaque tokens without a colon (UUIDs / short ids), so a + // bare workflow id can never look like a `${type}:${id}` namespaced name. + const workflow = roomName({ type: ROOM_TYPES.WORKFLOW, id: 'a1b2c3d4-uuid' }) + const files = roomName({ type: ROOM_TYPES.WORKSPACE_FILES, id: 'a1b2c3d4-uuid' }) + expect(workflow).not.toBe(files) + expect(workflow.includes(':')).toBe(false) + }) +}) + +describe('parseRoomName', () => { + it('round-trips every room type through roomName', () => { + const refs: RoomRef[] = [ + { type: ROOM_TYPES.WORKFLOW, id: 'wf-123' }, + { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-456' }, + { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: 'file-789' }, + { type: ROOM_TYPES.TABLE, id: 'table-abc' }, + ] + for (const ref of refs) { + expect(parseRoomName(roomName(ref))).toEqual(ref) + } + }) + + it('treats an unprefixed name as a workflow room', () => { + expect(parseRoomName('bare-uuid')).toEqual({ type: ROOM_TYPES.WORKFLOW, id: 'bare-uuid' }) + }) + + it('preserves ids that themselves contain colons', () => { + expect(parseRoomName('workspace-files:a:b:c')).toEqual({ + type: ROOM_TYPES.WORKSPACE_FILES, + id: 'a:b:c', + }) + }) + + it('does not treat an unknown prefix as a room type', () => { + expect(parseRoomName('unknown:x')).toEqual({ type: ROOM_TYPES.WORKFLOW, id: 'unknown:x' }) + }) + + it('returns null for the empty string', () => { + expect(parseRoomName('')).toBeNull() + }) +}) + +describe('isRoomType', () => { + it('accepts known types and rejects others', () => { + expect(isRoomType(ROOM_TYPES.WORKFLOW)).toBe(true) + expect(isRoomType(ROOM_TYPES.WORKSPACE_FILES)).toBe(true) + expect(isRoomType('nope')).toBe(false) + }) +}) + +describe('ALL_ROOM_TYPES', () => { + it('contains every declared room type', () => { + expect([...ALL_ROOM_TYPES].sort()).toEqual([...Object.values(ROOM_TYPES)].sort()) + }) +}) + +describe('presenceEventName', () => { + it('keeps the bare historical name for workflow and namespaces the rest', () => { + // Workflow keeps `presence-update` for client back-compat; every other type is namespaced so a + // socket in more than one room can demux its presence streams. + expect(presenceEventName(ROOM_TYPES.WORKFLOW)).toBe('presence-update') + expect(presenceEventName(ROOM_TYPES.TABLE)).toBe('table:presence-update') + expect(presenceEventName(ROOM_TYPES.WORKSPACE_FILES)).toBe('workspace-files:presence-update') + // No two types share an event name. + const names = ALL_ROOM_TYPES.map(presenceEventName) + expect(new Set(names).size).toBe(names.length) + }) +}) diff --git a/packages/realtime-protocol/src/rooms.ts b/packages/realtime-protocol/src/rooms.ts new file mode 100644 index 00000000000..afc3edde5a8 --- /dev/null +++ b/packages/realtime-protocol/src/rooms.ts @@ -0,0 +1,115 @@ +/** + * Room identity for the realtime layer. + * + * A {@link RoomRef} is the universal address shared by every realtime mechanism + * in Sim — the Socket.IO presence server (`apps/realtime`), the durable SSE + * event log, and the ephemeral pub/sub fanout. Each mechanism encodes a room + * differently on the wire, but they all agree on this `{ type, id }` identity + * and authorize it through the same workspace-permission resolver + * (`@sim/platform-authz/rooms`). + * + * This module is pure (no runtime dependencies) so both `apps/sim` and + * `apps/realtime` can import it. + */ + +/** + * The kinds of realtime room. Each value is a stable wire token — changing one + * is a breaking protocol change (it renames Socket.IO rooms and Redis keys), so + * treat these like enum values that ship to clients. + */ +export const ROOM_TYPES = { + /** The collaborative workflow editor canvas (one room per workflow). */ + WORKFLOW: 'workflow', + /** The workspace file browser (one room per workspace). */ + WORKSPACE_FILES: 'workspace-files', + /** + * A single collaborative file document — the rich-text editor for one file + * (one room per file). Carries Yjs document sync + awareness (live carets and + * text selection), so its id space is the file id, distinct from the + * workspace-scoped {@link ROOM_TYPES.WORKSPACE_FILES} browser room. + */ + WORKSPACE_FILE_DOC: 'workspace-file-doc', + /** + * A single table's grid (one room per table). Carries live cell-selection + * presence — which cells each viewer has selected — so its id space is the + * table id. + */ + TABLE: 'table', + /** + * The workspace tables browser (one room per workspace). The list-level + * counterpart to {@link ROOM_TYPES.TABLE}: it carries NO presence, only a + * lossy `workspace-tables-changed` invalidation signal so every viewer's + * tables list refetches when a table is created/renamed/moved/deleted. Its id + * space is the workspace id, mirroring {@link ROOM_TYPES.WORKSPACE_FILES}. + */ + WORKSPACE_TABLES: 'workspace-tables', +} as const + +export type RoomType = (typeof ROOM_TYPES)[keyof typeof ROOM_TYPES] + +/** Every known room type, for exhaustive iteration/validation. */ +export const ALL_ROOM_TYPES = Object.values(ROOM_TYPES) as readonly RoomType[] + +/** Universal address of a realtime room. */ +export interface RoomRef { + type: RoomType + id: string +} + +/** Type guard: whether an arbitrary string is a known {@link RoomType}. */ +export function isRoomType(value: string): value is RoomType { + return (ALL_ROOM_TYPES as readonly string[]).includes(value) +} + +/** + * The Socket.IO room name (and default key segment) for a room. + * + * `WORKFLOW` maps to the **bare id** — deliberately. The workflow editor has + * ~40 existing `io.to(workflowId)` / `socket.join(workflowId)` callsites that + * pass the bare workflow id, plus stale-cleanup that cross-references + * `io.in(workflowId).fetchSockets()` against Redis presence state. Preserving + * the bare name keeps every one of those callsites correct with zero diff and + * zero presence-state migration. Every *other* room type is namespaced + * (`${type}:${id}`) so a new id space can never collide with a workflow UUID. + * + * The inverse ({@link parseRoomName}) relies on this: an unprefixed name is a + * workflow, a prefixed name splits on the first `:`. + * + * Precondition: room ids are opaque tokens that never contain `:` — satisfied by + * every id in Sim (`generateId()` UUIDs, `generateShortId()` URL-safe tokens, + * workspace ids). This is what makes a bare workflow id unambiguous against a + * `${type}:${id}` namespace and keeps {@link parseRoomName} lossless. + */ +export function roomName(room: RoomRef): string { + return room.type === ROOM_TYPES.WORKFLOW ? room.id : `${room.type}:${room.id}` +} + +/** + * Inverse of {@link roomName}. A name carrying a known `${type}:` prefix parses + * to that type; any other (unprefixed) name is a {@link ROOM_TYPES.WORKFLOW} + * room whose id is the whole string — see {@link roomName} for why workflow is + * unprefixed. Returns `null` only for the empty string. + */ +export function parseRoomName(name: string): RoomRef | null { + if (!name) return null + + const separatorIndex = name.indexOf(':') + if (separatorIndex > 0) { + const maybeType = name.slice(0, separatorIndex) + if (isRoomType(maybeType) && maybeType !== ROOM_TYPES.WORKFLOW) { + return { type: maybeType, id: name.slice(separatorIndex + 1) } + } + } + + return { type: ROOM_TYPES.WORKFLOW, id: name } +} + +/** + * The `presence-update` broadcast event name for a room type. `WORKFLOW` keeps + * the historical bare `presence-update` name (client backward-compat); every + * other type is namespaced so a socket joined to more than one room can tell the + * presence streams apart on a single connection. + */ +export function presenceEventName(type: RoomType): string { + return type === ROOM_TYPES.WORKFLOW ? 'presence-update' : `${type}:presence-update` +} diff --git a/packages/realtime-protocol/src/table-presence.ts b/packages/realtime-protocol/src/table-presence.ts new file mode 100644 index 00000000000..45d06a94dc7 --- /dev/null +++ b/packages/realtime-protocol/src/table-presence.ts @@ -0,0 +1,103 @@ +/** + * Wire protocol for live table presence — which cell(s) each viewer has selected + * in a table grid. Carried over the shared, already-authenticated Socket.IO + * connection (the server relay is `apps/realtime/src/handlers/tables.ts`), + * separate from the one-way durable cell-status stream (`lib/table/events.ts`). + * + * Centralized here so the server emits and the client subscriptions cannot drift. + * This module is pure so both `apps/sim` and `apps/realtime` can import it. + */ + +/** Socket.IO event names for the table presence channel. */ +export const TABLE_PRESENCE_EVENTS = { + JOIN: 'join-table', + JOIN_SUCCESS: 'join-table-success', + JOIN_ERROR: 'join-table-error', + LEAVE: 'leave-table', + /** + * The sender's current cell selection. Sent client→server, then relayed + * server→peers with the sender's identity attached. + */ + CELL_SELECTION: 'table-cell-selection', +} as const + +/** + * A single cell address. Keyed by stable ids (never positional indices): row and + * column order differ per client under their own sort/filter, so an index would + * point at the wrong cell on the receiver. + */ +export interface TableCellRef { + rowId: string + columnId: string +} + +/** + * A viewer's grid selection: the `anchor` and `focus` corners of a rectangular + * range (a single-cell selection has `anchor` equal to `focus`). `null` clears + * the selection (the viewer has nothing selected). + */ +export type TableCellSelection = { + anchor: TableCellRef + focus: TableCellRef + /** + * True while the viewer is actively editing the `focus` cell (they double-clicked + * or started typing). Peers render the cell with a slightly darker fill — the + * Google-Sheets "someone is typing here" signal — on top of the color border. + */ + editing?: boolean +} | null + +/** Client→server join request for a table presence room. */ +export interface JoinTablePayload { + tableId: string + /** Stable per-tab id so a reconnecting tab replaces its own stale presence entry. */ + tabSessionId?: string +} + +/** Server→client rejection of a {@link TABLE_PRESENCE_EVENTS.JOIN}. */ +export interface JoinTableError { + tableId: string + error: string + code: + | 'AUTHENTICATION_REQUIRED' + | 'ROOM_MANAGER_UNAVAILABLE' + | 'INVALID_PAYLOAD' + | 'VERIFY_ACCESS_FAILED' + | 'NOT_FOUND' + | 'ACCESS_DENIED' + | 'JOIN_FAILED' + /** Whether re-attempting the join (e.g. after reconnect) could succeed. */ + retryable: boolean +} + +/** + * A remote viewer of a table, as carried in the join ack and every + * `table:presence-update` broadcast. `cell` is the viewer's current selection at + * broadcast time (absent until they select something). + */ +export interface TablePresenceUser { + socketId: string + userId: string + userName: string + avatarUrl?: string | null + cell?: TableCellSelection +} + +/** Server→client ack of a successful join, carrying the room's current viewers. */ +export interface JoinTableSuccess { + tableId: string + socketId: string + presenceUsers: TablePresenceUser[] +} + +/** + * A single remote viewer's cell-selection delta, relayed to peers on + * {@link TABLE_PRESENCE_EVENTS.CELL_SELECTION}. Lower-latency than a full presence + * broadcast for the frequent case of just moving the selection. Carries only the + * socket id + selection — peers already hold the viewer's identity (name, color) from + * the presence roster, so it is not repeated on this high-frequency channel. + */ +export interface TableCellSelectionBroadcast { + socketId: string + cell: TableCellSelection +} diff --git a/packages/realtime-protocol/vitest.config.ts b/packages/realtime-protocol/vitest.config.ts new file mode 100644 index 00000000000..471771e48fe --- /dev/null +++ b/packages/realtime-protocol/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + globals: false, + environment: 'node', + include: ['src/**/*.test.ts'], + }, +}) diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 31fd989cf4f..b508c48c80f 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 993, - zodRoutes: 993, + totalRoutes: 996, + zodRoutes: 996, nonZodRoutes: 0, } as const