diff --git a/apps/sim/components/settings/navigation.ts b/apps/sim/components/settings/navigation.ts index 0c51706e1e2..468c1a280cb 100644 --- a/apps/sim/components/settings/navigation.ts +++ b/apps/sim/components/settings/navigation.ts @@ -27,7 +27,7 @@ import { Users, Wrench, } from '@sim/emcn/icons' -import { type PermissionType, permissionSatisfies } from '@sim/platform-authz/workspace' +import { type PermissionType, permissionSatisfies } from '@sim/platform-authz/predicates' import { CodeIcon, McpIcon, SlackIcon } from '@/components/icons' import type { SettingsHeaderMeta } from '@/components/settings/settings-header' import type { DeploymentFeatures, DeploymentShape } from '@/lib/api/contracts/workspaces' diff --git a/apps/sim/instrumentation-node.ts b/apps/sim/instrumentation-node.ts index 1cad219a68f..fa33c50b228 100644 --- a/apps/sim/instrumentation-node.ts +++ b/apps/sim/instrumentation-node.ts @@ -378,6 +378,9 @@ export async function register() { await initializeOpenTelemetry() + const { reportAccountTokenSealing } = await import('@sim/db/sealed-text') + reportAccountTokenSealing() + const shutdownPostHog = async () => { try { const { getPostHogClient } = await import('@/lib/posthog/server') diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 423c949174b..7aab0d9aeb5 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -142,6 +142,7 @@ export const env = createEnv({ TURNSTILE_SECRET_KEY: z.string().min(1).optional(), // Cloudflare Turnstile secret key for captcha verification ENCRYPTION_KEY: z.string().min(32), // Key for encrypting sensitive data API_ENCRYPTION_KEY: z.string().min(32).optional(), // Dedicated key for encrypting API keys (optional for OSS) + ENCRYPT_ACCOUNT_TOKENS: z.boolean().optional(), // Seal OAuth access/refresh tokens on write (read by @sim/db); takes effect only with a 64-hex ENCRYPTION_KEY. Enable only once every app, worker, and background-task deployment runs a release that reads sealed tokens INTERNAL_API_SECRET: z.string().min(32), // Secret for internal API authentication INTERNAL_JWT_SECRET: z.string().min(32).optional(), // Dedicated signing key for internal JWTs (falls back to INTERNAL_API_SECRET); separating limits blast radius if one leaks diff --git a/apps/sim/lib/oauth/credential-revocation.integration.ts b/apps/sim/lib/oauth/credential-revocation.integration.ts index 9ec7b2f4c51..4f88564c700 100644 --- a/apps/sim/lib/oauth/credential-revocation.integration.ts +++ b/apps/sim/lib/oauth/credential-revocation.integration.ts @@ -5,7 +5,7 @@ import { account, credential, user, workspace } from '@sim/db/schema' import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' import { createDeferred, type Deferred } from '@sim/testing/helpers/deferred' import { generateId } from '@sim/utils/id' -import { eq, inArray } from 'drizzle-orm' +import { eq, inArray, sql } from 'drizzle-orm' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' const { redisUrl } = await vi.hoisted(async () => { @@ -36,6 +36,7 @@ import { fanOutSlackTokenChain } from '@/lib/oauth/slack' import { completeOAuthCredentialToken } from '@/lib/oauth/token-resolution' const originalFetch = globalThis.fetch +const fixtureEncryptionKey = process.env.ENCRYPTION_KEY const userId = generateId() const workspaceId = generateId() const accountId = generateId() @@ -81,6 +82,17 @@ async function storedRow(rowId: string) { return row } +async function rawTokens() { + const [row] = await db.execute<{ access_token: string | null; refresh_token: string | null }>( + sql`SELECT access_token, refresh_token FROM account WHERE id = ${accountId}` + ) + return row +} + +function sealWrites(enabled: boolean) { + process.env.ENCRYPT_ACCOUNT_TOKENS = enabled ? 'true' : 'false' +} + async function resolveToken(rowId = accountId) { return refreshTokenIfNeeded('revocation-integration', await storedRow(rowId), rowId) } @@ -177,6 +189,8 @@ beforeEach(async () => { }) afterEach((context) => { + sealWrites(false) + process.env.ENCRYPTION_KEY = fixtureEncryptionKey pausedProvider?.release.resolve() checks.push({ name: context.task.name, @@ -453,3 +467,79 @@ describe('OAuth refresh-token revocation against PostgreSQL and Redis', () => { }) }) }) + +describe('OAuth refresh chains with sealed account tokens', () => { + async function sealStoredChain() { + sealWrites(true) + await db + .update(account) + .set({ accessToken: 'expired-access', refreshToken: 'issued-refresh' }) + .where(eq(account.id, accountId)) + } + + it('stores tokens sealed and reads them back as plaintext', async () => { + await sealStoredChain() + + const raw = await rawTokens() + expect(raw.access_token).toMatch(/^enc:v1:/) + expect(raw.refresh_token).toMatch(/^enc:v1:/) + expect(JSON.stringify(raw)).not.toContain('issued-refresh') + expect(await storedAccount()).toMatchObject({ + accessToken: 'expired-access', + refreshToken: 'issued-refresh', + }) + }) + + it('keeps writing readable tokens when sealing is requested without a usable key', async () => { + sealWrites(true) + process.env.ENCRYPTION_KEY = 'not-hex'.repeat(6) + providerAnswer = REFRESHED + + await expect(resolveToken()).resolves.toEqual({ accessToken: 'fresh-access', refreshed: true }) + expect(await rawTokens()).toMatchObject({ access_token: 'fresh-access' }) + }) + + it('rotates a chain stored before sealing was enabled', async () => { + sealWrites(true) + providerAnswer = REFRESHED + + await expect(resolveToken()).resolves.toEqual({ accessToken: 'fresh-access', refreshed: true }) + expect(await rawTokens()).toMatchObject({ + access_token: expect.stringMatching(/^enc:v1:/), + refresh_token: 'issued-refresh', + }) + expect((await storedAccount()).accessToken).toBe('fresh-access') + }) + + it('rotates a sealed chain on a process that does not seal writes', async () => { + await sealStoredChain() + sealWrites(false) + providerAnswer = REFRESHED + + await expect(resolveToken()).resolves.toEqual({ accessToken: 'fresh-access', refreshed: true }) + expect(await rawTokens()).toMatchObject({ + access_token: 'fresh-access', + refresh_token: expect.stringMatching(/^enc:v1:/), + }) + }) + + it('records a revocation against a sealed chain on a process that does not seal writes', async () => { + await sealStoredChain() + sealWrites(false) + + await expect(resolveToken()).rejects.toBeInstanceOf(CredentialRevokedError) + await expect(getCredentialTerminalRefreshError(credentialId)).resolves.toEqual({ + errorCode: 'invalid_grant', + providerId: PROVIDER_ID, + }) + }) + + it('keeps a revocation recorded before the chain was sealed', async () => { + await expect(resolveToken()).rejects.toBeInstanceOf(CredentialRevokedError) + await expireRedisFlags() + await sealStoredChain() + + await expect(resolveToken()).rejects.toBeInstanceOf(CredentialRevokedError) + expect(providerRequests).toBe(1) + }) +}) diff --git a/apps/sim/lib/oauth/credential-service.test.ts b/apps/sim/lib/oauth/credential-service.test.ts index 4f54193f1ce..0553e56cce0 100644 --- a/apps/sim/lib/oauth/credential-service.test.ts +++ b/apps/sim/lib/oauth/credential-service.test.ts @@ -86,6 +86,11 @@ const serviceLogger = getMockLogger('OAuthCredentialService') const transportLogger = getMockLogger('GoogleServiceAccountTransport') const mocks = { ...hoisted, decryptSecret: encryptionMockFns.mockDecryptSecret } +beforeEach(() => { + // Schema mocks carry no table to seal against; credential-revocation.integration.ts covers sealing. + vi.stubEnv('ENCRYPTION_KEY', '') +}) + /** Every call the credential service and its token transport logged at the given levels. */ function loggedCalls(...levels: Array<'info' | 'warn' | 'error'>) { return levels.flatMap((level) => [ diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 2a7a79f0709..80ef8c8f9ad 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -1,6 +1,7 @@ import { createHmac, createSign } from 'crypto' import { db } from '@sim/db' import { account, credential } from '@sim/db/schema' +import { sealedTextEquals } from '@sim/db/sealed-text' import { createLogger } from '@sim/logger' import { getPostgresErrorCode, toError } from '@sim/utils/errors' import { and, desc, eq, isNotNull, sql } from 'drizzle-orm' @@ -973,7 +974,7 @@ async function recordRefreshRevocation( .where( and( chainRowsFilter(scope.accountId, scope.slackTeamId), - eq(account.refreshToken, rejectedRefreshToken), + sealedTextEquals(account.refreshToken, rejectedRefreshToken), // Raw sql params skip drizzle's column mapping, so the Date is serialized explicitly; // truncation keeps a version read back at millisecond precision comparable. A row that // vanished before it could be read has no version and matches nothing anyway. @@ -1366,7 +1367,9 @@ async function performCoalescedRefresh({ const rotated = await db .update(account) .set(updateData) - .where(and(eq(account.id, accountId), eq(account.refreshToken, refreshToken))) + .where( + and(eq(account.id, accountId), sealedTextEquals(account.refreshToken, refreshToken)) + ) .returning({ id: account.id }) if (rotated.length === 0) { const stored = await readStoredChain(accountId) diff --git a/apps/sim/trigger.config.ts b/apps/sim/trigger.config.ts index c6eca213fea..156c15712f7 100644 --- a/apps/sim/trigger.config.ts +++ b/apps/sim/trigger.config.ts @@ -133,6 +133,8 @@ export default defineConfig({ createLogger('TriggerInit').error('Could not distrust compromised CA roots', tlsTrust) } markInsideTriggerRun() + const { reportAccountTokenSealing } = await import('@sim/db/sealed-text') + reportAccountTokenSealing() const { warmRedisConnection } = await import('./lib/core/config/redis') await warmRedisConnection() }, diff --git a/bun.lock b/bun.lock index c5df653ec42..9dde037246c 100644 --- a/bun.lock +++ b/bun.lock @@ -500,6 +500,7 @@ "version": "0.1.0", "dependencies": { "@sim/logger": "workspace:*", + "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "drizzle-orm": "^0.45.4", "postgres": "^3.4.5", diff --git a/docker/db.Dockerfile b/docker/db.Dockerfile index 30ae055ae0c..b9e59e50481 100644 --- a/docker/db.Dockerfile +++ b/docker/db.Dockerfile @@ -12,9 +12,10 @@ WORKDIR /app # Copy only package files needed for migrations (these change less frequently) COPY package.json bun.lock turbo.json ./ COPY patches ./patches -RUN mkdir -p packages/db packages/logger packages/tsconfig packages/utils +RUN mkdir -p packages/db packages/logger packages/security packages/tsconfig packages/utils COPY packages/db/package.json ./packages/db/package.json COPY packages/logger/package.json ./packages/logger/package.json +COPY packages/security/package.json ./packages/security/package.json COPY packages/tsconfig/package.json ./packages/tsconfig/package.json COPY packages/utils/package.json ./packages/utils/package.json @@ -52,6 +53,9 @@ COPY --chown=nextjs:nodejs packages/utils ./packages/utils # Copy logger package (needed by @sim/db's tx-tripwire at import time) COPY --chown=nextjs:nodejs packages/logger ./packages/logger +# Copy security package (the schema's sealed columns import its encryption) +COPY --chown=nextjs:nodejs packages/security ./packages/security + # Copy database package source code (changes most frequently - placed last) COPY --chown=nextjs:nodejs packages/db ./packages/db diff --git a/package.json b/package.json index a840e359384..78ea6bb3339 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "check:spec-example-ids": "bun run scripts/check-spec-example-ids.ts", "check:script-tests": "bun run scripts/check-script-test-coverage.ts", "check:test-patterns": "bun run scripts/check-test-patterns.ts", + "check:sealed-columns": "bun run scripts/check-sealed-columns.ts", "check:schema-mock": "bun run scripts/generate-schema-mock.ts --check", "check:audits": "bun run scripts/run-audits.ts", "check:skills": "bun run scripts/sync-skills.ts --check", diff --git a/packages/db/package.json b/packages/db/package.json index 7157481f85b..3a9be2e1358 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -25,6 +25,10 @@ "types": "./pool-profiles.ts", "default": "./pool-profiles.ts" }, + "./sealed-text": { + "types": "./sealed-text.ts", + "default": "./sealed-text.ts" + }, "./sso-primary-provider": { "types": "./sso-primary-provider.ts", "default": "./sso-primary-provider.ts" @@ -44,6 +48,7 @@ "db:migrate": "bun --env-file=.env run ./scripts/migrate.ts", "db:reconcile-fork-kb-file-ownership": "bun --env-file=.env run ./scripts/reconcile-fork-kb-file-ownership.ts", "db:reconcile-workspace-storage": "bun --env-file=.env run ./scripts/reconcile-workspace-storage.ts", + "db:seal-account-tokens": "bun --env-file=.env run ./scripts/seal-account-tokens.ts", "db:retire-indexed-search": "bun --no-env-file ./scripts/retire-indexed-search.ts", "db:studio": "bunx drizzle-kit studio --config=./drizzle.config.ts", "test": "vitest run", @@ -55,6 +60,7 @@ }, "dependencies": { "@sim/logger": "workspace:*", + "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "drizzle-orm": "^0.45.4", "postgres": "^3.4.5" diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 4781ebb9165..753c90e7993 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -28,6 +28,7 @@ import { vector, } from 'drizzle-orm/pg-core' import { DEFAULT_FREE_CREDITS, TAG_SLOTS } from './constants' +import { sealedText } from './sealed-text' /** * Drizzle push compares index options as JSON, while Postgres introspection @@ -176,8 +177,8 @@ export const account = pgTable( userId: text('user_id') .notNull() .references(() => user.id, { onDelete: 'cascade' }), - accessToken: text('access_token'), - refreshToken: text('refresh_token'), + accessToken: sealedText('account', 'access_token'), + refreshToken: sealedText('account', 'refresh_token'), idToken: text('id_token'), accessTokenExpiresAt: timestamp('access_token_expires_at'), refreshTokenExpiresAt: timestamp('refresh_token_expires_at'), diff --git a/packages/db/script-migrations/0037_seal_account_tokens.integration.ts b/packages/db/script-migrations/0037_seal_account_tokens.integration.ts new file mode 100644 index 00000000000..0e3bb85de79 --- /dev/null +++ b/packages/db/script-migrations/0037_seal_account_tokens.integration.ts @@ -0,0 +1,199 @@ +import { sealAccountTokens } from '@sim/db/script-migrations/0037_seal_account_tokens' +import { openText } from '@sim/db/sealed-text' +import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' +import { seal, sealingKeyId } from '@sim/security/encryption' +import { generateId } from '@sim/utils/id' +import postgres, { type Sql } from 'postgres' +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +const databaseUrl = readTestDatabaseUrl() +const KEY_HEX = 'a'.repeat(64) +const KEY_ID = sealingKeyId(Buffer.from(KEY_HEX, 'hex')) +const ACCESS_CONTEXT = 'account.access_token' +const REFRESH_CONTEXT = 'account.refresh_token' + +interface StoredRow { + id: string + access_token: string | null + refresh_token: string | null + updated_at: string +} + +/** Copies the account table into an isolated disposable schema; no public rows are touched. */ +describe('sealing stored account tokens in PostgreSQL', () => { + let sql: Sql + let admin: Sql + const schemaName = `account_sealing_${generateId().replaceAll('-', '')}` + const previousKey = process.env.ENCRYPTION_KEY + + beforeAll(async () => { + process.env.ENCRYPTION_KEY = KEY_HEX + admin = postgres(databaseUrl, { max: 1, onnotice: () => undefined }) + await admin.unsafe(`CREATE SCHEMA "${schemaName}"`) + await admin.unsafe( + `CREATE TABLE "${schemaName}"."account" (LIKE public."account" INCLUDING DEFAULTS)` + ) + sql = postgres(databaseUrl, { + max: 2, + connection: { search_path: schemaName }, + onnotice: () => undefined, + }) + }) + + afterAll(async () => { + if (previousKey === undefined) Reflect.deleteProperty(process.env, 'ENCRYPTION_KEY') + else process.env.ENCRYPTION_KEY = previousKey + await sql?.end() + if (admin) { + await admin.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`) + await admin.end() + } + }) + + beforeEach(async () => { + await sql`TRUNCATE account` + }) + + async function insertAccount(accessToken: string | null, refreshToken: string | null) { + const id = generateId() + await sql`INSERT INTO account (id, account_id, provider_id, user_id, access_token, refresh_token, created_at, updated_at) + VALUES (${id}, ${`subject-${id}`}, 'google-drive', ${`user-${id}`}, ${accessToken}, ${refreshToken}, + '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')` + return id + } + + async function storedRows() { + return sql` + SELECT id, access_token, refresh_token, updated_at::text FROM account ORDER BY id` + } + + it('seals every stored token across batches without moving the chain version', async () => { + await Promise.all( + Array.from({ length: 5 }, (_, index) => insertAccount(`access-${index}`, `refresh-${index}`)) + ) + const accessOnly = await insertAccount('access-only', null) + const before = await storedRows() + + expect( + await sealAccountTokens(sql, { + mode: 'seal', + expectedKeyId: KEY_ID, + batchSize: 2, + pauseMs: 0, + }) + ).toEqual({ + keyId: KEY_ID, + scanned: 6, + pending: 6, + updated: 6, + foreignKey: 0, + }) + + const after = await storedRows() + for (const row of after) { + const original = before.find((candidate) => candidate.id === row.id) + expect(row.updated_at).toBe(original?.updated_at) + expect(row.access_token).toMatch(/^enc:v1:/) + expect(openText(row.access_token ?? '', ACCESS_CONTEXT)).toBe(original?.access_token) + if (row.id === accessOnly) { + expect(row.refresh_token).toBeNull() + } else { + expect(row.refresh_token).toMatch(/^enc:v1:/) + expect(openText(row.refresh_token ?? '', REFRESH_CONTEXT)).toBe(original?.refresh_token) + } + } + }) + + it('finds nothing left to do on a rerun or a dry run after a completed pass', async () => { + await insertAccount('access', 'refresh') + await sealAccountTokens(sql, { mode: 'seal', expectedKeyId: KEY_ID, pauseMs: 0 }) + const sealed = await storedRows() + + expect( + await sealAccountTokens(sql, { mode: 'seal', expectedKeyId: KEY_ID, pauseMs: 0 }) + ).toMatchObject({ + pending: 0, + updated: 0, + }) + expect( + await sealAccountTokens(sql, { + mode: 'seal', + expectedKeyId: KEY_ID, + dryRun: true, + pauseMs: 0, + }) + ).toMatchObject({ + pending: 0, + }) + expect(await storedRows()).toEqual(sealed) + }) + + it('reports pending rows on a dry run without writing them', async () => { + await insertAccount('access', 'refresh') + const before = await storedRows() + + expect( + await sealAccountTokens(sql, { + mode: 'seal', + expectedKeyId: KEY_ID, + dryRun: true, + pauseMs: 0, + }) + ).toEqual({ + keyId: KEY_ID, + scanned: 1, + pending: 1, + updated: 0, + foreignKey: 0, + }) + expect(await storedRows()).toEqual(before) + }) + + it('restores the exact plaintext when unsealing for a rollback', async () => { + await insertAccount('ya29.access:with:colons', 'refresh') + await insertAccount(null, 'refresh-only') + const plaintext = await storedRows() + await sealAccountTokens(sql, { mode: 'seal', expectedKeyId: KEY_ID, pauseMs: 0 }) + + expect( + await sealAccountTokens(sql, { mode: 'unseal', expectedKeyId: KEY_ID, pauseMs: 0 }) + ).toMatchObject({ + updated: 2, + }) + expect(await storedRows()).toEqual(plaintext) + }) + + it('refuses to start under a key the fleet does not use, before touching any row', async () => { + await insertAccount('access', 'refresh') + const before = await storedRows() + + for (const dryRun of [false, true]) { + await expect( + sealAccountTokens(sql, { mode: 'seal', expectedKeyId: 'ffffffff', dryRun, pauseMs: 0 }) + ).rejects.toThrow(KEY_ID) + } + await expect(sealAccountTokens(sql, { mode: 'seal', pauseMs: 0 })).rejects.toThrow() + expect(await storedRows()).toEqual(before) + }) + + it('skips and reports rows sealed under another key instead of failing the pass', async () => { + const otherKey = Buffer.from('b'.repeat(64), 'hex') + const foreignAccess = seal('foreign-access', otherKey, ACCESS_CONTEXT) + const foreign = await insertAccount(foreignAccess, 'foreign-refresh') + await insertAccount('own-access', 'own-refresh') + await sealAccountTokens(sql, { mode: 'seal', expectedKeyId: KEY_ID, pauseMs: 0 }) + + expect( + await sealAccountTokens(sql, { mode: 'unseal', expectedKeyId: KEY_ID, pauseMs: 0 }) + ).toMatchObject({ scanned: 2, updated: 1, foreignKey: 1 }) + const rows = await storedRows() + expect(rows.find((row) => row.id === foreign)).toMatchObject({ + access_token: foreignAccess, + refresh_token: 'foreign-refresh', + }) + expect(rows.find((row) => row.id !== foreign)).toMatchObject({ + access_token: 'own-access', + refresh_token: 'own-refresh', + }) + }) +}) diff --git a/packages/db/script-migrations/0037_seal_account_tokens.ts b/packages/db/script-migrations/0037_seal_account_tokens.ts new file mode 100644 index 00000000000..6a4354bf8c7 --- /dev/null +++ b/packages/db/script-migrations/0037_seal_account_tokens.ts @@ -0,0 +1,136 @@ +import { account } from '@sim/db/schema' +import { openText, readEncryptionKey, sealedTextContext, sealText } from '@sim/db/sealed-text' +import { createLogger } from '@sim/logger' +import { sealedKeyId, sealingKeyId } from '@sim/security/encryption' +import { sleep } from '@sim/utils/helpers' +import type { Sql } from 'postgres' + +const logger = createLogger('AccountTokenSealing', { enabled: true, logLevel: 'INFO' }) + +interface AccountTokenSealingOptions { + mode: 'seal' | 'unseal' + /** + * The id the fleet logs at startup (`Account token sealing`). A pass refuses to start under any + * other `ENCRYPTION_KEY`, so it can never seal rows with a key the fleet cannot open. Optional + * only for a dry run. + */ + expectedKeyId?: string + /** Counts the rows that would change without writing any. */ + dryRun?: boolean + batchSize?: number + /** Pause between batches, which keeps the backfill off the write path's latency. */ + pauseMs?: number +} + +interface AccountTokenSealingResult { + keyId: string + scanned: number + pending: number + updated: number + /** Rows left untouched because a token is sealed under another key. */ + foreignKey: number +} + +interface StoredTokens { + id: string + access_token: string | null + refresh_token: string | null +} + +const ACCESS_TOKEN_CONTEXT = sealedTextContext(account.accessToken) +const REFRESH_TOKEN_CONTEXT = sealedTextContext(account.refreshToken) + +/** + * Rewrites `account.access_token` and `account.refresh_token` into the sealed form (`seal`) or back + * to plaintext (`unseal`, the rollback). Idempotent and resumable: a value already in the target + * form is skipped, so a rerun only finishes what an interrupted run left. Each row is rewritten only + * while it still holds the value this pass read, so a refresh that rotates the chain concurrently + * wins, and `updated_at` is left alone because the refresh path reads it as the chain version. Rows + * sealed under another key are skipped and counted rather than failing the pass. + */ +export async function sealAccountTokens( + sql: Sql, + { + mode, + expectedKeyId, + dryRun = false, + batchSize = 200, + pauseMs = 100, + }: AccountTokenSealingOptions +): Promise { + const keyId = sealingKeyId(readEncryptionKey()) + if ((!dryRun || expectedKeyId !== undefined) && expectedKeyId !== keyId) { + throw new Error( + `ENCRYPTION_KEY has key id ${keyId}, not the expected ${expectedKeyId ?? '(none)'}` + ) + } + const isForeign = (value: string | null) => { + const sealedUnder = value === null ? null : sealedKeyId(value) + return sealedUnder !== null && sealedUnder !== keyId + } + const convert = (value: string | null, context: string) => + value === null ? null : mode === 'seal' ? sealText(value, context) : openText(value, context) + const result: AccountTokenSealingResult = { + keyId, + scanned: 0, + pending: 0, + updated: 0, + foreignKey: 0, + } + let afterId = '' + + for (;;) { + const rows = await sql` + SELECT id, access_token, refresh_token FROM account + WHERE id > ${afterId} AND (access_token IS NOT NULL OR refresh_token IS NOT NULL) + ORDER BY id LIMIT ${batchSize} + ` + if (rows.length === 0) break + afterId = rows[rows.length - 1].id + result.scanned += rows.length + + const ownRows = rows.filter( + (row) => !isForeign(row.access_token) && !isForeign(row.refresh_token) + ) + result.foreignKey += rows.length - ownRows.length + const changes = ownRows + .map((row) => ({ + row, + accessToken: convert(row.access_token, ACCESS_TOKEN_CONTEXT), + refreshToken: convert(row.refresh_token, REFRESH_TOKEN_CONTEXT), + })) + .filter( + ({ row, accessToken, refreshToken }) => + accessToken !== row.access_token || refreshToken !== row.refresh_token + ) + result.pending += changes.length + + if (changes.length > 0 && !dryRun) { + const updated = await sql.begin(async (tx) => { + await tx`SELECT set_config('lock_timeout', '5s', true), + set_config('statement_timeout', '30s', true)` + return tx<{ id: string }[]>` + UPDATE account + SET access_token = change.access_token, refresh_token = change.refresh_token + FROM unnest( + ${changes.map((change) => change.row.id)}::text[], + ${changes.map((change) => change.row.access_token)}::text[], + ${changes.map((change) => change.row.refresh_token)}::text[], + ${changes.map((change) => change.accessToken)}::text[], + ${changes.map((change) => change.refreshToken)}::text[] + ) AS change(id, stored_access_token, stored_refresh_token, access_token, refresh_token) + WHERE account.id = change.id + AND account.access_token IS NOT DISTINCT FROM change.stored_access_token + AND account.refresh_token IS NOT DISTINCT FROM change.stored_refresh_token + RETURNING account.id + ` + }) + result.updated += updated.length + } + + logger.info('Account token batch processed', { mode, dryRun, afterId, ...result }) + if (pauseMs > 0) await sleep(pauseMs) + } + + return result +} diff --git a/packages/db/script-migrations/types.ts b/packages/db/script-migrations/types.ts index d7008144ce0..9228e7ebd97 100644 --- a/packages/db/script-migrations/types.ts +++ b/packages/db/script-migrations/types.ts @@ -11,9 +11,9 @@ import type { Sql } from 'postgres' * resolves; a crash mid-run means the whole migration re-runs from the top * on the next upgrade. Guard with cheap preconditions (`WHERE x IS NULL`, * `ON CONFLICT DO NOTHING`) so re-runs are near-free. - * - **db-level imports only** (`postgres`, `drizzle-orm`, `@sim/utils`). The - * migrations docker image ships `packages/db` + `utils` + `logger` and - * nothing from `apps/*`. + * - **db-level imports only** (`postgres`, `drizzle-orm`, `@sim/utils`, + * `@sim/security`). The migrations docker image ships `packages/db` + `utils` + * + `logger` + `security` and nothing from `apps/*`. * - **Runs at whatever schema HEAD the release ships** — scripts are not * interleaved with SQL migrations. A SQL migration must never drop or * repurpose a column that a registered script still reads; delete the diff --git a/packages/db/scripts/seal-account-tokens.ts b/packages/db/scripts/seal-account-tokens.ts new file mode 100644 index 00000000000..41c0970eb9c --- /dev/null +++ b/packages/db/scripts/seal-account-tokens.ts @@ -0,0 +1,57 @@ +import { parseArgs } from 'node:util' +import { sealAccountTokens } from '@sim/db/script-migrations/0037_seal_account_tokens' +import { createLogger } from '@sim/logger' +import postgres from 'postgres' + +const logger = createLogger('AccountTokenSealing', { enabled: true, logLevel: 'INFO' }) +const REQUIRED_ACK = 'sealed-readers-deployed' + +const { values } = parseArgs({ + options: { + unseal: { type: 'boolean', default: false }, + 'dry-run': { type: 'boolean', default: false }, + 'expect-key-id': { type: 'string' }, + }, +}) + +const url = process.env.MIGRATION_DATABASE_URL || process.env.DATABASE_URL +if (!url) throw new Error('Missing MIGRATION_DATABASE_URL or DATABASE_URL') +if ( + !values['dry-run'] && + !values.unseal && + process.env.ACCOUNT_TOKEN_SEALING_ACK !== REQUIRED_ACK +) { + throw new Error( + `Set ACCOUNT_TOKEN_SEALING_ACK=${REQUIRED_ACK} only once every app, worker, and background ` + + 'task deployment runs a release that reads sealed account tokens' + ) +} + +if (!values['dry-run'] && !values['expect-key-id']) { + throw new Error( + 'Pass --expect-key-id with the key id the fleet logs at startup (a dry run prints this key id)' + ) +} + +/** The pass reads and writes the unqualified `account` table, so a URL-supplied path cannot redirect it. */ +const databaseUrl = new URL(url) +databaseUrl.searchParams.set('search_path', 'public, pg_temp') + +const sql = postgres(databaseUrl.toString(), { + max: 1, + prepare: false, + connect_timeout: 10, + max_lifetime: null, + connection: { application_name: 'sim-account-token-sealing' }, +}) + +try { + const result = await sealAccountTokens(sql, { + mode: values.unseal ? 'unseal' : 'seal', + expectedKeyId: values['expect-key-id'], + dryRun: values['dry-run'], + }) + logger.info('Account token sealing finished', result) +} finally { + await sql.end() +} diff --git a/packages/db/sealed-text.test.ts b/packages/db/sealed-text.test.ts new file mode 100644 index 00000000000..0c99a2d0bb2 --- /dev/null +++ b/packages/db/sealed-text.test.ts @@ -0,0 +1,21 @@ +import { account } from '@sim/db/schema' +import { sealedTextEquals } from '@sim/db/sealed-text' +import { alias, PgDialect } from 'drizzle-orm/pg-core' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +describe('sealedTextEquals', () => { + beforeEach(() => { + vi.stubEnv('ENCRYPTION_KEY', 'c'.repeat(64)) + vi.stubEnv('ENCRYPT_ACCOUNT_TOKENS', 'true') + }) + + it('matches the sealed form a write stores, through an alias of the table', () => { + const stored = account.refreshToken.mapToDriverValue('refresh-token') + const sibling = alias(account, 'sibling') + + const { params } = new PgDialect().sqlToQuery( + sealedTextEquals(sibling.refreshToken, 'refresh-token') + ) + expect(params).toEqual(['refresh-token', stored]) + }) +}) diff --git a/packages/db/sealed-text.ts b/packages/db/sealed-text.ts new file mode 100644 index 00000000000..abc6ec5abc2 --- /dev/null +++ b/packages/db/sealed-text.ts @@ -0,0 +1,116 @@ +import { createLogger } from '@sim/logger' +import { isSealed, seal, sealingKeyId, unseal } from '@sim/security/encryption' +import { toRecord } from '@sim/utils/object' +import { type SQL, sql } from 'drizzle-orm' +import { type AnyPgColumn, customType } from 'drizzle-orm/pg-core' + +const logger = createLogger('SealedText') +/** Operators read the key id from this report, so it logs at any configured level. */ +const reportLogger = createLogger('AccountTokenSealing', { enabled: true, logLevel: 'INFO' }) + +let reportedUnusableKey = false +let reportedSealing = false + +function readEncryptionKeyOrNull(): Buffer | null { + const hex = process.env.ENCRYPTION_KEY + return hex && /^[0-9a-f]{64}$/i.test(hex) ? Buffer.from(hex, 'hex') : null +} + +/** The deployment's `ENCRYPTION_KEY`; throws when it is not a 64-character hex string. */ +export function readEncryptionKey(): Buffer { + const key = readEncryptionKeyOrNull() + if (!key) throw new Error('ENCRYPTION_KEY must be set to a 64-character hex string (32 bytes)') + return key +} + +function isWriteSealingRequested(): boolean { + const flag = process.env.ENCRYPT_ACCOUNT_TOKENS?.toLowerCase() + return flag === 'true' || flag === '1' +} + +/** + * The key writes are sealed with, or null to write plaintext. Reads open sealed values regardless, + * so every process can read both forms before any process writes the sealed one. Sealing requested + * without a usable key writes plaintext, which every process can still read, and reports the + * misconfiguration once instead of failing every token write. + */ +function writeSealingKey(): Buffer | null { + if (!isWriteSealingRequested()) return null + const key = readEncryptionKeyOrNull() + if (!key && !reportedUnusableKey) { + reportedUnusableKey = true + logger.error( + 'ENCRYPT_ACCOUNT_TOKENS is set but ENCRYPTION_KEY is not a 64-character hex key; account tokens are written unsealed' + ) + } + return key +} + +/** Logs once per process whether writes seal account tokens, and the key id (never the key). */ +export function reportAccountTokenSealing(): void { + if (reportedSealing) return + reportedSealing = true + const writeKey = writeSealingKey() + const key = writeKey ?? readEncryptionKeyOrNull() + reportLogger.info(`Account token sealing: ${writeKey ? 'enabled' : 'disabled'}`, { + keyId: key ? sealingKeyId(key) : null, + }) +} + +/** + * The context a sealed-text column authenticates its values with (`account.access_token`), read + * from the column's own definition rather than rebuilt from its table, so an `alias()` of the + * table or a later rename never changes it. Drizzle keeps the field config on the column's + * `config`, which aliased columns share. + */ +export function sealedTextContext(column: AnyPgColumn): string { + const context = toRecord(toRecord(Reflect.get(column, 'config')).fieldConfig).context + if (typeof context !== 'string') + throw new Error(`Column ${column.name} is not a sealed-text column`) + return context +} + +/** Seals `plaintext` for `context` with `ENCRYPTION_KEY`; an already sealed value is returned as is. */ +export function sealText(plaintext: string, context: string): string { + return isSealed(plaintext) ? plaintext : seal(plaintext, readEncryptionKey(), context) +} + +/** The plaintext of a stored value, which is either sealed or written before sealing was enabled. */ +export function openText(stored: string, context: string): string { + return isSealed(stored) ? unseal(stored, readEncryptionKey(), context) : stored +} + +/** + * A `text` column whose values are sealed at rest (see `seal` in `@sim/security/encryption`) and + * bound to `.`, so a sealed value cannot be moved to another column. Drizzle reads + * return plaintext. Never compare the column with `eq`: a bound parameter is sealed on + * write-enabled processes and misses unsealed rows; use {@link sealedTextEquals} + * (`check:sealed-columns` enforces both). + */ +export function sealedText(table: string, column: string) { + const context = `${table}.${column}` + return customType<{ data: string; driverData: string; config: { context: string } }>({ + dataType() { + return 'text' + }, + toDriver(value) { + const key = isSealed(value) ? null : writeSealingKey() + return key ? seal(value, key, context) : value + }, + fromDriver(value) { + return openText(value, context) + }, + })(column, { context }) +} + +/** + * Matches rows whose sealed-text `column` holds `plaintext`, stored sealed or not. Sealing is + * deterministic, so equality stays a single atomic predicate. Without a usable key nothing can have + * been sealed, so only the plaintext form is compared. + */ +export function sealedTextEquals(column: AnyPgColumn, plaintext: string): SQL { + const key = readEncryptionKeyOrNull() + return key && !isSealed(plaintext) + ? sql`${column} IN (${plaintext}, ${seal(plaintext, key, sealedTextContext(column))})` + : sql`${column} = ${plaintext}` +} diff --git a/packages/security/src/encryption.test.ts b/packages/security/src/encryption.test.ts index cce33142a40..43f7da7fa0b 100644 --- a/packages/security/src/encryption.test.ts +++ b/packages/security/src/encryption.test.ts @@ -1,8 +1,9 @@ import { randomBytes } from 'node:crypto' import { describe, expect, it } from 'vitest' -import { decrypt, encrypt } from './encryption' +import { decrypt, encrypt, isSealed, seal, sealedKeyId, sealingKeyId, unseal } from './encryption' const KEY = Buffer.from('0'.repeat(64), 'hex') +const CONTEXT = 'account.refresh_token' describe('encrypt', () => { it('produces distinct ciphertexts for the same input', async () => { @@ -40,3 +41,64 @@ describe('decrypt', () => { await expect(decrypt(encrypted, otherKey)).rejects.toThrow() }) }) + +describe('seal', () => { + it('seals equal plaintexts to equal envelopes so stored values match by SQL equality', () => { + expect(seal('refresh-token', KEY, CONTEXT)).toBe(seal('refresh-token', KEY, CONTEXT)) + expect(seal('refresh-token', KEY, CONTEXT)).not.toBe(seal('refresh-tokem', KEY, CONTEXT)) + }) + + it('never leaves the plaintext readable in the envelope', () => { + const plaintext = 'xoxe-1-plaintext-token' + const sealed = seal(plaintext, KEY, CONTEXT) + expect(isSealed(sealed)).toBe(true) + expect(sealed).not.toContain(plaintext) + expect(sealed).not.toContain(Buffer.from(plaintext).toString('hex')) + }) +}) + +describe('unseal', () => { + it.each(['', 'ya29.a0:with:colons', 'unicode ✓ token'])('round-trips %j', (plaintext) => { + expect(unseal(seal(plaintext, KEY, CONTEXT), KEY, CONTEXT)).toBe(plaintext) + }) + + it('treats a hex-only or colon-separated raw token as unsealed', () => { + expect(isSealed('0123456789abcdef0123456789abcdef')).toBe(false) + expect(isSealed('aa:bb:cc')).toBe(false) + expect(() => unseal('aa:bb:cc', KEY, CONTEXT)).toThrow() + }) + + it('rejects an envelope moved to a different context', () => { + expect(() => unseal(seal('token', KEY, CONTEXT), KEY, 'account.access_token')).toThrow() + }) + + it('names the key an envelope was sealed under', () => { + expect(sealedKeyId(seal('token', KEY, CONTEXT))).toBe(sealingKeyId(KEY)) + expect(sealedKeyId('raw-token')).toBeNull() + }) + + it('rejects an envelope sealed under a different key', () => { + expect(() => unseal(seal('token', KEY, CONTEXT), randomBytes(32), CONTEXT)).toThrow( + /different key/ + ) + }) + + it('rejects a tampered ciphertext or auth tag', () => { + const parts = seal('token-value', KEY, CONTEXT).split(':') + const flip = (hex: string) => `${hex.slice(0, -1)}${hex.endsWith('0') ? '1' : '0'}` + const ciphertextIndex = parts.length - 2 + const tagIndex = parts.length - 1 + for (const index of [ciphertextIndex, tagIndex]) { + const tampered = [...parts] + tampered[index] = flip(tampered[index]) + expect(() => unseal(tampered.join(':'), KEY, CONTEXT)).toThrow() + } + }) + + it('treats a raw token that only starts like an envelope as unsealed', () => { + for (const value of ['enc:v1:raw-provider-token', `${seal('token', KEY, CONTEXT)}:extra`]) { + expect(isSealed(value)).toBe(false) + expect(() => unseal(value, KEY, CONTEXT)).toThrow(/not a sealed envelope/) + } + }) +}) diff --git a/packages/security/src/encryption.ts b/packages/security/src/encryption.ts index c1bbf078e38..31dfec3efb6 100644 --- a/packages/security/src/encryption.ts +++ b/packages/security/src/encryption.ts @@ -1,4 +1,11 @@ -import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto' +import { + createCipheriv, + createDecipheriv, + createHmac, + hkdfSync, + randomBytes, + timingSafeEqual, +} from 'node:crypto' /** * AES-256-GCM encryption primitive. Produces a self-contained string in the @@ -66,3 +73,89 @@ function assertKey(key: Buffer): void { throw new Error('Encryption key must be 32 bytes (256 bits)') } } + +const SEALED_PREFIX = 'enc:v1:' +const SEALED_IV_BYTES = 12 +const SEALED_KEY_ID_BYTES = 4 +const SEALED_ENVELOPE = /^enc:v1:([0-9a-f]{8}):([0-9a-f]{24}):((?:[0-9a-f]{2})*):([0-9a-f]{32})$/ + +interface SealingKeys { + key: Buffer + keyId: string + cipherKey: Buffer + ivKey: Buffer +} + +let cachedSealingKeys: SealingKeys | undefined + +function deriveSealingKeys(key: Buffer): SealingKeys { + const cached = cachedSealingKeys + if (cached && cached.key.length === key.length && timingSafeEqual(cached.key, key)) return cached + assertKey(key) + const derive = (info: string, length: number) => + Buffer.from(hkdfSync('sha256', key, Buffer.alloc(0), `sim:sealed:v1:${info}`, length)) + cachedSealingKeys = { + key: Buffer.from(key), + keyId: derive('key-id', SEALED_KEY_ID_BYTES).toString('hex'), + cipherKey: derive('cipher', 32), + ivKey: derive('iv', 32), + } + return cachedSealingKeys +} + +/** The non-secret id that envelopes sealed under `key` carry; safe to log. */ +export function sealingKeyId(key: Buffer): string { + return deriveSealingKeys(key).keyId +} + +/** Whether `value` is an envelope produced by {@link seal}. */ +export function isSealed(value: string): boolean { + return SEALED_ENVELOPE.test(value) +} + +/** The id of the key `value` was sealed under, or null when it is not an envelope. */ +export function sealedKeyId(value: string): string | null { + return SEALED_ENVELOPE.exec(value)?.[1] ?? null +} + +/** + * Deterministic AES-256-GCM envelope, `enc:v1::::` (hex). The IV + * is an HMAC of `context` and the plaintext under a subkey derived from `key` (the SIV + * construction), so equal plaintexts seal to equal envelopes and a stored envelope can be matched + * with SQL equality. That reveals only whether two sealed values are equal: use it for high-entropy + * secrets such as OAuth tokens, never for low-entropy values. `context` is authenticated, so an + * envelope opens only under the context it was sealed for (e.g. the column that stores it). + */ +export function seal(plaintext: string, key: Buffer, context: string): string { + const { keyId, cipherKey, ivKey } = deriveSealingKeys(key) + const iv = createHmac('sha256', ivKey) + .update(`${context}\0`, 'utf8') + .update(plaintext, 'utf8') + .digest() + .subarray(0, SEALED_IV_BYTES) + const cipher = createCipheriv('aes-256-gcm', cipherKey, iv, { authTagLength: 16 }) + cipher.setAAD(Buffer.from(context, 'utf8')) + const ciphertext = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]) + return `${SEALED_PREFIX}${keyId}:${iv.toString('hex')}:${ciphertext.toString('hex')}:${cipher.getAuthTag().toString('hex')}` +} + +/** + * Opens an envelope produced by {@link seal}. Throws when the value is not a well-formed envelope, + * was sealed under a different key or context, or fails GCM authentication. + */ +export function unseal(value: string, key: Buffer, context: string): string { + const match = SEALED_ENVELOPE.exec(value) + if (!match) throw new Error('Value is not a sealed envelope') + const [, keyId, ivHex, ciphertextHex, authTagHex] = match + const keys = deriveSealingKeys(key) + if (keyId !== keys.keyId) throw new Error('Sealed envelope was sealed under a different key') + const decipher = createDecipheriv('aes-256-gcm', keys.cipherKey, Buffer.from(ivHex, 'hex'), { + authTagLength: 16, + }) + decipher.setAAD(Buffer.from(context, 'utf8')) + decipher.setAuthTag(Buffer.from(authTagHex, 'hex')) + return Buffer.concat([ + decipher.update(Buffer.from(ciphertextHex, 'hex')), + decipher.final(), + ]).toString('utf8') +} diff --git a/scripts/check-sealed-columns.test.ts b/scripts/check-sealed-columns.test.ts new file mode 100644 index 00000000000..6d9e92a2154 --- /dev/null +++ b/scripts/check-sealed-columns.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from 'vitest' +import { findViolations, readSealedColumns } from './check-sealed-columns' + +const COLUMNS = readSealedColumns(` +export const account = pgTable( + 'account', + { + accessToken: sealedText('account', 'access_token'), + refreshToken: sealedText('account', 'refresh_token'), + } +) +`) + +function found(source: string, file = 'lib/file.ts') { + return findViolations(file, source, COLUMNS) + .map(({ rule, line }) => [rule, line]) + .sort(([, a], [, b]) => Number(a) - Number(b)) +} + +describe('check-sealed-columns', () => { + it('reads every sealed column with its physical name', () => { + expect(COLUMNS).toMatchObject([ + { reference: 'account.accessToken', columnName: 'access_token' }, + { reference: 'account.refreshToken', columnName: 'refresh_token' }, + ]) + }) + + it('flags comparison operators and raw interpolations but not the sealed helper', () => { + const source = [ + 'where(eq(account.refreshToken, token))', + 'where(inArray(schema.account.accessToken, tokens))', + `sql\`\${account.accessToken} IS NOT NULL\``, + 'where(like(account.accessToken, prefix))', + 'where(gte(account.refreshToken, token))', + 'where(sealedTextEquals(account.refreshToken, token))', + 'where(isNotNull(account.refreshToken))', + 'select({ accessToken: account.accessToken })', + ].join('\n') + expect(found(source)).toEqual([ + ['comparison', 1], + ['comparison', 2], + ['raw-interpolation', 3], + ['comparison', 4], + ['comparison', 5], + ]) + }) + + it('follows renamed imports and alias() bindings of the table', () => { + const source = [ + "import { account as accountTable, user } from '@sim/db/schema'", + 'where(eq(accountTable.refreshToken, token))', + "const sibling = alias(account, 'sibling')", + 'where(ne(sibling.accessToken, token))', + ].join('\n') + expect(found(source)).toEqual([ + ['comparison', 2], + ['comparison', 4], + ]) + }) + + it('flags physical column names in raw SQL outside tests and the backfill', () => { + const source = [ + `await db.execute(sql\`SELECT id FROM account WHERE id = \${id}\nAND access_token IS NULL\`)`, + "sql.raw('refresh_token IS NULL')", + `sql\`SELECT id FROM oauth_refresh_token WHERE grant = \${'refresh_token'}\``, + "sql`SELECT * FROM account WHERE user_id = 'u'`", + 'sql`SELECT a.* FROM account_activity a`', + "sql.identifier('refresh_token')", + ].join('\n') + expect(found(source)).toEqual([ + ['raw-column-name', 2], + ['raw-column-name', 3], + ['raw-column-name', 5], + ['raw-column-name', 7], + ]) + expect(found(source, 'lib/file.integration.ts')).toEqual([]) + expect(found(source, 'packages/db/script-migrations/0037_seal_account_tokens.ts')).toEqual([]) + }) +}) diff --git a/scripts/check-sealed-columns.ts b/scripts/check-sealed-columns.ts new file mode 100644 index 00000000000..ff2ef89c48f --- /dev/null +++ b/scripts/check-sealed-columns.ts @@ -0,0 +1,202 @@ +#!/usr/bin/env bun +/** + * Fails when a sealed-text column (`sealedText(...)` in `packages/db/schema.ts`) is compared or + * read in a way that bypasses its sealing. + * + * - `comparison`: a Drizzle comparison operator against the column. Its bound parameter goes + * through the column's mapper, so a process that seals writes compares the sealed form and + * misses rows stored before sealing (ordering and pattern operators are meaningless on + * ciphertext). `sealedTextEquals` from `@sim/db/sealed-text` matches both forms; `isNull` and + * `isNotNull` are unaffected and allowed. + * - `raw-interpolation`: `${table.column}` inside a raw `sql` template. A raw expression is read + * or compared as stored, so it sees ciphertext; select the column itself, or use the helper. + * - `raw-column-name`: the physical column name inside a `sql` template or `sql.raw(...)` body, a + * `sql.identifier(...)` of it, or a `* FROM
` wildcard over its table, all of which read + * the stored form the same way. Tests and the sealing backfill read it on purpose. + * + * Table aliases count as the table: renamed imports from `@sim/db` / `@sim/db/schema` and + * `alias(table, ...)` bindings. + * + * Swept clean when it landed, so there is no baseline: every hit fails. + * + * Run: `bun run check:sealed-columns` + */ +import { readdirSync, readFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { escapeRegExp } from '@sim/utils/string' + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const SCHEMA = 'packages/db/schema.ts' +const SKIPPED_FILES = new Set([ + 'packages/db/sealed-text.ts', + 'scripts/check-sealed-columns.test.ts', +]) +/** Files that read the stored form on purpose, so only `raw-column-name` skips them. */ +const STORED_FORM_READERS = + /(?:\.test\.tsx?|\.integration\.ts|\/scripts\/test-[\w-]+-e2e\.ts)$|^packages\/db\/script-migrations\/0037_seal_account_tokens\.ts$/ +const SCAN_DIRS = ['apps', 'packages', 'scripts'] +const SKIP_DIRS = new Set(['node_modules', '.next', '.turbo', 'coverage', 'dist', 'build', 'out']) +const SOURCE_FILE = /\.(?:ts|tsx|mts|cts)$/ + +const TABLE_DECLARATION = /export const (\w+) = pgTable\(\s*'(\w+)'/g +const SEALED_COLUMN = /(\w+):\s*sealedText\(\s*'(\w+)'\s*,\s*'(\w+)'\s*\)/g +const COMPARISON_OPERATORS = [ + 'eq', + 'ne', + 'gt', + 'gte', + 'lt', + 'lte', + 'between', + 'notBetween', + 'like', + 'notLike', + 'ilike', + 'notIlike', + 'inArray', + 'notInArray', + 'arrayContains', + 'arrayContained', + 'arrayOverlaps', +] +const SQL_TEMPLATE = /\bsql(?:<[^>\n]*>)?`((?:[^`\\]|\\.)*)`/g +const SQL_RAW = /\bsql\.raw\(\s*(['"`])((?:(?!\1)[\s\S])*)\1/g +const SQL_IDENTIFIER = /\bsql\.identifier\(\s*(['"`])(\w+)\1/g + +type Rule = 'comparison' | 'raw-interpolation' | 'raw-column-name' + +interface Violation { + file: string + line: number + rule: Rule + text: string +} + +interface SealedColumn { + /** The table's export name (`account`). */ + table: string + /** The column's property (`accessToken`). */ + property: string + /** The physical table name (`account`). */ + tableName: string + /** The physical column name (`access_token`). */ + columnName: string + /** How code references it, for reports (`account.accessToken`). */ + reference: string +} + +function lineOf(source: string, index: number): number { + return source.slice(0, index).split('\n').length +} + +/** Reads every sealed column from the schema. */ +export function readSealedColumns(schemaSource: string): SealedColumn[] { + const tables = [...schemaSource.matchAll(TABLE_DECLARATION)].map((match) => ({ + exportName: match[1], + tableName: match[2], + index: match.index ?? 0, + })) + const columns: SealedColumn[] = [] + for (const match of schemaSource.matchAll(SEALED_COLUMN)) { + const [, property, , columnName] = match + const table = tables.filter((candidate) => candidate.index < (match.index ?? 0)).at(-1) + if (!table) continue + columns.push({ + table: table.exportName, + tableName: table.tableName, + property, + columnName, + reference: `${table.exportName}.${property}`, + }) + } + return columns +} + +/** + * The local names a file binds `table` to: itself, any `import { table as x }` rename, and any + * `const x = alias(table, ...)`. + */ +function tableNames(source: string, table: string): string[] { + const escaped = escapeRegExp(table) + const rename = new RegExp(`\\b${escaped}\\s+as\\s+(\\w+)`, 'g') + const imports = + source.match(/import\s+(?:type\s+)?\{[^}]*\}\s*from\s*'@sim\/db(?:\/schema)?'/g) ?? [] + const renames = imports.flatMap((statement) => [...statement.matchAll(rename)].map((m) => m[1])) + const aliasBinding = new RegExp(`\\b(\\w+)\\s*=\\s*alias\\(\\s*(?:\\w+\\.)?${escaped}\\b`, 'g') + const aliases = [...source.matchAll(aliasBinding)].map((m) => m[1]) + return [table, ...renames, ...aliases] +} + +/** Finds comparisons and raw reads of `columns` in one source file. */ +export function findViolations(file: string, source: string, columns: SealedColumn[]): Violation[] { + const violations: Violation[] = [] + const report = (rule: Rule, index: number, text: string) => + violations.push({ file, line: lineOf(source, index), rule, text }) + const operators = COMPARISON_OPERATORS.join('|') + for (const { table, property } of columns) { + const names = tableNames(source, table).map(escapeRegExp).join('|') + const column = `(?:\\w+\\.)?(?:${names})\\.${escapeRegExp(property)}\\b` + const patterns: [Rule, RegExp][] = [ + ['comparison', new RegExp(`\\b(?:${operators})\\(\\s*${column}`, 'g')], + ['raw-interpolation', new RegExp(`\\$\\{\\s*${column}\\s*\\}`, 'g')], + ] + for (const [rule, pattern] of patterns) { + for (const match of source.matchAll(pattern)) report(rule, match.index ?? 0, match[0]) + } + } + if (STORED_FORM_READERS.test(file)) return violations + const columnNames = new Set(columns.map((column) => column.columnName)) + const names = new RegExp(`\\b(?:${[...columnNames].map(escapeRegExp).join('|')})\\b`) + const tables = [...new Set(columns.map((column) => column.tableName))].map(escapeRegExp).join('|') + const wildcard = new RegExp(`\\*\\s*from\\s+(?:"?public"?\\.)?"?(?:${tables})"?(?![\\w"])`, 'i') + for (const pattern of [SQL_TEMPLATE, SQL_RAW]) { + for (const match of source.matchAll(pattern)) { + const capture = match[match.length - 1] + const bodyOffset = (match.index ?? 0) + match[0].lastIndexOf(capture) + const body = capture.replace(/\$\{[^}]*\}/g, (value) => ' '.repeat(value.length)) + const hit = names.exec(body) ?? wildcard.exec(body) + if (hit) report('raw-column-name', bodyOffset + hit.index, hit[0]) + } + } + for (const match of source.matchAll(SQL_IDENTIFIER)) { + if (columnNames.has(match[2])) report('raw-column-name', match.index ?? 0, match[0]) + } + return violations +} + +function* sourceFiles(dir: string): Generator { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + if (!SKIP_DIRS.has(entry.name)) yield* sourceFiles(path.join(dir, entry.name)) + } else if (SOURCE_FILE.test(entry.name) && !entry.name.endsWith('.d.ts')) { + yield path.join(dir, entry.name) + } + } +} + +if (import.meta.main) { + const columns = readSealedColumns(readFileSync(path.join(ROOT, SCHEMA), 'utf8')) + const violations: Violation[] = [] + if (columns.length === 0) { + console.error(`No sealedText columns found in ${SCHEMA}; update this audit with the schema.`) + process.exit(1) + } + for (const dir of SCAN_DIRS) { + for (const absolute of sourceFiles(path.join(ROOT, dir))) { + const file = path.relative(ROOT, absolute) + if (SKIPPED_FILES.has(file)) continue + violations.push(...findViolations(file, readFileSync(absolute, 'utf8'), columns)) + } + } + if (violations.length > 0) { + for (const violation of violations) { + console.error(`${violation.file}:${violation.line} [${violation.rule}] ${violation.text}`) + } + console.error( + `\n${violations.length} sealed-column violation(s). Compare with sealedTextEquals from @sim/db/sealed-text, and select the column instead of interpolating it into raw SQL.` + ) + process.exit(1) + } + console.log(`✓ sealed columns (${columns.map((column) => column.reference).join(', ')})`) +}