diff --git a/apps/backend/docs/message-encryption-migration.md b/apps/backend/docs/message-encryption-migration.md new file mode 100644 index 00000000..c903d1f0 --- /dev/null +++ b/apps/backend/docs/message-encryption-migration.md @@ -0,0 +1,135 @@ +# Message encryption migration: plaintext → ciphertext-only + +This document is the canonical reference linked from `GET /conversations/:id/search`'s +410 response and from the schema-overhaul audit — it records the decision behind +`drizzle/0003_ciphertext_only_messages.sql` and the policy for any plaintext that +existed before it. + +## Background + +Earlier in this project's history, `messages` carried a plaintext `content` column +(plus a GIN index supporting server-side search) alongside the work to move to +per-recipient E2EE envelopes (`message_envelopes`) and a `ciphertext` column on +`messages` itself. `docs/e2ee-onboarding.md` documents the current, implemented +E2EE data model; this document covers the one-time cutover away from the old +plaintext column and what happens to data that predates it. + +Server-side message search was removed as part of this cutover — the server +cannot search ciphertext it cannot read, so `GET /conversations/:id/search` now +returns `410 Gone` pointing here, and search is client-side over decrypted +messages. + +## Decision: archive-then-purge (not a tombstone) + +Two options were considered for handling rows where `messages.content` is +non-null at migration time: + +1. **Tombstone in place** — keep a `content`-shaped column (or a + `legacyPlaintext` column) on `messages` forever, nulled out or flagged, so + the "this row used to be plaintext" fact stays attached to the row. +2. **Archive then purge** (chosen) — copy every non-null `content` value into + a separate `message_content_archive` table, then drop the column from + `messages` entirely. + +**Archive-then-purge was chosen** because: + +- It keeps `messages` fully ciphertext-shaped going forward. Every code path + that reads a message (`serializeMessage()` in `lib/messages.ts`, the + `conversations`/`messages` routes, the socket layer) only ever has a + `ciphertext` branch to reason about — there is no lingering plaintext + branch that a future change could accidentally serve to a client as if it + were normal ciphertext. +- `message_content_archive` has **no route or query path through the app's + API** (see `db/schema.ts` — the table is defined but never imported by + anything in `routes/`, `services/`, or `socket/`). It exists purely for + legal/compliance/audit access on its own retention schedule, separate from + the live message store's. +- It avoids growing the "hot path" `messages` table with a rarely-needed + legacy column and its GIN index indefinitely. + +The rejected tombstone approach was ruled out because it permanently taxes +every future `messages` migration and query with a legacy-shaped column for +data that, by definition, existed only before this cutover and will never +grow again. + +## No silent E2EE claim over old plaintext + +Rows whose plaintext was archived are **not** presented to clients as if they +were ciphertext. Once `content` is dropped, `serializeMessage()`'s existing +fallback chain (envelope → base `ciphertext` → `unavailable: true`) means a +pre-cutover row with no `ciphertext` and no matching envelope now correctly +resolves to `{ ciphertext: null, unavailable: true }` — the client sees the +message as unavailable rather than being shown a decrypted-looking payload +that was, in fact, sent before E2EE existed. Nothing in this migration +retroactively re-encrypts old plaintext or backdates an "encrypted" claim +onto it. + +## What the forward migration does + +`drizzle/0003_ciphertext_only_messages.sql`: + +1. Creates `message_content_archive` (id, `original_message_id` — intentionally + **not** a foreign key, so hard-deleting a `messages` row never cascades + into the archive — `conversation_id`, `sender_id`, `content`, + `original_created_at`, `archived_at`). +2. If (and only if) `messages.content` still exists — checked dynamically via + `information_schema.columns`, since a plain SQL statement referencing a + column that doesn't exist fails to parse — copies every non-null row into + the archive. +3. Drops the column's GIN index. The exact index name from before this + repo's migration history was squashed to a single `0000` migration is not + recoverable, so several plausible historical names are dropped + defensively with `DROP INDEX IF EXISTS` (a no-op for any name that + doesn't match). +4. Drops `messages.content` with `DROP COLUMN IF EXISTS` — a no-op on a + database that never had it, which is the case for this repo's own `0000` + migration. +5. Defensively ensures the ciphertext-model columns/tables (`messages.ciphertext`, + `senderDeviceId`, `fileId`, `editsMessageId`, `deletedAt`, + `message_envelopes`) exist, for any database whose applied-migration + history predates the `0000` squash. On every database that already ran + `0000` this is a pure no-op. + +This makes the migration safe to run both against this repo's current +(already-ciphertext) schema and against a hypothetical legacy database that +still has the old plaintext column and a populated `messages` table. + +## Rollback plan + +`drizzle-kit` has no built-in down-migration runner, so the rollback lives at +`drizzle/rollback/0003_ciphertext_only_messages.down.sql` and is invoked +manually, e.g.: + +```sh +psql "$DATABASE_URL" -f apps/backend/drizzle/rollback/0003_ciphertext_only_messages.down.sql +``` + +It: + +1. Re-adds `messages.content` (nullable). +2. Recreates a GIN full-text index over it. +3. Restores plaintext from `message_content_archive` back into `messages.content` + for every row the archive has a match for. + +**Rollback limitations** (also called out at the top of the script itself): + +- Only rows present in the archive get plaintext restored. Any message sent + *after* the forward migration ran was only ever stored as ciphertext — + there is nothing to restore for it, by design. +- The recreated index is a reasonable equivalent, not necessarily + byte-identical to whatever definition existed pre-squash. +- The rollback does **not** remove the ciphertext/envelope/device-capability/ + GC schema this migration set introduces — undoing those would remove the + entire E2EE data model, which is a separate, deliberate decision this + script does not make on an operator's behalf. +- `message_content_archive` is left in place after a rollback (nothing is + destroyed); drop it manually once you're satisfied the restore is correct. + +## Retention of the archive + +`message_content_archive` is not currently covered by the background GC jobs +in `services/deviceGc.ts` / `services/envelopeGc.ts` / `services/fileCleanup.ts` +— it is intentionally kept until a compliance/legal retention policy for +pre-E2EE history is decided separately. A future GC pass following the same +idempotent, env-configurable pattern as the existing jobs would be the +natural way to enforce that retention once a window is chosen. diff --git a/apps/backend/drizzle/0001_gc_background_jobs.sql b/apps/backend/drizzle/0001_gc_background_jobs.sql new file mode 100644 index 00000000..b31268fb --- /dev/null +++ b/apps/backend/drizzle/0001_gc_background_jobs.sql @@ -0,0 +1,21 @@ +-- Background GC jobs (prekey/envelope/file/device cleanup) — schema support. +-- +-- Adds: +-- * mls_key_packages — one-time MLS KeyPackages per device, mirroring +-- device_prekeys' consumed-flag model so issuance stays auditable. +-- * devices.stale_flagged_at — informational marker set by the device-GC +-- job once a revoked device ages past retention. Never deletes the row. +CREATE TABLE "mls_key_packages" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "device_id" uuid NOT NULL, + "key_package" text NOT NULL, + "consumed" boolean DEFAULT false NOT NULL, + "consumed_at" timestamp, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "mls_key_packages" ADD CONSTRAINT "mls_key_packages_device_id_devices_id_fk" FOREIGN KEY ("device_id") REFERENCES "public"."devices"("id") ON DELETE cascade ON UPDATE no action; +--> statement-breakpoint +CREATE INDEX "mls_key_packages_device_available_idx" ON "mls_key_packages" USING btree ("device_id") WHERE "mls_key_packages"."consumed" = false; +--> statement-breakpoint +ALTER TABLE "devices" ADD COLUMN IF NOT EXISTS "stale_flagged_at" timestamp; diff --git a/apps/backend/drizzle/0002_device_capabilities.sql b/apps/backend/drizzle/0002_device_capabilities.sql new file mode 100644 index 00000000..80f0b6ce --- /dev/null +++ b/apps/backend/drizzle/0002_device_capabilities.sql @@ -0,0 +1,8 @@ +-- Device capability/version negotiation (#180-follow-on). +-- +-- Advertises supported protocols/ciphersuites/file-transfer versions per +-- device so senders can pick an encryption path both sides support. Rows +-- written before this migration default to the sealed_box-only baseline — +-- the protocol every device in this codebase already implements — so +-- existing devices negotiate correctly without any backfill. +ALTER TABLE "devices" ADD COLUMN IF NOT EXISTS "capabilities" jsonb DEFAULT '{"protocols":["sealed_box"],"ciphersuites":[],"fileTransfer":[]}'::jsonb NOT NULL; diff --git a/apps/backend/drizzle/0003_ciphertext_only_messages.sql b/apps/backend/drizzle/0003_ciphertext_only_messages.sql new file mode 100644 index 00000000..80b27ca0 --- /dev/null +++ b/apps/backend/drizzle/0003_ciphertext_only_messages.sql @@ -0,0 +1,102 @@ +-- One-time migration: drop plaintext `messages.content`, reset to the +-- ciphertext-only model. +-- +-- Policy (documented in full at docs/message-encryption-migration.md): +-- ARCHIVE THEN PURGE. Any existing plaintext row is copied into +-- `message_content_archive` — a table with no read path through the app's +-- API — before the column is dropped from `messages`. This keeps `messages` +-- fully ciphertext-shaped going forward (so `serializeMessage()` never has a +-- plaintext branch to accidentally serve) while preserving a compliance/ +-- audit copy of pre-E2EE history on its own retention schedule, instead of +-- silently destroying it or leaving a tombstone column on `messages` forever. +-- +-- Safe to run on a DB that never had a `content` column (this repo's own +-- migration history: 0000 already created `messages` in its ciphertext +-- shape) — the archive step is a no-op guarded by an information_schema +-- check, and every DDL statement below uses IF EXISTS/IF NOT EXISTS so +-- nothing errors either way. See docs/message-encryption-migration.md for +-- the full rollback plan and its limitations. + +CREATE TABLE IF NOT EXISTS "message_content_archive" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + -- Intentionally not a foreign key: this archive must outlive the + -- `messages` row it was copied from (e.g. hard-deletion by the message + -- GC path must not cascade-delete the compliance copy). + "original_message_id" uuid NOT NULL, + "conversation_id" uuid, + "sender_id" uuid, + "content" text NOT NULL, + "original_created_at" timestamp, + "archived_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "message_content_archive_original_message_idx" ON "message_content_archive" USING btree ("original_message_id"); +--> statement-breakpoint + +-- Archive any existing plaintext before the column is dropped. Dynamic SQL +-- is required here (unlike the DROP statements below) because a plain +-- top-level statement referencing `messages.content` would fail to parse on +-- a database where that column never existed. +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'messages' AND column_name = 'content' + ) THEN + EXECUTE ' + INSERT INTO message_content_archive + (original_message_id, conversation_id, sender_id, content, original_created_at) + SELECT id, conversation_id, sender_id, content, created_at + FROM messages + WHERE content IS NOT NULL + '; + END IF; +END $$; +--> statement-breakpoint + +-- Drop the plaintext column's GIN index. The exact index name from +-- pre-squash history is not recoverable, so every plausible historical name +-- is dropped defensively — DROP INDEX IF EXISTS is a no-op for names that +-- don't exist, so this is safe regardless of which (if any) actually match. +DROP INDEX IF EXISTS "messages_content_gin_idx"; +--> statement-breakpoint +DROP INDEX IF EXISTS "messages_content_search_idx"; +--> statement-breakpoint +DROP INDEX IF EXISTS "messages_content_tsv_idx"; +--> statement-breakpoint +DROP INDEX IF EXISTS "messages_content_idx"; +--> statement-breakpoint + +ALTER TABLE "messages" DROP COLUMN IF EXISTS "content"; +--> statement-breakpoint + +-- Defensively ensure the ciphertext-model columns/tables this migration's +-- companion schema overhaul introduced are present, for any database whose +-- migration history predates the 0000 squash and therefore skipped them. +-- A no-op everywhere this repo's own 0000 migration already ran. +ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "ciphertext" text; +--> statement-breakpoint +ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "sender_device_id" uuid; +--> statement-breakpoint +ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "file_id" uuid; +--> statement-breakpoint +ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "edits_message_id" uuid; +--> statement-breakpoint +ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "deleted_at" timestamp; +--> statement-breakpoint + +-- Note: this fallback intentionally omits FK constraints — on every database +-- that already ran this repo's 0000 migration (the only realistic case), +-- the table already exists with its constraints and this is a pure no-op. +-- A database old enough to hit this branch predates the squash entirely and +-- needs a manually-reviewed reconciliation, not a silent constraint add. +CREATE TABLE IF NOT EXISTS "message_envelopes" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "message_id" uuid NOT NULL, + "recipient_device_id" uuid NOT NULL, + "recipient_user_id" uuid NOT NULL, + "ciphertext" text NOT NULL, + "delivered_at" timestamp, + "read_at" timestamp, + "created_at" timestamp DEFAULT now() NOT NULL +); diff --git a/apps/backend/drizzle/rollback/0003_ciphertext_only_messages.down.sql b/apps/backend/drizzle/rollback/0003_ciphertext_only_messages.down.sql new file mode 100644 index 00000000..239dd42e --- /dev/null +++ b/apps/backend/drizzle/rollback/0003_ciphertext_only_messages.down.sql @@ -0,0 +1,35 @@ +-- Rollback for 0003_ciphertext_only_messages.sql. +-- +-- drizzle-kit has no built-in "down" migration runner — this script is +-- invoked manually (e.g. `psql "$DATABASE_URL" -f drizzle/rollback/0003_ciphertext_only_messages.down.sql`) +-- and is NOT part of the drizzle migration journal, so it is never applied +-- automatically by `db:migrate`. +-- +-- LIMITATIONS (see docs/message-encryption-migration.md for full detail): +-- * Only rows present in `message_content_archive` get their plaintext +-- restored. Any message sent after the forward migration ran was only +-- ever stored as ciphertext — there is no plaintext to bring back for it. +-- * The recreated GIN index is a reasonable equivalent (full-text search +-- over `content`), not necessarily byte-identical to whatever indexdef +-- existed pre-squash — that definition was not recoverable (see the +-- forward migration's comment on why several index names are dropped +-- defensively rather than one exact name). +-- * This script does NOT drop the ciphertext/envelope/device-capability/ +-- GC columns and tables added by this migration set. Rolling those back +-- would remove the entire E2EE data model, not just this one migration's +-- change — treat that as a separate, deliberate decision, not a side +-- effect of undoing the content-column drop. +-- * `message_content_archive` is left in place after restoring so no data +-- is destroyed by running this script; drop it manually once satisfied. + +ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "content" text; +--> statement-breakpoint + +CREATE INDEX IF NOT EXISTS "messages_content_gin_idx" ON "messages" USING gin (to_tsvector('english', "content")); +--> statement-breakpoint + +UPDATE "messages" m +SET "content" = a."content" +FROM "message_content_archive" a +WHERE a."original_message_id" = m."id" + AND m."content" IS NULL; diff --git a/apps/backend/src/__tests__/devices.test.ts b/apps/backend/src/__tests__/devices.test.ts index 75707dd1..ccf954c3 100644 --- a/apps/backend/src/__tests__/devices.test.ts +++ b/apps/backend/src/__tests__/devices.test.ts @@ -182,6 +182,7 @@ describe('GET /devices', () => { 'platform', 'lastSeenAt', 'revokedAt', + 'capabilities', 'oneTimePreKeysRemaining', ].sort(), ); diff --git a/apps/backend/src/db/schema.ts b/apps/backend/src/db/schema.ts index aa0e0236..759ea8d4 100644 --- a/apps/backend/src/db/schema.ts +++ b/apps/backend/src/db/schema.ts @@ -10,9 +10,11 @@ import { jsonb, uniqueIndex, check, + jsonb, type AnyPgColumn, } from 'drizzle-orm/pg-core'; import { relations, sql } from 'drizzle-orm'; +import { DEFAULT_CAPABILITIES, type DeviceCapabilities } from '../lib/capabilities.js'; export const users = pgTable('users', { id: uuid('id').primaryKey().defaultRandom(), @@ -214,6 +216,18 @@ export const devices = pgTable( lastSeenAt: timestamp('last_seen_at'), pushEnabled: boolean('push_enabled').notNull().default(true), revokedAt: timestamp('revoked_at'), + // Set by the device-GC job once a revoked device has aged past + // DEVICE_STALE_AFTER_DAYS. Purely informational — flags the row for + // admin visibility/future hard-delete without destroying audit history. + staleFlaggedAt: timestamp('stale_flagged_at'), + // Supported protocols/ciphersuites/file-transfer versions this device + // advertises (lib/capabilities.ts). Defaults to the sealed_box-only + // baseline so rows written before this column existed, or clients that + // never send it, negotiate correctly rather than erroring — see + // lib/capabilities.ts `normalizeCapabilities`. + capabilities: jsonb('capabilities').$type().notNull().default( + DEFAULT_CAPABILITIES, + ), createdAt: timestamp('created_at').notNull().defaultNow(), updatedAt: timestamp('updated_at').notNull().defaultNow(), }, @@ -569,6 +583,7 @@ export const tokenTransfersRelations = relations(tokenTransfers, ({ one }) => ({ export const devicesRelations = relations(devices, ({ one, many }) => ({ user: one(users, { fields: [devices.userId], references: [users.id] }), prekeys: many(devicePrekeys), + mlsKeyPackages: many(mlsKeyPackages), messages: many(messages), pushSubscriptions: many(pushSubscriptions), keyHistory: many(deviceKeyHistory), @@ -583,6 +598,10 @@ export const devicePrekeysRelations = relations(devicePrekeys, ({ one }) => ({ device: one(devices, { fields: [devicePrekeys.deviceId], references: [devices.id] }), })); +export const mlsKeyPackagesRelations = relations(mlsKeyPackages, ({ one }) => ({ + device: one(devices, { fields: [mlsKeyPackages.deviceId], references: [devices.id] }), +})); + export const pushSubscriptionsRelations = relations(pushSubscriptions, ({ one }) => ({ device: one(devices, { fields: [pushSubscriptions.deviceId], references: [devices.id] }), })); @@ -624,3 +643,5 @@ export type Device = typeof devices.$inferSelect; export type NewDevice = typeof devices.$inferInsert; export type DevicePrekey = typeof devicePrekeys.$inferSelect; export type NewDevicePrekey = typeof devicePrekeys.$inferInsert; +export type MlsKeyPackage = typeof mlsKeyPackages.$inferSelect; +export type NewMlsKeyPackage = typeof mlsKeyPackages.$inferInsert; diff --git a/apps/backend/src/index.ts b/apps/backend/src/index.ts index 1a856690..296d533c 100644 --- a/apps/backend/src/index.ts +++ b/apps/backend/src/index.ts @@ -45,6 +45,8 @@ import { runForever as runStellarListener, } from './services/stellarListener.js'; import { startFileCleanupJob } from './services/fileCleanup.js'; +import { startDeviceGcJob } from './services/deviceGc.js'; +import { startEnvelopeGcJob } from './services/envelopeGc.js'; import { loadEnv } from './config.js'; import { getObjectStore } from './lib/objectStore.js'; import { presenceChurnTotal, connectedSockets } from './lib/metrics.js'; @@ -427,6 +429,13 @@ void attachRedisAdapter(); // #231 – start background file cleanup + push backoff re-enable job startFileCleanupJob(); +// Background GC: prune consumed/expired prekeys + MLS key packages, flag +// long-revoked devices, and delete fully-delivered/expired message envelopes. +// Retention windows are configurable via env (see services/deviceGc.ts and +// services/envelopeGc.ts) so operators can tune them per deployment. +startDeviceGcJob(); +startEnvelopeGcJob(); + // Subscribe to device_revoked:* channels so any gateway instance can // disconnect a revoked device's sockets within seconds, even when the // revocation was issued on a different node. diff --git a/apps/backend/src/lib/capabilities.ts b/apps/backend/src/lib/capabilities.ts new file mode 100644 index 00000000..5a7ea9be --- /dev/null +++ b/apps/backend/src/lib/capabilities.ts @@ -0,0 +1,115 @@ +/** + * Device capability/version negotiation (#180-follow-on). + * + * `devices.capabilities` is a small JSON document a device advertises at + * registration and can update later ("upgrade") without re-registering its + * identity key. It lets a sender pick an encryption path both the sender's + * and recipient's devices actually support, so new protocols (Signal + * double-ratchet, MLS group ratchet) can roll out gradually without + * breaking devices that only understand the original sealed-box/X3DH + * envelope scheme already in production. + * + * Shape: + * protocols — messaging encryption protocols this device can decrypt, + * e.g. ["sealed_box", "signal", "mls"]. "sealed_box" names + * the X3DH + per-device-envelope scheme every device in + * this codebase already implements (devices/messages + * routes) — it is the universal fallback. + * ciphersuites — MLS/Signal ciphersuite identifiers this device supports. + * Only meaningful when "mls" (or a future ciphersuite- + * parameterized protocol) is present in `protocols`. + * fileTransfer — file-encryption scheme versions this device supports, + * e.g. ["file-v1"]. Independent of the messaging protocol. + * + * All fields are optional and additional/unrecognized values are preserved + * but ignored by `selectProtocol` — this is what makes negotiation forward- + * and backward-compatible: an older server ignores protocol names it + * doesn't recognize instead of failing, and a device that never advertised + * capabilities at all (pre-dates this feature) is treated as sealed_box-only. + */ +import { z } from 'zod'; + +export const KNOWN_PROTOCOLS = ['sealed_box', 'signal', 'mls'] as const; +export type KnownProtocol = (typeof KNOWN_PROTOCOLS)[number]; + +/** Every device implicitly supports this — it predates the capabilities column. */ +export const BASELINE_PROTOCOL: KnownProtocol = 'sealed_box'; + +/** + * Preference order used when both sides advertise more than one mutually + * supported protocol — prefer the strongest/newest scheme available to both. + */ +const PROTOCOL_PRIORITY: readonly KnownProtocol[] = ['mls', 'signal', 'sealed_box']; + +export const DeviceCapabilitiesSchema = z + .object({ + protocols: z.array(z.string()).default([BASELINE_PROTOCOL]), + ciphersuites: z.array(z.string()).default([]), + fileTransfer: z.array(z.string()).default([]), + }) + .partial() + .default({}); + +export type DeviceCapabilities = z.infer; + +/** The capability set assumed for a device that has never advertised one. */ +export const DEFAULT_CAPABILITIES: DeviceCapabilities = { + protocols: [BASELINE_PROTOCOL], + ciphersuites: [], + fileTransfer: [], +}; + +/** + * Normalize a possibly-missing/possibly-malformed capabilities value into a + * concrete `DeviceCapabilities`. Unknown/older clients — including rows from + * before this column existed (`null`/`undefined`) — fall back to the + * sealed_box-only baseline rather than erroring, satisfying "unknown/older + * capabilities handled gracefully". + */ +export function normalizeCapabilities(raw: unknown): DeviceCapabilities { + const parsed = DeviceCapabilitiesSchema.safeParse(raw ?? {}); + if (!parsed.success) return { ...DEFAULT_CAPABILITIES }; + + const protocols = parsed.data.protocols?.length ? parsed.data.protocols : [BASELINE_PROTOCOL]; + return { + protocols, + ciphersuites: parsed.data.ciphersuites ?? [], + fileTransfer: parsed.data.fileTransfer ?? [], + }; +} + +/** + * Pick the best protocol both sides can speak. Falls back to the universal + * `sealed_box` baseline when there is no other overlap — every registered + * device supports it, so this never returns `null`. + */ +export function selectProtocol( + a: unknown, + b: unknown, +): { protocol: KnownProtocol; ciphersuite: string | null } { + const capsA = normalizeCapabilities(a); + const capsB = normalizeCapabilities(b); + const protocolsA = new Set(capsA.protocols); + const protocolsB = new Set(capsB.protocols); + + for (const candidate of PROTOCOL_PRIORITY) { + if (protocolsA.has(candidate) && protocolsB.has(candidate)) { + const ciphersuite = + candidate === 'mls' ? selectCiphersuite(capsA.ciphersuites, capsB.ciphersuites) : null; + return { protocol: candidate, ciphersuite }; + } + } + + return { protocol: BASELINE_PROTOCOL, ciphersuite: null }; +} + +/** First ciphersuite identifier both sides advertise, preserving the caller's preference order. */ +function selectCiphersuite(a: string[] = [], b: string[] = []): string | null { + const setB = new Set(b); + return a.find((suite) => setB.has(suite)) ?? null; +} + +/** Whether `capabilities` includes support for the given file-transfer scheme version. */ +export function supportsFileTransfer(capabilities: unknown, version: string): boolean { + return normalizeCapabilities(capabilities).fileTransfer?.includes(version) ?? false; +} diff --git a/apps/backend/src/routes/auth.ts b/apps/backend/src/routes/auth.ts index 339316ce..8a4c9628 100644 --- a/apps/backend/src/routes/auth.ts +++ b/apps/backend/src/routes/auth.ts @@ -8,6 +8,7 @@ import { users, wallets, devices } from '../db/schema.js'; import { eq, and } from 'drizzle-orm'; import { createNonce, consumeNonce } from '../lib/nonce.js'; import { signToken } from '../lib/jwt.js'; +import { normalizeCapabilities } from '../lib/capabilities.js'; import { validate } from '../middleware/validate.js'; import { recordAuditEvent, requestContext } from '../services/auditLog.js'; import { ipIdentifier, rateLimit } from '../middleware/rateLimit.js'; @@ -56,6 +57,7 @@ authRouter.post( const deviceName = device?.deviceName; const platform = device?.platform; const registrationId = device?.registrationId; + const capabilities = device?.capabilities; // Every failed sign-in is audited (#376). The wallet address is the only // identity available before verification succeeds, and it is a public @@ -154,6 +156,9 @@ authRouter.post( ...(deviceName ? { deviceName } : {}), ...(platform ? { platform } : {}), ...(registrationId !== undefined ? { registrationId } : {}), + // A client re-verifying with a newer `capabilities` set is the + // "upgrade" path (#180-follow-on) — no re-registration needed. + ...(capabilities !== undefined ? { capabilities: normalizeCapabilities(capabilities) } : {}), }) .where(eq(devices.id, deviceId)); } else { @@ -166,6 +171,7 @@ authRouter.post( platform: platform ?? null, registrationId: registrationId ?? null, lastSeenAt: new Date(), + ...(capabilities !== undefined ? { capabilities: normalizeCapabilities(capabilities) } : {}), }) .returning({ id: devices.id }); if (!newDevice) { diff --git a/apps/backend/src/routes/conversations.ts b/apps/backend/src/routes/conversations.ts index 6deb02d0..13f775b6 100644 --- a/apps/backend/src/routes/conversations.ts +++ b/apps/backend/src/routes/conversations.ts @@ -24,6 +24,16 @@ export const conversationsRouter: IRouter = Router(); conversationsRouter.use(requireAuth); +// Post-schema-overhaul audit (see PR description): every relation name below +// (`members`, `user`, `wallets`, `messages`, `sender`, `envelopes`) was +// checked against the current `relations()` declarations in db/schema.ts and +// every selected column against the current table definitions — none +// reference dropped columns/relations. The `as never` casts at the call +// sites below exist only because TS can't correlate this function's return +// type with drizzle's recursive `with:` generic when it's built dynamically +// (a known drizzle limitation, not a sign the shape is unverified); the +// `ConversationPayload`/`ConversationMemberPayload` types the results are +// cast to afterward are what's actually checked against the query shape. const getConversationRelations = (deviceId: string) => ({ members: { with: { @@ -1081,9 +1091,18 @@ conversationsRouter.get('/:id/devices', async (req: AuthRequest, res) => { identityPublicKey: true, deviceName: true, platform: true, + capabilities: true, }, }); + // Look up the caller's own device capabilities so each returned device can + // carry the protocol the sender should actually use with it (#180-follow- + // on) — sparing every client from re-implementing selectProtocol(). + const callerDevice = await db.query.devices.findFirst({ + where: eq(devices.id, req.auth!.deviceId), + columns: { capabilities: true }, + }); + res.json({ devices: deviceRows.map((d) => ({ id: d.id, @@ -1091,6 +1110,8 @@ conversationsRouter.get('/:id/devices', async (req: AuthRequest, res) => { identityPublicKey: d.identityPublicKey, deviceName: d.deviceName, platform: d.platform, + capabilities: normalizeCapabilities(d.capabilities), + negotiatedProtocol: selectProtocol(callerDevice?.capabilities, d.capabilities).protocol, })), }); }); diff --git a/apps/backend/src/routes/devices.ts b/apps/backend/src/routes/devices.ts index 2d41286d..1bf6278a 100644 --- a/apps/backend/src/routes/devices.ts +++ b/apps/backend/src/routes/devices.ts @@ -106,6 +106,7 @@ devicesRouter.get('/', async (req: AuthRequest, res) => { platform: device.platform, lastSeenAt: device.lastSeenAt, revokedAt: device.revokedAt, + capabilities: normalizeCapabilities(device.capabilities), oneTimePreKeysRemaining: remainingByDevice.get(device.id) ?? 0, createdAt: device.createdAt, current: device.id === currentDeviceId, diff --git a/apps/backend/src/routes/userDevices.ts b/apps/backend/src/routes/userDevices.ts index fa9b6c33..40b72fe2 100644 --- a/apps/backend/src/routes/userDevices.ts +++ b/apps/backend/src/routes/userDevices.ts @@ -11,6 +11,7 @@ import { and, eq, inArray, isNull } from 'drizzle-orm'; import { db } from '../db/index.js'; import { conversationMembers, devices } from '../db/schema.js'; import { requireAuth, type AuthRequest } from '../middleware/auth.js'; +import { normalizeCapabilities } from '../lib/capabilities.js'; export const userDevicesRouter: RouterType = Router(); @@ -27,6 +28,7 @@ userDevicesRouter.get('/:id/public-key', async (req: AuthRequest, res) => { id: true, userId: true, identityPublicKey: true, + capabilities: true, }, }); @@ -61,6 +63,7 @@ userDevicesRouter.get('/:id/public-key', async (req: AuthRequest, res) => { id: device.id, userId: device.userId, identityPublicKey: device.identityPublicKey, + capabilities: normalizeCapabilities(device.capabilities), }); } catch { res.status(500).json({ error: 'Failed to fetch device public key' }); diff --git a/apps/backend/src/routes/users.ts b/apps/backend/src/routes/users.ts index 1656657d..30c9fe32 100644 --- a/apps/backend/src/routes/users.ts +++ b/apps/backend/src/routes/users.ts @@ -354,6 +354,9 @@ usersRouter.get( deviceId: device.id, identityPublicKey: device.identityPublicKey, registrationId: device.registrationId, + // Lets the initiating sender pick an encryption path this recipient + // device supports before running X3DH (#180-follow-on). + capabilities: normalizeCapabilities(device.capabilities), signedPreKey: { keyId: signedPreKey.keyId, publicKey: signedPreKey.publicKey, diff --git a/apps/backend/src/schemas/auth.schemas.ts b/apps/backend/src/schemas/auth.schemas.ts index 915a958d..6c553233 100644 --- a/apps/backend/src/schemas/auth.schemas.ts +++ b/apps/backend/src/schemas/auth.schemas.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; import { IdentityPublicKeySchema } from '../lib/keys.js'; +import { DeviceCapabilitiesSchema } from '../lib/capabilities.js'; export const ChallengeSchema = z.object({ walletAddress: z.string().min(1, 'walletAddress is required'), @@ -13,6 +14,10 @@ export const DeviceSchema = z.object({ platform: z.enum(['web', 'ios', 'android']), identityPublicKey: IdentityPublicKeySchema, registrationId: z.number().int().nonnegative().optional(), + // Supported protocols/ciphersuites/file-transfer versions (#180-follow-on). + // Optional — omitting it defaults to the sealed_box-only baseline so older + // clients that predate this field keep working unchanged. + capabilities: DeviceCapabilitiesSchema.optional(), }); export const VerifySchema = z diff --git a/apps/backend/src/services/deviceGc.ts b/apps/backend/src/services/deviceGc.ts new file mode 100644 index 00000000..15066e6e --- /dev/null +++ b/apps/backend/src/services/deviceGc.ts @@ -0,0 +1,142 @@ +/** + * Background device/key GC service. + * + * Three independent, idempotent passes, all safe to retry after a crash + * because each only ever transitions rows forward (consumed -> pruned, + * unflagged -> flagged) and re-running a pass against already-pruned/flagged + * rows is a no-op: + * + * 1. Prune one-time prekeys (`device_prekeys`) that are either consumed or + * have aged past the "nobody claimed this" ceiling. + * 2. Prune MLS KeyPackages (`mls_key_packages`) under the same policy. + * 3. Flag devices that have been revoked longer than the stale window — + * flags only, never deletes, preserving revocation audit history. + * + * Retention windows are configurable via env so operators can tune them per + * deployment without a code change. + */ +import { and, eq, lt, or, isNull, isNotNull } from 'drizzle-orm'; +import { db } from '../db/index.js'; +import { devicePrekeys, mlsKeyPackages, devices } from '../db/schema.js'; + +const DAY_MS = 24 * 60 * 60 * 1_000; + +function envDays(name: string, defaultDays: number): number { + const raw = process.env[name]; + const parsed = raw !== undefined ? Number(raw) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : defaultDays; +} + +/** How long a consumed one-time prekey/MLS key package is kept for audit before deletion. */ +export function getConsumedKeyRetentionMs(): number { + return envDays('PREKEY_CONSUMED_RETENTION_DAYS', 30) * DAY_MS; +} + +/** How long an unconsumed one-time prekey/MLS key package may sit unclaimed before GC. */ +export function getUnconsumedKeyMaxAgeMs(): number { + return envDays('PREKEY_UNCONSUMED_MAX_AGE_DAYS', 90) * DAY_MS; +} + +/** How long a device must stay revoked before the GC job flags it as stale. */ +export function getDeviceStaleAfterMs(): number { + return envDays('DEVICE_STALE_AFTER_DAYS', 180) * DAY_MS; +} + +function getGcIntervalMs(): number { + const raw = process.env['DEVICE_GC_INTERVAL_MS']; + const parsed = raw !== undefined ? Number(raw) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : 60 * 60 * 1_000; // hourly +} + +/** + * Prune one-time prekeys that are either consumed-and-aged-out or have gone + * unconsumed past the max-age ceiling. Never touches signed prekeys — there + * is exactly one live signed prekey per device and it is replaced in place + * on upload, not GC'd. + */ +export async function runPrekeyGcPass(): Promise { + const consumedCutoff = new Date(Date.now() - getConsumedKeyRetentionMs()); + const unconsumedCutoff = new Date(Date.now() - getUnconsumedKeyMaxAgeMs()); + + const result = await db + .delete(devicePrekeys) + .where( + and( + eq(devicePrekeys.keyType, 'one_time'), + or( + and(eq(devicePrekeys.consumed, true), lt(devicePrekeys.createdAt, consumedCutoff)), + and(eq(devicePrekeys.consumed, false), lt(devicePrekeys.createdAt, unconsumedCutoff)), + ), + ), + ) + .returning({ id: devicePrekeys.id }); + + return result.length; +} + +/** Same retention policy as `runPrekeyGcPass`, applied to MLS KeyPackages. */ +export async function runMlsKeyPackageGcPass(): Promise { + const consumedCutoff = new Date(Date.now() - getConsumedKeyRetentionMs()); + const unconsumedCutoff = new Date(Date.now() - getUnconsumedKeyMaxAgeMs()); + + const result = await db + .delete(mlsKeyPackages) + .where( + or( + and(eq(mlsKeyPackages.consumed, true), lt(mlsKeyPackages.createdAt, consumedCutoff)), + and(eq(mlsKeyPackages.consumed, false), lt(mlsKeyPackages.createdAt, unconsumedCutoff)), + ), + ) + .returning({ id: mlsKeyPackages.id }); + + return result.length; +} + +/** + * Flag (never delete) devices that have been revoked longer than the stale + * window. Idempotent: only rows with `staleFlaggedAt IS NULL` are touched, so + * re-running this pass against already-flagged devices is a no-op. + */ +export async function runDeviceStaleFlagPass(): Promise { + const cutoff = new Date(Date.now() - getDeviceStaleAfterMs()); + + const result = await db + .update(devices) + .set({ staleFlaggedAt: new Date() }) + .where( + and(isNotNull(devices.revokedAt), lt(devices.revokedAt, cutoff), isNull(devices.staleFlaggedAt)), + ) + .returning({ id: devices.id }); + + return result.length; +} + +let gcTimer: ReturnType | null = null; + +export function startDeviceGcJob(): void { + if (gcTimer) return; + gcTimer = setInterval(() => { + void (async () => { + try { + const prekeys = await runPrekeyGcPass(); + const keyPackages = await runMlsKeyPackageGcPass(); + const flagged = await runDeviceStaleFlagPass(); + if (prekeys || keyPackages || flagged) { + console.log( + `[device-gc] pruned ${prekeys} prekey(s), ${keyPackages} MLS key package(s), flagged ${flagged} stale device(s)`, + ); + } + } catch (err) { + console.error('[device-gc] job error:', err); + } + })(); + }, getGcIntervalMs()); + gcTimer.unref(); +} + +export function stopDeviceGcJob(): void { + if (gcTimer) { + clearInterval(gcTimer); + gcTimer = null; + } +} diff --git a/apps/backend/src/services/envelopeGc.ts b/apps/backend/src/services/envelopeGc.ts new file mode 100644 index 00000000..533334ed --- /dev/null +++ b/apps/backend/src/services/envelopeGc.ts @@ -0,0 +1,78 @@ +/** + * Background message-envelope GC service. + * + * `message_envelopes` holds one row per (message, recipient device) — it is + * the actual delivery unit, and the only thing that keeps it around after + * delivery is auditability of delivered/read timestamps. Left unbounded it + * grows with every message * every recipient device, so this job deletes: + * + * 1. Envelopes that have been delivered and are older than the delivered- + * retention window (the common case: the recipient got it). + * 2. Envelopes past the max-age ceiling regardless of delivery state (a + * device that never comes back to collect its envelope should not pin + * storage forever). + * + * A plain DELETE ... WHERE is naturally idempotent and safe to retry: a + * crash mid-run just means the next tick deletes whatever is left. + */ +import { and, lt, or, isNotNull } from 'drizzle-orm'; +import { db } from '../db/index.js'; +import { messageEnvelopes } from '../db/schema.js'; + +function getEnvelopeDeliveredRetentionMs(): number { + const raw = process.env['ENVELOPE_DELIVERED_RETENTION_DAYS']; + const parsed = raw !== undefined ? Number(raw) : NaN; + const days = Number.isFinite(parsed) && parsed > 0 ? parsed : 7; + return days * 24 * 60 * 60 * 1_000; +} + +function getEnvelopeMaxAgeMs(): number { + const raw = process.env['ENVELOPE_MAX_AGE_DAYS']; + const parsed = raw !== undefined ? Number(raw) : NaN; + const days = Number.isFinite(parsed) && parsed > 0 ? parsed : 30; + return days * 24 * 60 * 60 * 1_000; +} + +function getGcIntervalMs(): number { + const raw = process.env['ENVELOPE_GC_INTERVAL_MS']; + const parsed = raw !== undefined ? Number(raw) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : 30 * 60 * 1_000; // every 30 minutes +} + +export async function runEnvelopeGcPass(): Promise { + const deliveredCutoff = new Date(Date.now() - getEnvelopeDeliveredRetentionMs()); + const maxAgeCutoff = new Date(Date.now() - getEnvelopeMaxAgeMs()); + + const result = await db + .delete(messageEnvelopes) + .where( + or( + and(isNotNull(messageEnvelopes.deliveredAt), lt(messageEnvelopes.deliveredAt, deliveredCutoff)), + lt(messageEnvelopes.createdAt, maxAgeCutoff), + ), + ) + .returning({ id: messageEnvelopes.id }); + + return result.length; +} + +let envelopeGcTimer: ReturnType | null = null; + +export function startEnvelopeGcJob(): void { + if (envelopeGcTimer) return; + envelopeGcTimer = setInterval(() => { + void runEnvelopeGcPass() + .then((count) => { + if (count) console.log(`[envelope-gc] pruned ${count} envelope(s)`); + }) + .catch((err) => console.error('[envelope-gc] job error:', err)); + }, getGcIntervalMs()); + envelopeGcTimer.unref(); +} + +export function stopEnvelopeGcJob(): void { + if (envelopeGcTimer) { + clearInterval(envelopeGcTimer); + envelopeGcTimer = null; + } +} diff --git a/apps/backend/src/services/fileCleanup.ts b/apps/backend/src/services/fileCleanup.ts index d0284ea8..97049a23 100644 --- a/apps/backend/src/services/fileCleanup.ts +++ b/apps/backend/src/services/fileCleanup.ts @@ -3,10 +3,14 @@ * * Implements #231 – soft-delete (files.deletedAt) is set immediately when a * message is retracted. This job hard-deletes the S3 object once every - * referencing message is also soft-deleted (ref-counting across envelopes). + * referencing message is also soft-deleted (ref-counting across envelopes) + * and, if configured, a grace period has elapsed since the soft-delete. * * The job is idempotent: it sets hardDeletedAt only after a successful S3 * delete, so a crash between steps is safe to retry. + * + * Retention windows are configurable via env so operators can tune them per + * deployment without a code change. */ import { isNotNull, isNull, sql, and, eq, lt } from 'drizzle-orm'; import { db } from '../db/index.js'; @@ -14,7 +18,25 @@ import { files } from '../db/schema.js'; import { getObjectStore } from '../lib/objectStore.js'; import { reenableExpiredBackoffs } from './pushNotification.js'; -const CLEANUP_INTERVAL_MS = 5 * 60 * 1_000; // every 5 minutes +function getCleanupIntervalMs(): number { + const raw = process.env['FILE_GC_INTERVAL_MS']; + const parsed = raw !== undefined ? Number(raw) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : 5 * 60 * 1_000; // every 5 minutes +} + +/** Grace period after soft-delete before a fully-unreferenced file is hard-deleted. */ +function getHardDeleteGraceMs(): number { + const raw = process.env['FILE_HARD_DELETE_GRACE_MS']; + const parsed = raw !== undefined ? Number(raw) : NaN; + return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0; +} + +/** How long an unconfirmed (`pending`) upload may sit before its slot is reclaimed. */ +function getPendingUploadTtlMs(): number { + const raw = process.env['PENDING_UPLOAD_TTL_MS']; + const parsed = raw !== undefined ? Number(raw) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : 24 * 60 * 60 * 1_000; // 24 hours +} /** * Soft-delete a file record when its owning message is retracted. @@ -37,8 +59,9 @@ export async function softDeleteFile(fileId: string): Promise { * remaining live message references. Idempotent and safe to retry. */ export async function runHardDeletePass(): Promise { + const graceCutoff = new Date(Date.now() - getHardDeleteGraceMs()); const candidates = await db.query.files.findMany({ - where: (f) => isNotNull(f.deletedAt) && isNull(f.hardDeletedAt), + where: (f) => and(isNotNull(f.deletedAt), isNull(f.hardDeletedAt), lt(f.deletedAt, graceCutoff)), columns: { id: true, storageKey: true }, }); @@ -65,8 +88,8 @@ export async function runHardDeletePass(): Promise { } } - // Garbage-collect unconfirmed pending files older than 24 hours - const stalePendingDate = new Date(Date.now() - 24 * 60 * 60 * 1000); + // Garbage-collect unconfirmed pending files past the configured TTL. + const stalePendingDate = new Date(Date.now() - getPendingUploadTtlMs()); const pendingCandidates = await db.query.files.findMany({ where: (f) => and(eq(f.status, 'pending'), lt(f.createdAt, stalePendingDate)), columns: { id: true, storageKey: true }, @@ -94,7 +117,7 @@ export function startFileCleanupJob(): void { } catch (err) { console.error('[file-cleanup] job error:', err); } - }, CLEANUP_INTERVAL_MS); + }, getCleanupIntervalMs()); cleanupTimer.unref(); }