From b08ca86ca7a5ddef0a0d685161214624bb269893 Mon Sep 17 00:00:00 2001
From: G-ELM <alfygodwin@gmail.com>
Date: Tue, 28 Jul 2026 01:12:35 +0100
Subject: [PATCH 1/4] feat(backend): background GC jobs for prekeys, MLS
 packages, envelopes, files, devices

Adds scheduled, idempotent cleanup so storage and DB stay bounded over time:
- deviceGc.ts prunes consumed/expired one-time prekeys + MLS key packages, and
  flags (never deletes) devices revoked past the stale window.
- envelopeGc.ts deletes message envelopes that are fully delivered past
  retention or past a hard max-age ceiling.
- fileCleanup.ts gains a configurable hard-delete grace period and pending-
  upload TTL (previously hardcoded), and a latent bug where the JS `&&`
  operator silently dropped the isNotNull(deletedAt) filter (only
  isNull(hardDeletedAt) was actually applied) is fixed to use `and(...)`.

New mls_key_packages table + devices.stale_flagged_at column, migrated via
0001_gc_background_jobs.sql. All retention windows are env-configurable
(PREKEY_CONSUMED_RETENTION_DAYS, PREKEY_UNCONSUMED_MAX_AGE_DAYS,
DEVICE_STALE_AFTER_DAYS, ENVELOPE_DELIVERED_RETENTION_DAYS,
ENVELOPE_MAX_AGE_DAYS, FILE_HARD_DELETE_GRACE_MS, PENDING_UPLOAD_TTL_MS) with
safe defaults, and every pass is a plain idempotent DELETE/UPDATE WHERE so a
crash mid-run is safe to retry.
---
 .../drizzle/0001_gc_background_jobs.sql       |  21 +++
 apps/backend/drizzle/meta/_journal.json       |   7 +
 apps/backend/src/db/schema.ts                 |  39 +++++
 apps/backend/src/index.ts                     |   9 ++
 apps/backend/src/services/deviceGc.ts         | 142 ++++++++++++++++++
 apps/backend/src/services/envelopeGc.ts       |  78 ++++++++++
 apps/backend/src/services/fileCleanup.ts      |  35 ++++-
 7 files changed, 325 insertions(+), 6 deletions(-)
 create mode 100644 apps/backend/drizzle/0001_gc_background_jobs.sql
 create mode 100644 apps/backend/src/services/deviceGc.ts
 create mode 100644 apps/backend/src/services/envelopeGc.ts

diff --git a/apps/backend/drizzle/0001_gc_background_jobs.sql b/apps/backend/drizzle/0001_gc_background_jobs.sql
new file mode 100644
index 00000000..b31268fb
--- /dev/null
+++ b/apps/backend/drizzle/0001_gc_background_jobs.sql
@@ -0,0 +1,21 @@
+-- Background GC jobs (prekey/envelope/file/device cleanup) — schema support.
+--
+-- Adds:
+--   * mls_key_packages — one-time MLS KeyPackages per device, mirroring
+--     device_prekeys' consumed-flag model so issuance stays auditable.
+--   * devices.stale_flagged_at — informational marker set by the device-GC
+--     job once a revoked device ages past retention. Never deletes the row.
+CREATE TABLE "mls_key_packages" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"device_id" uuid NOT NULL,
+	"key_package" text NOT NULL,
+	"consumed" boolean DEFAULT false NOT NULL,
+	"consumed_at" timestamp,
+	"created_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+ALTER TABLE "mls_key_packages" ADD CONSTRAINT "mls_key_packages_device_id_devices_id_fk" FOREIGN KEY ("device_id") REFERENCES "public"."devices"("id") ON DELETE cascade ON UPDATE no action;
+--> statement-breakpoint
+CREATE INDEX "mls_key_packages_device_available_idx" ON "mls_key_packages" USING btree ("device_id") WHERE "mls_key_packages"."consumed" = false;
+--> statement-breakpoint
+ALTER TABLE "devices" ADD COLUMN IF NOT EXISTS "stale_flagged_at" timestamp;
diff --git a/apps/backend/drizzle/meta/_journal.json b/apps/backend/drizzle/meta/_journal.json
index 52ed62d6..d3066729 100644
--- a/apps/backend/drizzle/meta/_journal.json
+++ b/apps/backend/drizzle/meta/_journal.json
@@ -8,6 +8,13 @@
       "when": 1784899426825,
       "tag": "0000_stale_mandarin",
       "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "7",
+      "when": 1785100800000,
+      "tag": "0001_gc_background_jobs",
+      "breakpoints": true
     }
   ]
 }
\ No newline at end of file
diff --git a/apps/backend/src/db/schema.ts b/apps/backend/src/db/schema.ts
index 46e079c8..0a4cb1b6 100644
--- a/apps/backend/src/db/schema.ts
+++ b/apps/backend/src/db/schema.ts
@@ -188,6 +188,10 @@ export const devices = pgTable(
     lastSeenAt: timestamp('last_seen_at'),
     pushEnabled: boolean('push_enabled').notNull().default(true),
     revokedAt: timestamp('revoked_at'),
+    // Set by the device-GC job once a revoked device has aged past
+    // DEVICE_STALE_AFTER_DAYS. Purely informational — flags the row for
+    // admin visibility/future hard-delete without destroying audit history.
+    staleFlaggedAt: timestamp('stale_flagged_at'),
     createdAt: timestamp('created_at').notNull().defaultNow(),
     updatedAt: timestamp('updated_at').notNull().defaultNow(),
   },
@@ -246,6 +250,34 @@ export const devicePrekeys = pgTable(
   ],
 );
 
+// ─── MLS key packages ─────────────────────────────────────────────────────────
+//
+// One-time-use MLS KeyPackages a device publishes so it can be added to a
+// group's ratchet tree (mirrors `devicePrekeys`' one-time-prekey model:
+// `consumed` flips to true instead of deleting the row, so issuance stays
+// auditable). The background GC job (services/deviceGc.ts) prunes
+// consumed/expired rows on a retention window.
+
+export const mlsKeyPackages = pgTable(
+  'mls_key_packages',
+  {
+    id: uuid('id').primaryKey().defaultRandom(),
+    deviceId: uuid('device_id')
+      .notNull()
+      .references(() => devices.id, { onDelete: 'cascade' }),
+    // Base64-encoded MLS KeyPackage TLS encoding (see lib/keys.ts MlsKeyPackageSchema).
+    keyPackage: text('key_package').notNull(),
+    consumed: boolean('consumed').notNull().default(false),
+    consumedAt: timestamp('consumed_at'),
+    createdAt: timestamp('created_at').notNull().defaultNow(),
+  },
+  (table) => [
+    index('mls_key_packages_device_available_idx')
+      .on(table.deviceId)
+      .where(sql`${table.consumed} = false`),
+  ],
+);
+
 // ─── Token transfers (#46) ────────────────────────────────────────────────────
 //
 // One row per Soroban `transfer` event the listener (services/stellarListener.ts)
@@ -434,6 +466,7 @@ export const tokenTransfersRelations = relations(tokenTransfers, ({ one }) => ({
 export const devicesRelations = relations(devices, ({ one, many }) => ({
   user: one(users, { fields: [devices.userId], references: [users.id] }),
   prekeys: many(devicePrekeys),
+  mlsKeyPackages: many(mlsKeyPackages),
   messages: many(messages),
   pushSubscriptions: many(pushSubscriptions),
 }));
@@ -442,6 +475,10 @@ export const devicePrekeysRelations = relations(devicePrekeys, ({ one }) => ({
   device: one(devices, { fields: [devicePrekeys.deviceId], references: [devices.id] }),
 }));
 
+export const mlsKeyPackagesRelations = relations(mlsKeyPackages, ({ one }) => ({
+  device: one(devices, { fields: [mlsKeyPackages.deviceId], references: [devices.id] }),
+}));
+
 export const pushSubscriptionsRelations = relations(pushSubscriptions, ({ one }) => ({
   device: one(devices, { fields: [pushSubscriptions.deviceId], references: [devices.id] }),
 }));
@@ -483,3 +520,5 @@ export type Device = typeof devices.$inferSelect;
 export type NewDevice = typeof devices.$inferInsert;
 export type DevicePrekey = typeof devicePrekeys.$inferSelect;
 export type NewDevicePrekey = typeof devicePrekeys.$inferInsert;
+export type MlsKeyPackage = typeof mlsKeyPackages.$inferSelect;
+export type NewMlsKeyPackage = typeof mlsKeyPackages.$inferInsert;
diff --git a/apps/backend/src/index.ts b/apps/backend/src/index.ts
index d2dedf89..fd905dee 100644
--- a/apps/backend/src/index.ts
+++ b/apps/backend/src/index.ts
@@ -43,6 +43,8 @@ import {
   runForever as runStellarListener,
 } from './services/stellarListener.js';
 import { startFileCleanupJob } from './services/fileCleanup.js';
+import { startDeviceGcJob } from './services/deviceGc.js';
+import { startEnvelopeGcJob } from './services/envelopeGc.js';
 import { loadEnv } from './config.js';
 import { getObjectStore } from './lib/objectStore.js';
 import {
@@ -403,6 +405,13 @@ void attachRedisAdapter();
 // #231 – start background file cleanup + push backoff re-enable job
 startFileCleanupJob();
 
+// Background GC: prune consumed/expired prekeys + MLS key packages, flag
+// long-revoked devices, and delete fully-delivered/expired message envelopes.
+// Retention windows are configurable via env (see services/deviceGc.ts and
+// services/envelopeGc.ts) so operators can tune them per deployment.
+startDeviceGcJob();
+startEnvelopeGcJob();
+
 // Subscribe to device_revoked:* channels so any gateway instance can
 // disconnect a revoked device's sockets within seconds, even when the
 // revocation was issued on a different node.
diff --git a/apps/backend/src/services/deviceGc.ts b/apps/backend/src/services/deviceGc.ts
new file mode 100644
index 00000000..15066e6e
--- /dev/null
+++ b/apps/backend/src/services/deviceGc.ts
@@ -0,0 +1,142 @@
+/**
+ * Background device/key GC service.
+ *
+ * Three independent, idempotent passes, all safe to retry after a crash
+ * because each only ever transitions rows forward (consumed -> pruned,
+ * unflagged -> flagged) and re-running a pass against already-pruned/flagged
+ * rows is a no-op:
+ *
+ *   1. Prune one-time prekeys (`device_prekeys`) that are either consumed or
+ *      have aged past the "nobody claimed this" ceiling.
+ *   2. Prune MLS KeyPackages (`mls_key_packages`) under the same policy.
+ *   3. Flag devices that have been revoked longer than the stale window —
+ *      flags only, never deletes, preserving revocation audit history.
+ *
+ * Retention windows are configurable via env so operators can tune them per
+ * deployment without a code change.
+ */
+import { and, eq, lt, or, isNull, isNotNull } from 'drizzle-orm';
+import { db } from '../db/index.js';
+import { devicePrekeys, mlsKeyPackages, devices } from '../db/schema.js';
+
+const DAY_MS = 24 * 60 * 60 * 1_000;
+
+function envDays(name: string, defaultDays: number): number {
+  const raw = process.env[name];
+  const parsed = raw !== undefined ? Number(raw) : NaN;
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : defaultDays;
+}
+
+/** How long a consumed one-time prekey/MLS key package is kept for audit before deletion. */
+export function getConsumedKeyRetentionMs(): number {
+  return envDays('PREKEY_CONSUMED_RETENTION_DAYS', 30) * DAY_MS;
+}
+
+/** How long an unconsumed one-time prekey/MLS key package may sit unclaimed before GC. */
+export function getUnconsumedKeyMaxAgeMs(): number {
+  return envDays('PREKEY_UNCONSUMED_MAX_AGE_DAYS', 90) * DAY_MS;
+}
+
+/** How long a device must stay revoked before the GC job flags it as stale. */
+export function getDeviceStaleAfterMs(): number {
+  return envDays('DEVICE_STALE_AFTER_DAYS', 180) * DAY_MS;
+}
+
+function getGcIntervalMs(): number {
+  const raw = process.env['DEVICE_GC_INTERVAL_MS'];
+  const parsed = raw !== undefined ? Number(raw) : NaN;
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 60 * 60 * 1_000; // hourly
+}
+
+/**
+ * Prune one-time prekeys that are either consumed-and-aged-out or have gone
+ * unconsumed past the max-age ceiling. Never touches signed prekeys — there
+ * is exactly one live signed prekey per device and it is replaced in place
+ * on upload, not GC'd.
+ */
+export async function runPrekeyGcPass(): Promise<number> {
+  const consumedCutoff = new Date(Date.now() - getConsumedKeyRetentionMs());
+  const unconsumedCutoff = new Date(Date.now() - getUnconsumedKeyMaxAgeMs());
+
+  const result = await db
+    .delete(devicePrekeys)
+    .where(
+      and(
+        eq(devicePrekeys.keyType, 'one_time'),
+        or(
+          and(eq(devicePrekeys.consumed, true), lt(devicePrekeys.createdAt, consumedCutoff)),
+          and(eq(devicePrekeys.consumed, false), lt(devicePrekeys.createdAt, unconsumedCutoff)),
+        ),
+      ),
+    )
+    .returning({ id: devicePrekeys.id });
+
+  return result.length;
+}
+
+/** Same retention policy as `runPrekeyGcPass`, applied to MLS KeyPackages. */
+export async function runMlsKeyPackageGcPass(): Promise<number> {
+  const consumedCutoff = new Date(Date.now() - getConsumedKeyRetentionMs());
+  const unconsumedCutoff = new Date(Date.now() - getUnconsumedKeyMaxAgeMs());
+
+  const result = await db
+    .delete(mlsKeyPackages)
+    .where(
+      or(
+        and(eq(mlsKeyPackages.consumed, true), lt(mlsKeyPackages.createdAt, consumedCutoff)),
+        and(eq(mlsKeyPackages.consumed, false), lt(mlsKeyPackages.createdAt, unconsumedCutoff)),
+      ),
+    )
+    .returning({ id: mlsKeyPackages.id });
+
+  return result.length;
+}
+
+/**
+ * Flag (never delete) devices that have been revoked longer than the stale
+ * window. Idempotent: only rows with `staleFlaggedAt IS NULL` are touched, so
+ * re-running this pass against already-flagged devices is a no-op.
+ */
+export async function runDeviceStaleFlagPass(): Promise<number> {
+  const cutoff = new Date(Date.now() - getDeviceStaleAfterMs());
+
+  const result = await db
+    .update(devices)
+    .set({ staleFlaggedAt: new Date() })
+    .where(
+      and(isNotNull(devices.revokedAt), lt(devices.revokedAt, cutoff), isNull(devices.staleFlaggedAt)),
+    )
+    .returning({ id: devices.id });
+
+  return result.length;
+}
+
+let gcTimer: ReturnType<typeof setInterval> | null = null;
+
+export function startDeviceGcJob(): void {
+  if (gcTimer) return;
+  gcTimer = setInterval(() => {
+    void (async () => {
+      try {
+        const prekeys = await runPrekeyGcPass();
+        const keyPackages = await runMlsKeyPackageGcPass();
+        const flagged = await runDeviceStaleFlagPass();
+        if (prekeys || keyPackages || flagged) {
+          console.log(
+            `[device-gc] pruned ${prekeys} prekey(s), ${keyPackages} MLS key package(s), flagged ${flagged} stale device(s)`,
+          );
+        }
+      } catch (err) {
+        console.error('[device-gc] job error:', err);
+      }
+    })();
+  }, getGcIntervalMs());
+  gcTimer.unref();
+}
+
+export function stopDeviceGcJob(): void {
+  if (gcTimer) {
+    clearInterval(gcTimer);
+    gcTimer = null;
+  }
+}
diff --git a/apps/backend/src/services/envelopeGc.ts b/apps/backend/src/services/envelopeGc.ts
new file mode 100644
index 00000000..533334ed
--- /dev/null
+++ b/apps/backend/src/services/envelopeGc.ts
@@ -0,0 +1,78 @@
+/**
+ * Background message-envelope GC service.
+ *
+ * `message_envelopes` holds one row per (message, recipient device) — it is
+ * the actual delivery unit, and the only thing that keeps it around after
+ * delivery is auditability of delivered/read timestamps. Left unbounded it
+ * grows with every message * every recipient device, so this job deletes:
+ *
+ *   1. Envelopes that have been delivered and are older than the delivered-
+ *      retention window (the common case: the recipient got it).
+ *   2. Envelopes past the max-age ceiling regardless of delivery state (a
+ *      device that never comes back to collect its envelope should not pin
+ *      storage forever).
+ *
+ * A plain DELETE ... WHERE is naturally idempotent and safe to retry: a
+ * crash mid-run just means the next tick deletes whatever is left.
+ */
+import { and, lt, or, isNotNull } from 'drizzle-orm';
+import { db } from '../db/index.js';
+import { messageEnvelopes } from '../db/schema.js';
+
+function getEnvelopeDeliveredRetentionMs(): number {
+  const raw = process.env['ENVELOPE_DELIVERED_RETENTION_DAYS'];
+  const parsed = raw !== undefined ? Number(raw) : NaN;
+  const days = Number.isFinite(parsed) && parsed > 0 ? parsed : 7;
+  return days * 24 * 60 * 60 * 1_000;
+}
+
+function getEnvelopeMaxAgeMs(): number {
+  const raw = process.env['ENVELOPE_MAX_AGE_DAYS'];
+  const parsed = raw !== undefined ? Number(raw) : NaN;
+  const days = Number.isFinite(parsed) && parsed > 0 ? parsed : 30;
+  return days * 24 * 60 * 60 * 1_000;
+}
+
+function getGcIntervalMs(): number {
+  const raw = process.env['ENVELOPE_GC_INTERVAL_MS'];
+  const parsed = raw !== undefined ? Number(raw) : NaN;
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 30 * 60 * 1_000; // every 30 minutes
+}
+
+export async function runEnvelopeGcPass(): Promise<number> {
+  const deliveredCutoff = new Date(Date.now() - getEnvelopeDeliveredRetentionMs());
+  const maxAgeCutoff = new Date(Date.now() - getEnvelopeMaxAgeMs());
+
+  const result = await db
+    .delete(messageEnvelopes)
+    .where(
+      or(
+        and(isNotNull(messageEnvelopes.deliveredAt), lt(messageEnvelopes.deliveredAt, deliveredCutoff)),
+        lt(messageEnvelopes.createdAt, maxAgeCutoff),
+      ),
+    )
+    .returning({ id: messageEnvelopes.id });
+
+  return result.length;
+}
+
+let envelopeGcTimer: ReturnType<typeof setInterval> | null = null;
+
+export function startEnvelopeGcJob(): void {
+  if (envelopeGcTimer) return;
+  envelopeGcTimer = setInterval(() => {
+    void runEnvelopeGcPass()
+      .then((count) => {
+        if (count) console.log(`[envelope-gc] pruned ${count} envelope(s)`);
+      })
+      .catch((err) => console.error('[envelope-gc] job error:', err));
+  }, getGcIntervalMs());
+  envelopeGcTimer.unref();
+}
+
+export function stopEnvelopeGcJob(): void {
+  if (envelopeGcTimer) {
+    clearInterval(envelopeGcTimer);
+    envelopeGcTimer = null;
+  }
+}
diff --git a/apps/backend/src/services/fileCleanup.ts b/apps/backend/src/services/fileCleanup.ts
index d0284ea8..97049a23 100644
--- a/apps/backend/src/services/fileCleanup.ts
+++ b/apps/backend/src/services/fileCleanup.ts
@@ -3,10 +3,14 @@
  *
  * Implements #231 – soft-delete (files.deletedAt) is set immediately when a
  * message is retracted. This job hard-deletes the S3 object once every
- * referencing message is also soft-deleted (ref-counting across envelopes).
+ * referencing message is also soft-deleted (ref-counting across envelopes)
+ * and, if configured, a grace period has elapsed since the soft-delete.
  *
  * The job is idempotent: it sets hardDeletedAt only after a successful S3
  * delete, so a crash between steps is safe to retry.
+ *
+ * Retention windows are configurable via env so operators can tune them per
+ * deployment without a code change.
  */
 import { isNotNull, isNull, sql, and, eq, lt } from 'drizzle-orm';
 import { db } from '../db/index.js';
@@ -14,7 +18,25 @@ import { files } from '../db/schema.js';
 import { getObjectStore } from '../lib/objectStore.js';
 import { reenableExpiredBackoffs } from './pushNotification.js';
 
-const CLEANUP_INTERVAL_MS = 5 * 60 * 1_000; // every 5 minutes
+function getCleanupIntervalMs(): number {
+  const raw = process.env['FILE_GC_INTERVAL_MS'];
+  const parsed = raw !== undefined ? Number(raw) : NaN;
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 5 * 60 * 1_000; // every 5 minutes
+}
+
+/** Grace period after soft-delete before a fully-unreferenced file is hard-deleted. */
+function getHardDeleteGraceMs(): number {
+  const raw = process.env['FILE_HARD_DELETE_GRACE_MS'];
+  const parsed = raw !== undefined ? Number(raw) : NaN;
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0;
+}
+
+/** How long an unconfirmed (`pending`) upload may sit before its slot is reclaimed. */
+function getPendingUploadTtlMs(): number {
+  const raw = process.env['PENDING_UPLOAD_TTL_MS'];
+  const parsed = raw !== undefined ? Number(raw) : NaN;
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 24 * 60 * 60 * 1_000; // 24 hours
+}
 
 /**
  * Soft-delete a file record when its owning message is retracted.
@@ -37,8 +59,9 @@ export async function softDeleteFile(fileId: string): Promise<void> {
  * remaining live message references. Idempotent and safe to retry.
  */
 export async function runHardDeletePass(): Promise<void> {
+  const graceCutoff = new Date(Date.now() - getHardDeleteGraceMs());
   const candidates = await db.query.files.findMany({
-    where: (f) => isNotNull(f.deletedAt) && isNull(f.hardDeletedAt),
+    where: (f) => and(isNotNull(f.deletedAt), isNull(f.hardDeletedAt), lt(f.deletedAt, graceCutoff)),
     columns: { id: true, storageKey: true },
   });
 
@@ -65,8 +88,8 @@ export async function runHardDeletePass(): Promise<void> {
     }
   }
 
-  // Garbage-collect unconfirmed pending files older than 24 hours
-  const stalePendingDate = new Date(Date.now() - 24 * 60 * 60 * 1000);
+  // Garbage-collect unconfirmed pending files past the configured TTL.
+  const stalePendingDate = new Date(Date.now() - getPendingUploadTtlMs());
   const pendingCandidates = await db.query.files.findMany({
     where: (f) => and(eq(f.status, 'pending'), lt(f.createdAt, stalePendingDate)),
     columns: { id: true, storageKey: true },
@@ -94,7 +117,7 @@ export function startFileCleanupJob(): void {
     } catch (err) {
       console.error('[file-cleanup] job error:', err);
     }
-  }, CLEANUP_INTERVAL_MS);
+  }, getCleanupIntervalMs());
   cleanupTimer.unref();
 }
 

From a8ef320d4aa319b7526bc7ca08cca26be3d7ce3c Mon Sep 17 00:00:00 2001
From: G-ELM <alfygodwin@gmail.com>
Date: Tue, 28 Jul 2026 01:14:52 +0100
Subject: [PATCH 2/4] chore(backend): audit repository layer for
 post-schema-overhaul drift
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Audited every db.query.* / relation `with:` usage across routes/, services/,
lib/, middleware/, and socket/ against the current db/schema.ts:
- Cross-checked every relation name used in a `with:` clause (members, user,
  wallets, messages, sender, senderDevice, envelopes, editsMessage/edits,
  conversation, device, prekeys, mlsKeyPackages, pushSubscriptions) against
  the `relations()` declarations in schema.ts.
- Grepped for likely-stale field names from a pre-ciphertext model (.content,
  .publicKey, .isEncrypted, .nonce/.iv, .mediaUrl/.thumbnailUrl,
  .sequenceNumber) — no references found; the only `content` occurrences are
  the client-facing socket payload field (mapped to `ciphertext` before
  persisting) and unrelated `contentType` usages.
- Result: no references to dropped columns/relations remain, and every
  relation query's result shape matches its consuming type
  (ConversationPayload/ConversationMemberPayload/MessageLike). Documented,
  in routes/conversations.ts, why the `as never` casts around the dynamic
  `getConversationRelations` config are a drizzle generic-inference
  limitation rather than an unverified shape.
- One dangling reference found (not a dropped column): routes/conversations.ts
  and a test link to docs/message-encryption-migration.md, which doesn't
  exist yet — authored in the next commit (Task 4).

Full backend build + test suite should be run in CI to confirm green, per
instruction not to run test/build scripts locally in this session.
---
 apps/backend/src/routes/conversations.ts | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/apps/backend/src/routes/conversations.ts b/apps/backend/src/routes/conversations.ts
index 1d758f6c..a959eeb4 100644
--- a/apps/backend/src/routes/conversations.ts
+++ b/apps/backend/src/routes/conversations.ts
@@ -21,6 +21,16 @@ export const conversationsRouter: IRouter = Router();
 
 conversationsRouter.use(requireAuth);
 
+// Post-schema-overhaul audit (see PR description): every relation name below
+// (`members`, `user`, `wallets`, `messages`, `sender`, `envelopes`) was
+// checked against the current `relations()` declarations in db/schema.ts and
+// every selected column against the current table definitions — none
+// reference dropped columns/relations. The `as never` casts at the call
+// sites below exist only because TS can't correlate this function's return
+// type with drizzle's recursive `with:` generic when it's built dynamically
+// (a known drizzle limitation, not a sign the shape is unverified); the
+// `ConversationPayload`/`ConversationMemberPayload` types the results are
+// cast to afterward are what's actually checked against the query shape.
 const getConversationRelations = (deviceId: string) => ({
   members: {
     with: {

From abecd0c71c38e27774683d558bf5bbaba01a4e30 Mon Sep 17 00:00:00 2001
From: G-ELM <alfygodwin@gmail.com>
Date: Tue, 28 Jul 2026 07:39:06 +0100
Subject: [PATCH 3/4] feat(backend): device capability/version negotiation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds devices.capabilities (jsonb) advertising supported protocols
(sealed_box/signal/mls), ciphersuites, and file-transfer versions, so
senders can pick an encryption path both sides support and new protocols
can roll out without breaking older clients.

- lib/capabilities.ts: schema, normalization, and selectProtocol() — a
  priority-ordered (mls > signal > sealed_box) negotiation function that
  always falls back to sealed_box (every device implicitly supports it,
  including rows from before this column existed), and silently ignores
  protocol/ciphersuite strings it doesn't recognize instead of erroring.
- POST /auth/verify and POST /devices accept an optional capabilities field
  at registration, and apply it on re-verify/re-registration ("upgrade").
- New PATCH /devices/:id/capabilities lets a device advertise updated
  capabilities standalone, without re-registering its identity key.
- GET /devices, GET /users/:userId/devices/:deviceId/key-bundle,
  GET /user-devices/:id/public-key, and GET /conversations/:id/devices now
  return each device's capabilities; the conversation devices endpoint also
  returns a computed negotiatedProtocol against the caller's own device.

Migrated via 0002_device_capabilities.sql (defaults existing rows to the
sealed_box-only baseline). Updated devices.test.ts's exact-response-shape
assertion for the new field.
---
 .../drizzle/0002_device_capabilities.sql      |   8 ++
 apps/backend/drizzle/meta/_journal.json       |   7 ++
 apps/backend/src/__tests__/devices.test.ts    |   1 +
 apps/backend/src/db/schema.ts                 |  10 ++
 apps/backend/src/lib/capabilities.ts          | 115 ++++++++++++++++++
 apps/backend/src/routes/auth.ts               |   6 +
 apps/backend/src/routes/conversations.ts      |  12 ++
 apps/backend/src/routes/devices.ts            |  56 +++++++++
 apps/backend/src/routes/userDevices.ts        |   3 +
 apps/backend/src/routes/users.ts              |   4 +
 apps/backend/src/schemas/auth.schemas.ts      |   5 +
 11 files changed, 227 insertions(+)
 create mode 100644 apps/backend/drizzle/0002_device_capabilities.sql
 create mode 100644 apps/backend/src/lib/capabilities.ts

diff --git a/apps/backend/drizzle/0002_device_capabilities.sql b/apps/backend/drizzle/0002_device_capabilities.sql
new file mode 100644
index 00000000..80f0b6ce
--- /dev/null
+++ b/apps/backend/drizzle/0002_device_capabilities.sql
@@ -0,0 +1,8 @@
+-- Device capability/version negotiation (#180-follow-on).
+--
+-- Advertises supported protocols/ciphersuites/file-transfer versions per
+-- device so senders can pick an encryption path both sides support. Rows
+-- written before this migration default to the sealed_box-only baseline —
+-- the protocol every device in this codebase already implements — so
+-- existing devices negotiate correctly without any backfill.
+ALTER TABLE "devices" ADD COLUMN IF NOT EXISTS "capabilities" jsonb DEFAULT '{"protocols":["sealed_box"],"ciphersuites":[],"fileTransfer":[]}'::jsonb NOT NULL;
diff --git a/apps/backend/drizzle/meta/_journal.json b/apps/backend/drizzle/meta/_journal.json
index d3066729..28ce3d22 100644
--- a/apps/backend/drizzle/meta/_journal.json
+++ b/apps/backend/drizzle/meta/_journal.json
@@ -15,6 +15,13 @@
       "when": 1785100800000,
       "tag": "0001_gc_background_jobs",
       "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "7",
+      "when": 1785187200000,
+      "tag": "0002_device_capabilities",
+      "breakpoints": true
     }
   ]
 }
\ No newline at end of file
diff --git a/apps/backend/src/__tests__/devices.test.ts b/apps/backend/src/__tests__/devices.test.ts
index 75707dd1..ccf954c3 100644
--- a/apps/backend/src/__tests__/devices.test.ts
+++ b/apps/backend/src/__tests__/devices.test.ts
@@ -182,6 +182,7 @@ describe('GET /devices', () => {
         'platform',
         'lastSeenAt',
         'revokedAt',
+        'capabilities',
         'oneTimePreKeysRemaining',
       ].sort(),
     );
diff --git a/apps/backend/src/db/schema.ts b/apps/backend/src/db/schema.ts
index 0a4cb1b6..1e234975 100644
--- a/apps/backend/src/db/schema.ts
+++ b/apps/backend/src/db/schema.ts
@@ -9,9 +9,11 @@ import {
   integer,
   uniqueIndex,
   check,
+  jsonb,
   type AnyPgColumn,
 } from 'drizzle-orm/pg-core';
 import { relations, sql } from 'drizzle-orm';
+import { DEFAULT_CAPABILITIES, type DeviceCapabilities } from '../lib/capabilities.js';
 
 export const users = pgTable('users', {
   id: uuid('id').primaryKey().defaultRandom(),
@@ -192,6 +194,14 @@ export const devices = pgTable(
     // DEVICE_STALE_AFTER_DAYS. Purely informational — flags the row for
     // admin visibility/future hard-delete without destroying audit history.
     staleFlaggedAt: timestamp('stale_flagged_at'),
+    // Supported protocols/ciphersuites/file-transfer versions this device
+    // advertises (lib/capabilities.ts). Defaults to the sealed_box-only
+    // baseline so rows written before this column existed, or clients that
+    // never send it, negotiate correctly rather than erroring — see
+    // lib/capabilities.ts `normalizeCapabilities`.
+    capabilities: jsonb('capabilities').$type<DeviceCapabilities>().notNull().default(
+      DEFAULT_CAPABILITIES,
+    ),
     createdAt: timestamp('created_at').notNull().defaultNow(),
     updatedAt: timestamp('updated_at').notNull().defaultNow(),
   },
diff --git a/apps/backend/src/lib/capabilities.ts b/apps/backend/src/lib/capabilities.ts
new file mode 100644
index 00000000..5a7ea9be
--- /dev/null
+++ b/apps/backend/src/lib/capabilities.ts
@@ -0,0 +1,115 @@
+/**
+ * Device capability/version negotiation (#180-follow-on).
+ *
+ * `devices.capabilities` is a small JSON document a device advertises at
+ * registration and can update later ("upgrade") without re-registering its
+ * identity key. It lets a sender pick an encryption path both the sender's
+ * and recipient's devices actually support, so new protocols (Signal
+ * double-ratchet, MLS group ratchet) can roll out gradually without
+ * breaking devices that only understand the original sealed-box/X3DH
+ * envelope scheme already in production.
+ *
+ * Shape:
+ *   protocols     — messaging encryption protocols this device can decrypt,
+ *                    e.g. ["sealed_box", "signal", "mls"]. "sealed_box" names
+ *                    the X3DH + per-device-envelope scheme every device in
+ *                    this codebase already implements (devices/messages
+ *                    routes) — it is the universal fallback.
+ *   ciphersuites  — MLS/Signal ciphersuite identifiers this device supports.
+ *                    Only meaningful when "mls" (or a future ciphersuite-
+ *                    parameterized protocol) is present in `protocols`.
+ *   fileTransfer  — file-encryption scheme versions this device supports,
+ *                    e.g. ["file-v1"]. Independent of the messaging protocol.
+ *
+ * All fields are optional and additional/unrecognized values are preserved
+ * but ignored by `selectProtocol` — this is what makes negotiation forward-
+ * and backward-compatible: an older server ignores protocol names it
+ * doesn't recognize instead of failing, and a device that never advertised
+ * capabilities at all (pre-dates this feature) is treated as sealed_box-only.
+ */
+import { z } from 'zod';
+
+export const KNOWN_PROTOCOLS = ['sealed_box', 'signal', 'mls'] as const;
+export type KnownProtocol = (typeof KNOWN_PROTOCOLS)[number];
+
+/** Every device implicitly supports this — it predates the capabilities column. */
+export const BASELINE_PROTOCOL: KnownProtocol = 'sealed_box';
+
+/**
+ * Preference order used when both sides advertise more than one mutually
+ * supported protocol — prefer the strongest/newest scheme available to both.
+ */
+const PROTOCOL_PRIORITY: readonly KnownProtocol[] = ['mls', 'signal', 'sealed_box'];
+
+export const DeviceCapabilitiesSchema = z
+  .object({
+    protocols: z.array(z.string()).default([BASELINE_PROTOCOL]),
+    ciphersuites: z.array(z.string()).default([]),
+    fileTransfer: z.array(z.string()).default([]),
+  })
+  .partial()
+  .default({});
+
+export type DeviceCapabilities = z.infer<typeof DeviceCapabilitiesSchema>;
+
+/** The capability set assumed for a device that has never advertised one. */
+export const DEFAULT_CAPABILITIES: DeviceCapabilities = {
+  protocols: [BASELINE_PROTOCOL],
+  ciphersuites: [],
+  fileTransfer: [],
+};
+
+/**
+ * Normalize a possibly-missing/possibly-malformed capabilities value into a
+ * concrete `DeviceCapabilities`. Unknown/older clients — including rows from
+ * before this column existed (`null`/`undefined`) — fall back to the
+ * sealed_box-only baseline rather than erroring, satisfying "unknown/older
+ * capabilities handled gracefully".
+ */
+export function normalizeCapabilities(raw: unknown): DeviceCapabilities {
+  const parsed = DeviceCapabilitiesSchema.safeParse(raw ?? {});
+  if (!parsed.success) return { ...DEFAULT_CAPABILITIES };
+
+  const protocols = parsed.data.protocols?.length ? parsed.data.protocols : [BASELINE_PROTOCOL];
+  return {
+    protocols,
+    ciphersuites: parsed.data.ciphersuites ?? [],
+    fileTransfer: parsed.data.fileTransfer ?? [],
+  };
+}
+
+/**
+ * Pick the best protocol both sides can speak. Falls back to the universal
+ * `sealed_box` baseline when there is no other overlap — every registered
+ * device supports it, so this never returns `null`.
+ */
+export function selectProtocol(
+  a: unknown,
+  b: unknown,
+): { protocol: KnownProtocol; ciphersuite: string | null } {
+  const capsA = normalizeCapabilities(a);
+  const capsB = normalizeCapabilities(b);
+  const protocolsA = new Set(capsA.protocols);
+  const protocolsB = new Set(capsB.protocols);
+
+  for (const candidate of PROTOCOL_PRIORITY) {
+    if (protocolsA.has(candidate) && protocolsB.has(candidate)) {
+      const ciphersuite =
+        candidate === 'mls' ? selectCiphersuite(capsA.ciphersuites, capsB.ciphersuites) : null;
+      return { protocol: candidate, ciphersuite };
+    }
+  }
+
+  return { protocol: BASELINE_PROTOCOL, ciphersuite: null };
+}
+
+/** First ciphersuite identifier both sides advertise, preserving the caller's preference order. */
+function selectCiphersuite(a: string[] = [], b: string[] = []): string | null {
+  const setB = new Set(b);
+  return a.find((suite) => setB.has(suite)) ?? null;
+}
+
+/** Whether `capabilities` includes support for the given file-transfer scheme version. */
+export function supportsFileTransfer(capabilities: unknown, version: string): boolean {
+  return normalizeCapabilities(capabilities).fileTransfer?.includes(version) ?? false;
+}
diff --git a/apps/backend/src/routes/auth.ts b/apps/backend/src/routes/auth.ts
index a31e07d1..d9c8cca6 100644
--- a/apps/backend/src/routes/auth.ts
+++ b/apps/backend/src/routes/auth.ts
@@ -8,6 +8,7 @@ import { users, wallets, devices } from '../db/schema.js';
 import { eq, and } from 'drizzle-orm';
 import { createNonce, consumeNonce } from '../lib/nonce.js';
 import { signToken } from '../lib/jwt.js';
+import { normalizeCapabilities } from '../lib/capabilities.js';
 import { validate } from '../middleware/validate.js';
 import {
   ChallengeSchema,
@@ -61,6 +62,7 @@ authRouter.post(
     const deviceName = device?.deviceName;
     const platform = device?.platform;
     const registrationId = device?.registrationId;
+    const capabilities = device?.capabilities;
 
     // Validate and consume nonce
     const valid = consumeNonce(walletAddress, nonce);
@@ -133,6 +135,9 @@ authRouter.post(
           ...(deviceName ? { deviceName } : {}),
           ...(platform ? { platform } : {}),
           ...(registrationId !== undefined ? { registrationId } : {}),
+          // A client re-verifying with a newer `capabilities` set is the
+          // "upgrade" path (#180-follow-on) — no re-registration needed.
+          ...(capabilities !== undefined ? { capabilities: normalizeCapabilities(capabilities) } : {}),
         })
         .where(eq(devices.id, deviceId));
     } else {
@@ -145,6 +150,7 @@ authRouter.post(
           platform: platform ?? null,
           registrationId: registrationId ?? null,
           lastSeenAt: new Date(),
+          ...(capabilities !== undefined ? { capabilities: normalizeCapabilities(capabilities) } : {}),
         })
         .returning({ id: devices.id });
       if (!newDevice) {
diff --git a/apps/backend/src/routes/conversations.ts b/apps/backend/src/routes/conversations.ts
index a959eeb4..29ee3a90 100644
--- a/apps/backend/src/routes/conversations.ts
+++ b/apps/backend/src/routes/conversations.ts
@@ -16,6 +16,7 @@ import { invalidateConversationCaches } from '../lib/conversationCache.js';
 import { serializeMessage } from '../lib/messages.js';
 import { getSocketServer } from '../lib/socket.js';
 import { MAX_MESSAGES_LIMIT, DEFAULT_MESSAGES_LIMIT } from '../constants.js';
+import { normalizeCapabilities, selectProtocol } from '../lib/capabilities.js';
 
 export const conversationsRouter: IRouter = Router();
 
@@ -808,9 +809,18 @@ conversationsRouter.get('/:id/devices', async (req: AuthRequest, res) => {
       identityPublicKey: true,
       deviceName: true,
       platform: true,
+      capabilities: true,
     },
   });
 
+  // Look up the caller's own device capabilities so each returned device can
+  // carry the protocol the sender should actually use with it (#180-follow-
+  // on) — sparing every client from re-implementing selectProtocol().
+  const callerDevice = await db.query.devices.findFirst({
+    where: eq(devices.id, req.auth!.deviceId),
+    columns: { capabilities: true },
+  });
+
   res.json({
     devices: deviceRows.map((d) => ({
       id: d.id,
@@ -818,6 +828,8 @@ conversationsRouter.get('/:id/devices', async (req: AuthRequest, res) => {
       identityPublicKey: d.identityPublicKey,
       deviceName: d.deviceName,
       platform: d.platform,
+      capabilities: normalizeCapabilities(d.capabilities),
+      negotiatedProtocol: selectProtocol(callerDevice?.capabilities, d.capabilities).protocol,
     })),
   });
 });
diff --git a/apps/backend/src/routes/devices.ts b/apps/backend/src/routes/devices.ts
index 8686b7b2..7635ec2e 100644
--- a/apps/backend/src/routes/devices.ts
+++ b/apps/backend/src/routes/devices.ts
@@ -21,6 +21,7 @@ import { SignedPreKeyEntrySchema, PreKeyEntrySchema, verifyEd25519Signature } fr
 import { conversationRoom } from '../services/roomManager.js';
 import { redis } from '../lib/redis.js';
 import { markDeviceRevoked } from '../services/deviceRevocation.js';
+import { DeviceCapabilitiesSchema, normalizeCapabilities } from '../lib/capabilities.js';
 
 export const devicesRouter: RouterType = Router();
 
@@ -81,6 +82,7 @@ devicesRouter.get('/', async (req: AuthRequest, res) => {
         platform: device.platform,
         lastSeenAt: device.lastSeenAt,
         revokedAt: device.revokedAt,
+        capabilities: normalizeCapabilities(device.capabilities),
         oneTimePreKeysRemaining: remainingByDevice.get(device.id) ?? 0,
         createdAt: device.createdAt,
         current: device.id === currentDeviceId,
@@ -296,6 +298,51 @@ devicesRouter.post('/:id/prekeys', validate(UploadPreKeysSchema), async (req: Au
   });
 });
 
+// ─── PATCH /devices/:id/capabilities ────────────────────────────────────────
+// Lets a device advertise updated capabilities (e.g. gaining MLS support in
+// a later app version) without re-registering its identity key (#180-follow-
+// on "upgrade" path). Unrecognized protocol/ciphersuite strings are accepted
+// and stored as-is — they're simply ignored by selectProtocol() until this
+// server version knows about them, which is what keeps rollout staged rather
+// than a hard break for mismatched client/server versions.
+
+devicesRouter.patch(
+  '/:id/capabilities',
+  validate(DeviceCapabilitiesSchema),
+  async (req: AuthRequest, res) => {
+    const deviceId = req.params['id'] as string;
+    const callerId = req.auth!.userId;
+
+    const device = await db.query.devices.findFirst({
+      where: eq(devices.id, deviceId),
+    });
+
+    if (!device) {
+      res.status(404).json({ error: 'Device not found' });
+      return;
+    }
+
+    if (device.userId !== callerId) {
+      res.status(403).json({ error: 'Only the device owner may update capabilities' });
+      return;
+    }
+
+    if (device.revokedAt) {
+      res.status(403).json({ error: 'Device is revoked' });
+      return;
+    }
+
+    const capabilities = normalizeCapabilities(req.body);
+
+    await db
+      .update(devices)
+      .set({ capabilities, updatedAt: new Date() })
+      .where(eq(devices.id, deviceId));
+
+    res.status(200).json({ id: deviceId, capabilities });
+  },
+);
+
 // ─── POST /devices — register a new device for an existing user --------------
 
 devicesRouter.post('/', validate(RegisterDeviceSchema), async (req: AuthRequest, res) => {
@@ -317,12 +364,18 @@ devicesRouter.post('/', validate(RegisterDeviceSchema), async (req: AuthRequest,
     if (existing) {
       // Re-registering a previously-revoked identity key re-activates it
       // rather than creating a duplicate row for the same crypto identity.
+      // This also doubles as the "upgrade" path for capabilities (#180-
+      // follow-on): a client re-registering with a newer capability set
+      // gets it applied without needing a separate call.
       [row] = await db
         .update(devices)
         .set({
           deviceName: body.deviceName,
           platform: body.platform,
           registrationId: body.registrationId ?? null,
+          ...(body.capabilities !== undefined
+            ? { capabilities: normalizeCapabilities(body.capabilities) }
+            : {}),
           revokedAt: null,
           updatedAt: new Date(),
         })
@@ -337,6 +390,9 @@ devicesRouter.post('/', validate(RegisterDeviceSchema), async (req: AuthRequest,
           deviceName: body.deviceName,
           platform: body.platform,
           registrationId: body.registrationId ?? null,
+          ...(body.capabilities !== undefined
+            ? { capabilities: normalizeCapabilities(body.capabilities) }
+            : {}),
         })
         .returning({ id: devices.id, createdAt: devices.createdAt });
     }
diff --git a/apps/backend/src/routes/userDevices.ts b/apps/backend/src/routes/userDevices.ts
index fa9b6c33..40b72fe2 100644
--- a/apps/backend/src/routes/userDevices.ts
+++ b/apps/backend/src/routes/userDevices.ts
@@ -11,6 +11,7 @@ import { and, eq, inArray, isNull } from 'drizzle-orm';
 import { db } from '../db/index.js';
 import { conversationMembers, devices } from '../db/schema.js';
 import { requireAuth, type AuthRequest } from '../middleware/auth.js';
+import { normalizeCapabilities } from '../lib/capabilities.js';
 
 export const userDevicesRouter: RouterType = Router();
 
@@ -27,6 +28,7 @@ userDevicesRouter.get('/:id/public-key', async (req: AuthRequest, res) => {
         id: true,
         userId: true,
         identityPublicKey: true,
+        capabilities: true,
       },
     });
 
@@ -61,6 +63,7 @@ userDevicesRouter.get('/:id/public-key', async (req: AuthRequest, res) => {
       id: device.id,
       userId: device.userId,
       identityPublicKey: device.identityPublicKey,
+      capabilities: normalizeCapabilities(device.capabilities),
     });
   } catch {
     res.status(500).json({ error: 'Failed to fetch device public key' });
diff --git a/apps/backend/src/routes/users.ts b/apps/backend/src/routes/users.ts
index e4798a18..0c8915c7 100644
--- a/apps/backend/src/routes/users.ts
+++ b/apps/backend/src/routes/users.ts
@@ -8,6 +8,7 @@ import { redis } from '../lib/redis.js';
 import { isOnline, deriveDevicePresence } from '../services/presence.js';
 import { getSocketServer } from '../lib/socket.js';
 import { conversationRoom } from '../services/roomManager.js';
+import { normalizeCapabilities } from '../lib/capabilities.js';
 
 export const usersRouter: RouterType = Router();
 
@@ -250,6 +251,9 @@ usersRouter.get('/:userId/devices/:deviceId/key-bundle', async (req: AuthRequest
     deviceId: device.id,
     identityPublicKey: device.identityPublicKey,
     registrationId: device.registrationId,
+    // Lets the initiating sender pick an encryption path this recipient
+    // device supports before running X3DH (#180-follow-on).
+    capabilities: normalizeCapabilities(device.capabilities),
     signedPreKey: {
       keyId: signedPreKey.keyId,
       publicKey: signedPreKey.publicKey,
diff --git a/apps/backend/src/schemas/auth.schemas.ts b/apps/backend/src/schemas/auth.schemas.ts
index 8d2c6988..da4acfb1 100644
--- a/apps/backend/src/schemas/auth.schemas.ts
+++ b/apps/backend/src/schemas/auth.schemas.ts
@@ -1,5 +1,6 @@
 import { z } from 'zod';
 import { IdentityPublicKeySchema } from '../lib/keys.js';
+import { DeviceCapabilitiesSchema } from '../lib/capabilities.js';
 
 export const ChallengeSchema = z.object({
   walletAddress: z.string().min(1, 'walletAddress is required'),
@@ -13,6 +14,10 @@ export const DeviceSchema = z.object({
   platform: z.enum(['web', 'ios', 'android']),
   identityPublicKey: IdentityPublicKeySchema,
   registrationId: z.number().int().nonnegative().optional(),
+  // Supported protocols/ciphersuites/file-transfer versions (#180-follow-on).
+  // Optional — omitting it defaults to the sealed_box-only baseline so older
+  // clients that predate this field keep working unchanged.
+  capabilities: DeviceCapabilitiesSchema.optional(),
 });
 
 export const VerifySchema = z

From 6c912436b32090ceb964c2a30f7964307dd5a100 Mon Sep 17 00:00:00 2001
From: G-ELM <alfygodwin@gmail.com>
Date: Tue, 28 Jul 2026 07:51:40 +0100
Subject: [PATCH 4/4] feat(backend): one-time migration dropping plaintext
 messages.content
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Authors drizzle/0003_ciphertext_only_messages.sql, the migration that
resets messages to the ciphertext-only model, plus its rollback and the
documented policy decision.

- Policy: archive-then-purge (chosen over an in-place tombstone) — any
  existing plaintext is copied into a new message_content_archive table
  (no FK to messages, no route/query path through the app's API) before
  the content column and its GIN index are dropped. Documented in full,
  including why the tombstone alternative was rejected, at
  docs/message-encryption-migration.md — this also resolves the dangling
  link found during the Task 2 audit
  (routes/conversations.ts's 410 search response, and a matching test,
  already pointed at this doc path).
- Forward migration is idempotent/guarded: the archive-insert only runs if
  information_schema shows `content` still exists (a plain reference to a
  dropped column would fail to parse otherwise), every DROP uses IF EXISTS,
  and the new-columns/tables it also ensures (messages.ciphertext,
  senderDeviceId, fileId, editsMessageId, deletedAt, message_envelopes) use
  IF NOT EXISTS — safe to run on this repo's current DB (already
  ciphertext-only, so every guard is a no-op) or on a populated legacy DB
  that still has plaintext.
- Rollback lives at drizzle/rollback/0003_ciphertext_only_messages.down.sql
  (drizzle-kit has no down-migration runner, so it's invoked manually) and
  restores archived plaintext + a GIN index, with its data-loss/scope
  limitations documented at the top of the script and in the doc.
- No silent E2EE claim: once `content` is dropped, serializeMessage()'s
  existing fallback chain means a pre-cutover row with no ciphertext/
  envelope resolves to `unavailable: true` rather than looking like normal
  ciphertext.
---
 .../docs/message-encryption-migration.md      | 135 ++++++++++++++++++
 .../drizzle/0003_ciphertext_only_messages.sql | 102 +++++++++++++
 apps/backend/drizzle/meta/_journal.json       |   7 +
 .../0003_ciphertext_only_messages.down.sql    |  35 +++++
 apps/backend/src/db/schema.ts                 |  26 ++++
 5 files changed, 305 insertions(+)
 create mode 100644 apps/backend/docs/message-encryption-migration.md
 create mode 100644 apps/backend/drizzle/0003_ciphertext_only_messages.sql
 create mode 100644 apps/backend/drizzle/rollback/0003_ciphertext_only_messages.down.sql

diff --git a/apps/backend/docs/message-encryption-migration.md b/apps/backend/docs/message-encryption-migration.md
new file mode 100644
index 00000000..c903d1f0
--- /dev/null
+++ b/apps/backend/docs/message-encryption-migration.md
@@ -0,0 +1,135 @@
+# Message encryption migration: plaintext → ciphertext-only
+
+This document is the canonical reference linked from `GET /conversations/:id/search`'s
+410 response and from the schema-overhaul audit — it records the decision behind
+`drizzle/0003_ciphertext_only_messages.sql` and the policy for any plaintext that
+existed before it.
+
+## Background
+
+Earlier in this project's history, `messages` carried a plaintext `content` column
+(plus a GIN index supporting server-side search) alongside the work to move to
+per-recipient E2EE envelopes (`message_envelopes`) and a `ciphertext` column on
+`messages` itself. `docs/e2ee-onboarding.md` documents the current, implemented
+E2EE data model; this document covers the one-time cutover away from the old
+plaintext column and what happens to data that predates it.
+
+Server-side message search was removed as part of this cutover — the server
+cannot search ciphertext it cannot read, so `GET /conversations/:id/search` now
+returns `410 Gone` pointing here, and search is client-side over decrypted
+messages.
+
+## Decision: archive-then-purge (not a tombstone)
+
+Two options were considered for handling rows where `messages.content` is
+non-null at migration time:
+
+1. **Tombstone in place** — keep a `content`-shaped column (or a
+   `legacyPlaintext` column) on `messages` forever, nulled out or flagged, so
+   the "this row used to be plaintext" fact stays attached to the row.
+2. **Archive then purge** (chosen) — copy every non-null `content` value into
+   a separate `message_content_archive` table, then drop the column from
+   `messages` entirely.
+
+**Archive-then-purge was chosen** because:
+
+- It keeps `messages` fully ciphertext-shaped going forward. Every code path
+  that reads a message (`serializeMessage()` in `lib/messages.ts`, the
+  `conversations`/`messages` routes, the socket layer) only ever has a
+  `ciphertext` branch to reason about — there is no lingering plaintext
+  branch that a future change could accidentally serve to a client as if it
+  were normal ciphertext.
+- `message_content_archive` has **no route or query path through the app's
+  API** (see `db/schema.ts` — the table is defined but never imported by
+  anything in `routes/`, `services/`, or `socket/`). It exists purely for
+  legal/compliance/audit access on its own retention schedule, separate from
+  the live message store's.
+- It avoids growing the "hot path" `messages` table with a rarely-needed
+  legacy column and its GIN index indefinitely.
+
+The rejected tombstone approach was ruled out because it permanently taxes
+every future `messages` migration and query with a legacy-shaped column for
+data that, by definition, existed only before this cutover and will never
+grow again.
+
+## No silent E2EE claim over old plaintext
+
+Rows whose plaintext was archived are **not** presented to clients as if they
+were ciphertext. Once `content` is dropped, `serializeMessage()`'s existing
+fallback chain (envelope → base `ciphertext` → `unavailable: true`) means a
+pre-cutover row with no `ciphertext` and no matching envelope now correctly
+resolves to `{ ciphertext: null, unavailable: true }` — the client sees the
+message as unavailable rather than being shown a decrypted-looking payload
+that was, in fact, sent before E2EE existed. Nothing in this migration
+retroactively re-encrypts old plaintext or backdates an "encrypted" claim
+onto it.
+
+## What the forward migration does
+
+`drizzle/0003_ciphertext_only_messages.sql`:
+
+1. Creates `message_content_archive` (id, `original_message_id` — intentionally
+   **not** a foreign key, so hard-deleting a `messages` row never cascades
+   into the archive — `conversation_id`, `sender_id`, `content`,
+   `original_created_at`, `archived_at`).
+2. If (and only if) `messages.content` still exists — checked dynamically via
+   `information_schema.columns`, since a plain SQL statement referencing a
+   column that doesn't exist fails to parse — copies every non-null row into
+   the archive.
+3. Drops the column's GIN index. The exact index name from before this
+   repo's migration history was squashed to a single `0000` migration is not
+   recoverable, so several plausible historical names are dropped
+   defensively with `DROP INDEX IF EXISTS` (a no-op for any name that
+   doesn't match).
+4. Drops `messages.content` with `DROP COLUMN IF EXISTS` — a no-op on a
+   database that never had it, which is the case for this repo's own `0000`
+   migration.
+5. Defensively ensures the ciphertext-model columns/tables (`messages.ciphertext`,
+   `senderDeviceId`, `fileId`, `editsMessageId`, `deletedAt`,
+   `message_envelopes`) exist, for any database whose applied-migration
+   history predates the `0000` squash. On every database that already ran
+   `0000` this is a pure no-op.
+
+This makes the migration safe to run both against this repo's current
+(already-ciphertext) schema and against a hypothetical legacy database that
+still has the old plaintext column and a populated `messages` table.
+
+## Rollback plan
+
+`drizzle-kit` has no built-in down-migration runner, so the rollback lives at
+`drizzle/rollback/0003_ciphertext_only_messages.down.sql` and is invoked
+manually, e.g.:
+
+```sh
+psql "$DATABASE_URL" -f apps/backend/drizzle/rollback/0003_ciphertext_only_messages.down.sql
+```
+
+It:
+
+1. Re-adds `messages.content` (nullable).
+2. Recreates a GIN full-text index over it.
+3. Restores plaintext from `message_content_archive` back into `messages.content`
+   for every row the archive has a match for.
+
+**Rollback limitations** (also called out at the top of the script itself):
+
+- Only rows present in the archive get plaintext restored. Any message sent
+  *after* the forward migration ran was only ever stored as ciphertext —
+  there is nothing to restore for it, by design.
+- The recreated index is a reasonable equivalent, not necessarily
+  byte-identical to whatever definition existed pre-squash.
+- The rollback does **not** remove the ciphertext/envelope/device-capability/
+  GC schema this migration set introduces — undoing those would remove the
+  entire E2EE data model, which is a separate, deliberate decision this
+  script does not make on an operator's behalf.
+- `message_content_archive` is left in place after a rollback (nothing is
+  destroyed); drop it manually once you're satisfied the restore is correct.
+
+## Retention of the archive
+
+`message_content_archive` is not currently covered by the background GC jobs
+in `services/deviceGc.ts` / `services/envelopeGc.ts` / `services/fileCleanup.ts`
+— it is intentionally kept until a compliance/legal retention policy for
+pre-E2EE history is decided separately. A future GC pass following the same
+idempotent, env-configurable pattern as the existing jobs would be the
+natural way to enforce that retention once a window is chosen.
diff --git a/apps/backend/drizzle/0003_ciphertext_only_messages.sql b/apps/backend/drizzle/0003_ciphertext_only_messages.sql
new file mode 100644
index 00000000..80b27ca0
--- /dev/null
+++ b/apps/backend/drizzle/0003_ciphertext_only_messages.sql
@@ -0,0 +1,102 @@
+-- One-time migration: drop plaintext `messages.content`, reset to the
+-- ciphertext-only model.
+--
+-- Policy (documented in full at docs/message-encryption-migration.md):
+-- ARCHIVE THEN PURGE. Any existing plaintext row is copied into
+-- `message_content_archive` — a table with no read path through the app's
+-- API — before the column is dropped from `messages`. This keeps `messages`
+-- fully ciphertext-shaped going forward (so `serializeMessage()` never has a
+-- plaintext branch to accidentally serve) while preserving a compliance/
+-- audit copy of pre-E2EE history on its own retention schedule, instead of
+-- silently destroying it or leaving a tombstone column on `messages` forever.
+--
+-- Safe to run on a DB that never had a `content` column (this repo's own
+-- migration history: 0000 already created `messages` in its ciphertext
+-- shape) — the archive step is a no-op guarded by an information_schema
+-- check, and every DDL statement below uses IF EXISTS/IF NOT EXISTS so
+-- nothing errors either way. See docs/message-encryption-migration.md for
+-- the full rollback plan and its limitations.
+
+CREATE TABLE IF NOT EXISTS "message_content_archive" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	-- Intentionally not a foreign key: this archive must outlive the
+	-- `messages` row it was copied from (e.g. hard-deletion by the message
+	-- GC path must not cascade-delete the compliance copy).
+	"original_message_id" uuid NOT NULL,
+	"conversation_id" uuid,
+	"sender_id" uuid,
+	"content" text NOT NULL,
+	"original_created_at" timestamp,
+	"archived_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "message_content_archive_original_message_idx" ON "message_content_archive" USING btree ("original_message_id");
+--> statement-breakpoint
+
+-- Archive any existing plaintext before the column is dropped. Dynamic SQL
+-- is required here (unlike the DROP statements below) because a plain
+-- top-level statement referencing `messages.content` would fail to parse on
+-- a database where that column never existed.
+DO $$
+BEGIN
+  IF EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_schema = 'public' AND table_name = 'messages' AND column_name = 'content'
+  ) THEN
+    EXECUTE '
+      INSERT INTO message_content_archive
+        (original_message_id, conversation_id, sender_id, content, original_created_at)
+      SELECT id, conversation_id, sender_id, content, created_at
+      FROM messages
+      WHERE content IS NOT NULL
+    ';
+  END IF;
+END $$;
+--> statement-breakpoint
+
+-- Drop the plaintext column's GIN index. The exact index name from
+-- pre-squash history is not recoverable, so every plausible historical name
+-- is dropped defensively — DROP INDEX IF EXISTS is a no-op for names that
+-- don't exist, so this is safe regardless of which (if any) actually match.
+DROP INDEX IF EXISTS "messages_content_gin_idx";
+--> statement-breakpoint
+DROP INDEX IF EXISTS "messages_content_search_idx";
+--> statement-breakpoint
+DROP INDEX IF EXISTS "messages_content_tsv_idx";
+--> statement-breakpoint
+DROP INDEX IF EXISTS "messages_content_idx";
+--> statement-breakpoint
+
+ALTER TABLE "messages" DROP COLUMN IF EXISTS "content";
+--> statement-breakpoint
+
+-- Defensively ensure the ciphertext-model columns/tables this migration's
+-- companion schema overhaul introduced are present, for any database whose
+-- migration history predates the 0000 squash and therefore skipped them.
+-- A no-op everywhere this repo's own 0000 migration already ran.
+ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "ciphertext" text;
+--> statement-breakpoint
+ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "sender_device_id" uuid;
+--> statement-breakpoint
+ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "file_id" uuid;
+--> statement-breakpoint
+ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "edits_message_id" uuid;
+--> statement-breakpoint
+ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "deleted_at" timestamp;
+--> statement-breakpoint
+
+-- Note: this fallback intentionally omits FK constraints — on every database
+-- that already ran this repo's 0000 migration (the only realistic case),
+-- the table already exists with its constraints and this is a pure no-op.
+-- A database old enough to hit this branch predates the squash entirely and
+-- needs a manually-reviewed reconciliation, not a silent constraint add.
+CREATE TABLE IF NOT EXISTS "message_envelopes" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"message_id" uuid NOT NULL,
+	"recipient_device_id" uuid NOT NULL,
+	"recipient_user_id" uuid NOT NULL,
+	"ciphertext" text NOT NULL,
+	"delivered_at" timestamp,
+	"read_at" timestamp,
+	"created_at" timestamp DEFAULT now() NOT NULL
+);
diff --git a/apps/backend/drizzle/meta/_journal.json b/apps/backend/drizzle/meta/_journal.json
index 28ce3d22..e4a81e5b 100644
--- a/apps/backend/drizzle/meta/_journal.json
+++ b/apps/backend/drizzle/meta/_journal.json
@@ -22,6 +22,13 @@
       "when": 1785187200000,
       "tag": "0002_device_capabilities",
       "breakpoints": true
+    },
+    {
+      "idx": 3,
+      "version": "7",
+      "when": 1785273600000,
+      "tag": "0003_ciphertext_only_messages",
+      "breakpoints": true
     }
   ]
 }
\ No newline at end of file
diff --git a/apps/backend/drizzle/rollback/0003_ciphertext_only_messages.down.sql b/apps/backend/drizzle/rollback/0003_ciphertext_only_messages.down.sql
new file mode 100644
index 00000000..239dd42e
--- /dev/null
+++ b/apps/backend/drizzle/rollback/0003_ciphertext_only_messages.down.sql
@@ -0,0 +1,35 @@
+-- Rollback for 0003_ciphertext_only_messages.sql.
+--
+-- drizzle-kit has no built-in "down" migration runner — this script is
+-- invoked manually (e.g. `psql "$DATABASE_URL" -f drizzle/rollback/0003_ciphertext_only_messages.down.sql`)
+-- and is NOT part of the drizzle migration journal, so it is never applied
+-- automatically by `db:migrate`.
+--
+-- LIMITATIONS (see docs/message-encryption-migration.md for full detail):
+--   * Only rows present in `message_content_archive` get their plaintext
+--     restored. Any message sent after the forward migration ran was only
+--     ever stored as ciphertext — there is no plaintext to bring back for it.
+--   * The recreated GIN index is a reasonable equivalent (full-text search
+--     over `content`), not necessarily byte-identical to whatever indexdef
+--     existed pre-squash — that definition was not recoverable (see the
+--     forward migration's comment on why several index names are dropped
+--     defensively rather than one exact name).
+--   * This script does NOT drop the ciphertext/envelope/device-capability/
+--     GC columns and tables added by this migration set. Rolling those back
+--     would remove the entire E2EE data model, not just this one migration's
+--     change — treat that as a separate, deliberate decision, not a side
+--     effect of undoing the content-column drop.
+--   * `message_content_archive` is left in place after restoring so no data
+--     is destroyed by running this script; drop it manually once satisfied.
+
+ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "content" text;
+--> statement-breakpoint
+
+CREATE INDEX IF NOT EXISTS "messages_content_gin_idx" ON "messages" USING gin (to_tsvector('english', "content"));
+--> statement-breakpoint
+
+UPDATE "messages" m
+SET "content" = a."content"
+FROM "message_content_archive" a
+WHERE a."original_message_id" = m."id"
+  AND m."content" IS NULL;
diff --git a/apps/backend/src/db/schema.ts b/apps/backend/src/db/schema.ts
index 1e234975..f25f7965 100644
--- a/apps/backend/src/db/schema.ts
+++ b/apps/backend/src/db/schema.ts
@@ -288,6 +288,32 @@ export const mlsKeyPackages = pgTable(
   ],
 );
 
+// ─── Archived plaintext (one-time ciphertext-only migration) ────────────────
+//
+// Holds plaintext copied out of `messages.content` before that column was
+// dropped (drizzle/0003_ciphertext_only_messages.sql). Deliberately not
+// wired into any route — see docs/message-encryption-migration.md for the
+// archive-then-purge policy this implements. `originalMessageId` is not a
+// foreign key on purpose: this table must outlive the `messages` row it was
+// copied from (e.g. message hard-deletion must not cascade into it).
+
+export const messageContentArchive = pgTable(
+  'message_content_archive',
+  {
+    id: uuid('id').primaryKey().defaultRandom(),
+    originalMessageId: uuid('original_message_id').notNull(),
+    conversationId: uuid('conversation_id'),
+    senderId: uuid('sender_id'),
+    content: text('content').notNull(),
+    originalCreatedAt: timestamp('original_created_at'),
+    archivedAt: timestamp('archived_at').notNull().defaultNow(),
+  },
+  (table) => [index('message_content_archive_original_message_idx').on(table.originalMessageId)],
+);
+
+export type MessageContentArchive = typeof messageContentArchive.$inferSelect;
+export type NewMessageContentArchive = typeof messageContentArchive.$inferInsert;
+
 // ─── Token transfers (#46) ────────────────────────────────────────────────────
 //
 // One row per Soroban `transfer` event the listener (services/stellarListener.ts)