feat: build disaster recovery automation, monitoring, and failover scripts

backend/dr/drMonitoring.ts

@@ -0,0 +1,64 @@
+import { disasterRecoveryService, RPO_SECONDS } from './DisasterRecoveryService';
+import { logger } from '../services/shared/logging';
+
+export class DisasterRecoveryMonitor {
+  private intervalId: NodeJS.Timeout | null = null;
+  private readonly checkIntervalMs: number;
+
+  constructor(checkIntervalMs = 5 * 60 * 1000) { // Default 5 minutes
+    this.checkIntervalMs = checkIntervalMs;
+  }
+
+  start() {
+    if (this.intervalId) return;
+    this.intervalId = setInterval(() => this.checkHealth(), this.checkIntervalMs);
+    logger.info('Disaster Recovery Monitor started.');
+  }
+
+  stop() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+      logger.info('Disaster Recovery Monitor stopped.');
+    }
+  }
+
+  async checkHealth(): Promise<boolean> {
+    try {
+      const backups = await disasterRecoveryService.listBackups();
+
+      if (backups.length === 0) {
+        this.triggerAlert('CRITICAL_RPO_BREACH', 'No backups found. RPO is severely breached.');
+        return false;
+      }
+
+      const mostRecent = backups[0];
+      const verification = await disasterRecoveryService.verifyBackup(mostRecent.id);
+
+      if (!verification.valid) {
+        this.triggerAlert('BACKUP_CORRUPTION', `Most recent backup (${mostRecent.id}) is corrupted: ${verification.errors.join(', ')}`);
+        return false;
+      }
+
+      const ageMs = Date.now() - mostRecent.createdAt;
+      if (ageMs > RPO_SECONDS * 1000) {
+        this.triggerAlert('RPO_BREACH', `Most recent backup is ${Math.round(ageMs / 1000)}s old, exceeding RPO of ${RPO_SECONDS}s.`);
+        return false;
+      }
+
+      logger.info(`DR Health Check passed. Backup age: ${Math.round(ageMs / 1000)}s.`);
+      return true;
+    } catch (error) {
+      logger.error('Failed to run DR health check', { error });
+      this.triggerAlert('MONITORING_FAILURE', 'Failed to run DR health check.');
+      return false;
+    }
+  }
+
+  private triggerAlert(type: string, message: string) {
+    // Integrate with log-based alerting
+    logger.error(`[DR ALERT: ${type}] ${message}`, { alertType: 'DR' });
+  }
+}
+
+export const drMonitor = new DisasterRecoveryMonitor();

docs/DISASTER_RECOVERY_RUNBOOK.md

@@ -50,6 +50,19 @@ Up to **5 backups** are retained; older ones are pruned automatically.
 
 ---
 
+## DR Monitoring & Alerts
+
+We employ an automated monitoring service to continuously check the integrity and age of our backups. This ensures we never quietly drift past our RPO targets.
+
+```ts
+import { drMonitor } from '../backend/dr/drMonitoring';
+drMonitor.start(); // Runs every 5 minutes by default
+```
+
+The monitor logs alerts such as `CRITICAL_RPO_BREACH` and `BACKUP_CORRUPTION` directly into our structured logging system.
+
+---
+
 ## Backup Verification
 
 Run after every backup to confirm integrity:
@@ -72,6 +85,15 @@ Verification checks:
 
 ## Failover Procedure
 
+### Automated CLI Failover (Emergency)
+
+In emergency scenarios where you need to trigger a failover instantly from your local environment or CI:
+
+```bash
+node scripts/dr-failover.js
+```
+This script will automatically iterate through the backups, find the most recent valid one, restore it, and report on the RTO metrics.
+
 ### Automatic failover (data corruption / app crash)
 
 ```ts
@@ -158,7 +180,13 @@ const result = await disasterRecoveryService.restoreBackup(backups[0].id);
 
 ## Regular DR Testing
 
-Run the built-in drill on every CI pipeline and before each release:
+You can run the full automated DR testing framework (which creates a backup, verifies it, restores it, and checks RTO/RPO compliance) using the provided script:
+
+```bash
+node scripts/dr-test.js
+```
+
+Alternatively, run the built-in drill on every CI pipeline and before each release:
 
 ```ts
 const drill = await disasterRecoveryService.runDrDrill();
@@ -188,3 +216,9 @@ The drill:
 | All backups corrupted | Re-sync from Soroban contract; prompt user                           |
 | RTO exceeded in drill | Investigate AsyncStorage performance; consider reducing backup scope |
 | RPO warning on verify | Increase backup frequency (trigger on every state mutation)          |
+
+---
+
+## Post-Mortem Process
+
+After any failover event or significant RTO/RPO breach, a post-mortem must be conducted. See [06-post-mortem.md](./runbooks/06-post-mortem.md) for the required template and process workflow. This blameless analysis helps us identify root causes and improve our systemic resilience.

docs/runbooks/06-post-mortem.md

@@ -0,0 +1,48 @@
+# Disaster Recovery Post-Mortem Process
+
+## Purpose
+A post-mortem is a blameless analysis conducted after any disaster recovery event or automated failover is triggered. The goal is to identify root causes, understand the timeline of events, and improve our RTO, RPO, and systemic resilience.
+
+## When to write a post-mortem
+- Anytime `scripts/dr-failover.js` is executed (manually or automatically).
+- When a `CRITICAL_RPO_BREACH` or `BACKUP_CORRUPTION` alert is fired by `drMonitoring.ts`.
+- Any outage lasting longer than the 5-minute RTO target.
+
+## Template
+
+### 1. Incident Summary
+* **Date & Time of Incident**: (UTC)
+* **Duration**: (Time to detect + Time to resolve)
+* **Severity**: (Low / Medium / High / Critical)
+* **Lead Responder**: (Name / Team)
+
+### 2. Timeline
+*Provide a chronological breakdown of events. Include log timestamps and alerts.*
+* `10:00 UTC` - App crash spike detected in monitoring.
+* `10:02 UTC` - `drMonitoring.ts` alerted of corrupted state.
+* `10:05 UTC` - `dr-failover.js` was executed manually by the on-call engineer.
+* `10:06 UTC` - Services restored from the last known good backup.
+
+### 3. Root Cause Analysis (The 5 Whys)
+1. **Why did the system fail?** (e.g., A malformed state object was persisted to AsyncStorage.)
+2. **Why was the malformed object persisted?** (e.g., The schema migration script failed to account for legacy accounts.)
+3. **Why wasn't this caught in testing?** (e.g., Missing unit tests for legacy account edge cases.)
+4. **Why?** ...
+5. **Why?** ...
+
+### 4. Resolution and Recovery
+* **How was it resolved?** (Detail the failover procedure)
+* **Was the RTO met?** (Did recovery complete within 5 minutes?)
+* **Was the RPO met?** (Was data loss limited to the 1-hour window?)
+
+### 5. Action Items
+*List preventative measures to ensure this exact incident doesn't happen again.*
+* [ ] Fix the schema migration script. (Owner: @dev)
+* [ ] Add legacy account edge case unit tests. (Owner: @dev)
+* [ ] Adjust `drMonitoring.ts` thresholds if RTO was exceeded.
+
+## Process Workflow
+1. **Within 24 hours** of the incident, the lead responder drafts the post-mortem using this template.
+2. **Within 48 hours**, the engineering team reviews the document in a blameless post-mortem meeting.
+3. Action items are assigned and added to the sprint backlog.
+4. The finalized post-mortem is stored in the company's internal wiki or incident repository.

scripts/dr-failover.js

@@ -0,0 +1,28 @@
+const { disasterRecoveryService } = require('../backend/dr/DisasterRecoveryService');
+
+async function triggerFailover() {
+  console.log('🚨 EMERGENCY: Triggering Automated Disaster Recovery Failover 🚨');
+
+  console.log('\nInitiating failover sequence...');
+  const start = Date.now();
+
+  const result = await disasterRecoveryService.failover();
+
+  const elapsed = Date.now() - start;
+
+  if (result.success) {
+    console.log(`\n✅ Failover successful in ${elapsed}ms.`);
+    console.log(`Restored ${result.restoredKeys.length} storage keys:`, result.restoredKeys);
+    process.exit(0);
+  } else {
+    console.error(`\n❌ Failover failed after ${elapsed}ms.`);
+    console.error('Errors encountered:', result.errors);
+    console.error('\nPlease escalate to the incident response team immediately and consult docs/runbooks/02-incident-response.md');
+    process.exit(1);
+  }
+}
+
+triggerFailover().catch((e) => {
+  console.error('Fatal error during failover execution:', e);
+  process.exit(1);
+});

scripts/dr-test.js

@@ -0,0 +1,37 @@
+const { disasterRecoveryService } = require('../backend/dr/DisasterRecoveryService');
+const { drMonitor } = require('../backend/dr/drMonitoring');
+
+async function runTest() {
+  console.log('--- Starting Disaster Recovery Automated Drill ---');
+
+  // 1. Run the DR Drill
+  console.log('\nRunning core DR drill (Backup -> Verify -> Restore)...');
+  const drillResult = await disasterRecoveryService.runDrDrill();
+
+  console.log('Drill passed:', drillResult.passed);
+  console.log('Backup ID:', drillResult.backupId);
+  console.log('RTO Compliant:', drillResult.rtoCompliant, `(${drillResult.recovery.durationMs}ms)`);
+
+  if (!drillResult.passed) {
+    console.error('DR Drill failed details:', JSON.stringify(drillResult, null, 2));
+    process.exit(1);
+  }
+
+  // 2. Test Monitoring logic
+  console.log('\nRunning DR Health Monitor Check...');
+  const isHealthy = await drMonitor.checkHealth();
+  console.log('DR Health check passed:', isHealthy);
+
+  if (!isHealthy) {
+    console.error('DR Monitor health check failed. Review logs for RPO or integrity breaches.');
+    process.exit(1);
+  }
+
+  console.log('\n✅ Disaster Recovery Automation Test Complete.');
+  process.exit(0);
+}
+
+runTest().catch((e) => {
+  console.error('Fatal error during DR testing:', e);
+  process.exit(1);
+});