offload data loss fixes

lib/compute/derivation_engine.dart

@@ -697,7 +697,21 @@ import 'substrate.dart';
 //   floor, and now bills through the same per-sample gate, resting floor and
 //   gap cap as the re-score. Already-stored sessions are left alone — they are
 //   not re-derived — so the change applies from this version forward.
-const int kAlgoVersion = 62;
+// v63 - RR BEATS ARE FETCHED BY rec_ts, NOT BY COUNTER SPAN.
+//   The derivation pages 1 Hz frames ordered by rec_ts and then pulled that
+//   page's RR beats with `decodedRrByCounterRange(first.counter, last.counter)`.
+//   The strap's counter resets on every reboot, so the moment a page straddled
+//   one the span was inverted or nonsensical and the query returned nothing:
+//   the page decoded with an EMPTY beat list, and every beat-derived figure for
+//   that stretch — RMSSD, SDNN, the HRV curve, and the readiness that leans on
+//   them — silently came back absent or computed off whatever beats survived on
+//   the other pages. Both tables are keyed by rec_ts now, so the lookup uses the
+//   page's own rec_ts bounds and pulls exactly its beats.
+//
+//   Days already finalized at v62 hold those RR-less results permanently — they
+//   are never revisited at the same version — so this needs the bump to be
+//   re-derived onto real beats.
+const int kAlgoVersion = 63;
 
 // Fold idempotency, the minimum-nights warm-up, and legacy-payload handling
 // all live in SleepProfilePolicy (pure, unit-tested) — see
@@ -1836,13 +1850,16 @@ class DerivationEngine {
             rangePages: rangePages,
             rangeRows: rangeRows,
           );
-          final firstCounter = (decodedRows.first['counter'] as num?)?.toInt();
-          final lastCounter = (decodedRows.last['counter'] as num?)?.toInt();
-          final rrRows = firstCounter == null || lastCounter == null
+          // The page is ordered rec_ts ASC, so first = min second, last = max.
+          // decoded_rr shares the rec_ts key, so this pulls exactly the page's
+          // beats — no counter span (which broke across the strap's reboot reset).
+          final firstRecTs = (decodedRows.first['rec_ts'] as num?)?.toInt();
+          final lastRecTs = (decodedRows.last['rec_ts'] as num?)?.toInt();
+          final rrRows = firstRecTs == null || lastRecTs == null
               ? const <Map<String, dynamic>>[]
-              : await LocalDb.decodedRrByCounterRange(
-                  fromCounter: firstCounter,
-                  toCounter: lastCounter,
+              : await LocalDb.decodedRrByRecTsRange(
+                  fromRecTs: firstRecTs,
+                  toRecTs: lastRecTs,
                 );
           worker.send({'type': 'page', 'frames': decodedRows, 'rr': rrRows});
           final last = decodedRows.last;

lib/compute/derive_prepare.dart

@@ -431,11 +431,14 @@ class _PrepareAccumulator {
     List<Map<String, dynamic>> rrRows,
   ) {
     if (frames.isEmpty) return;
-    final rrByCounter = <int, List<Map<String, dynamic>>>{};
+    // Associate beats to frames by rec_ts (their shared key). The strap's counter
+    // resets on reboot, so grouping by counter mis-joined two seconds that reused
+    // one counter within a page.
+    final rrByRecTs = <int, List<Map<String, dynamic>>>{};
     for (final row in rrRows) {
-      final counter = _num(row['counter'])?.toInt();
-      if (counter == null) continue;
-      rrByCounter.putIfAbsent(counter, () => <Map<String, dynamic>>[]).add(row);
+      final recTs = _num(row['rec_ts'])?.toInt();
+      if (recTs == null) continue;
+      rrByRecTs.putIfAbsent(recTs, () => <Map<String, dynamic>>[]).add(row);
     }
     for (final row in frames) {
       final recTs = _num(row['rec_ts'])?.toInt();
@@ -454,9 +457,7 @@ class _PrepareAccumulator {
       // tsSec is what lets `Substrate.fromJson` tell "absent" (empty ⇒
       // zero-filled) from "present but zero".
       skinContact.add(_num(row['skin_contact'])?.toInt() ?? 0);
-      final counter = _num(row['counter'])?.toInt();
-      if (counter == null) continue;
-      final beats = rrByCounter[counter];
+      final beats = rrByRecTs[recTs];
       if (beats == null) continue;
       for (final beat in beats) {
         final rr = _num(beat['rr_ms'])?.toDouble();

lib/sync/sync_policy.dart

@@ -137,6 +137,46 @@ class ClockPolicy {
     return drift > 86400 || deviceClock < kMinPlausibleUnix;
   }
 
+  /// True when the strap RTC reads a PLAUSIBLE absolute time but sits more than
+  /// [kFutureMargin] in the FUTURE relative to the phone — the signature of a
+  /// PHONE clock running slow (dead-battery reboot, bad NTP, a manual set-back).
+  ///
+  /// This is the one clock-disagreement we must NOT act on destructively. We
+  /// cannot prove which clock is right, but both wrong moves are unsafe:
+  ///   - draining now drops the strap's (correctly-stamped, real-now) records as
+  ///     "implausibly future", and a mixed-burst ACK then TRIMS them off the
+  ///     band — permanent, silent loss; and
+  ///   - SET_CLOCK-ing the strap backward to match the slow phone would corrupt
+  ///     a correct RTC.
+  /// So the caller DEFERS history offload until the clocks agree (the phone
+  /// clock almost always self-corrects via NTP within minutes; the strap keeps
+  /// every record until then). The `>= kMinPlausibleUnix` guard excludes an
+  /// unset/garbage-low RTC (that is a strap problem [shouldSetClock] fixes, not
+  /// a phone problem); the strap-BEHIND case is a plausible-past time that is
+  /// not dropped as future and is corrected forward by [shouldSetClock].
+  /// How long a suspect-clock disagreement may defer history before we stop
+  /// believing the phone is the wrong one. A phone that rebooted with a dead
+  /// battery re-syncs over NTP within minutes, so a disagreement that survives
+  /// this long is a strap RTC that is genuinely running fast — not a slow
+  /// phone. Past this the gate stops deferring and the strap clock is corrected
+  /// normally, so a bad strap RTC cannot stall history forever.
+  static const int suspectGraceSeconds = 12 * 3600;
+
+  /// True once a suspect-clock state has persisted past [suspectGraceSeconds].
+  ///
+  /// Both arguments are MONOTONIC seconds (a `Stopwatch`), never wall clock.
+  /// The state being timed is "we do not trust `DateTime.now()`", so timing it
+  /// with `DateTime.now()` is self-defeating: a phone that steps forward a day
+  /// over NTP — while possibly still more than a day behind the strap — would
+  /// instantly age the suspicion past the grace window and re-authorize the
+  /// drain-and-trim this gate exists to hold back.
+  static bool suspectGraceExpired(double? sinceSecs, double nowSecs) =>
+      sinceSecs != null && nowSecs - sinceSecs >= suspectGraceSeconds;
+
+  static bool phoneClockSuspect(int deviceClock, int wallNow) =>
+      deviceClock >= kMinPlausibleUnix &&
+      deviceClock > wallNow + kFutureMargin;
+
   /// Salvage an implausible record time using the strap↔wall clock offset
   /// (device→wall = [clockWall] - [deviceClock]). A wandering/unset RTC offsets
   /// EVERY record in a session by the same amount, so shifting by that offset

pubspec.lock

@@ -892,10 +892,10 @@ packages:
     dependency: transitive
     description:
       name: meta
-      sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394"
+      sha256: "1741988757a65eb6b36abe716829688cf01910bbf91c34354ff7ec1c3de2b349"
       url: "https://pub.dev"
     source: hosted
-    version: "1.17.0"
+    version: "1.18.0"
   mgrs_dart:
     dependency: transitive
     description:
@@ -1328,7 +1328,7 @@ packages:
     source: hosted
     version: "2.4.2+3"
   sqflite_common:
-    dependency: transitive
+    dependency: "direct dev"
     description:
       name: sqflite_common
       sha256: "1581ffbf7a0e333b380d6a30737d78516b826cb35beb7fb0bf8a3ea0c678b465"
@@ -1411,26 +1411,26 @@ packages:
     dependency: "direct dev"
     description:
       name: test
-      sha256: "280d6d890011ca966ad08df7e8a4ddfab0fb3aa49f96ed6de56e3521347a9ae7"
+      sha256: "8d9ceddbab833f180fbefed08afa76d7c03513dfdba87ffcec2718b02bbcbf20"
       url: "https://pub.dev"
     source: hosted
-    version: "1.30.0"
+    version: "1.31.0"
   test_api:
     dependency: transitive
     description:
       name: test_api
-      sha256: "8161c84903fd860b26bfdefb7963b3f0b68fee7adea0f59ef805ecca346f0c7a"
+      sha256: "949a932224383300f01be9221c39180316445ecb8e7547f70a41a35bf421fb9e"
       url: "https://pub.dev"
     source: hosted
-    version: "0.7.10"
+    version: "0.7.11"
   test_core:
     dependency: transitive
     description:
       name: test_core
-      sha256: "0381bd1585d1a924763c308100f2138205252fb90c9d4eeaf28489ee65ccde51"
+      sha256: "1991d4cfe85d5043241acac92962c3977c8d2f2add1ee73130c7b286417d1d34"
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.16"
+    version: "0.6.17"
   timezone:
     dependency: "direct main"
     description:

pubspec.yaml

@@ -48,6 +48,15 @@ dependencies:
       # activity/steps_inc null-instead-of-0 fix. Edge does not read the latter
       # two fields, so only the RR bound is behaviour-visible here.
       # Verified present: `git show <sha>:lib/src/live.dart | grep -c kMinRrMs`.
+      # DELIBERATELY NOT MOVED for the clock-gate work in this PR. Correlating
+      # a GET_CLOCK reply to its own request needs the echoed request seq
+      # surfaced (OpenStrap/protocol#28), and the only in-convention way to pin
+      # that is a merge commit on protocol main — which now also carries the
+      # gen5/multiband surface (gen5_records.dart, the frame-revision changes
+      # in framing.dart, new dangerousCmds entries). Adopting ~1850 lines of
+      # that as a side effect of a one-field addition is the pin drift that
+      # shipped the v42 ANRs. _readClock degrades to accepting any fresh
+      # clock_epoch, which is what it did before the correlation existed.
       ref: 7edcb3e377329968118c62cb03a81d95e2f6db8e
   openstrap_analytics:
     git:
@@ -260,6 +269,13 @@ dev_dependencies:
   # exportDaysDb can run without a platform plugin (transitive via
   # path_provider; declared directly since test/ now imports it).
   path_provider_platform_interface: ^2.1.0
+  # ack_commit_sync_full_test wraps the ffi factory in SqfliteDatabaseFactoryLogger
+  # to spy the PRAGMA synchronous=FULL/NORMAL bracket around the ACK-gating commit
+  # (transitive via sqflite_common_ffi; declared directly since test/ now imports it).
+  # PINNED, not a caret range: that constructor is @experimental upstream, so a
+  # minor bump is allowed to change or withdraw it and would break the test on
+  # someone else's `pub upgrade` rather than on a deliberate one here.
+  sqflite_common: 2.5.8
 
 flutter_launcher_icons:
   android: "launcher_icon"

test/ack_commit_sync_full_test.dart

@@ -0,0 +1,98 @@
+// POWER-LOSS DURABILITY of the ACK-gating commit. `commitSyncBatch` is the one
+// commit the safe-trim invariant hangs on: it must be durable (fsynced) BEFORE
+// the caller writes the BLE batch-ACK that lets the band trim its flash. The DB
+// otherwise runs WAL + synchronous=NORMAL (durable only at a checkpoint), so
+// this path raises synchronous=FULL for its single transaction and restores
+// NORMAL afterward — leaving every other (recomputable) path fast. This test
+// pins that bracket: FULL is set around the commit, NORMAL is restored after,
+// AND the restore still happens when the commit THROWS (a leaked FULL would
+// fsync every subsequent write on the connection forever).
+//
+// We spy the real SQL stream via SqfliteDatabaseFactoryLogger (synchronous is
+// per-connection and invisible from a second connection, so the log is the only
+// honest observation point) and also read `PRAGMA synchronous` on LocalDb's own
+// connection — the same one commitSyncBatch uses — to confirm the resting value.
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:path/path.dart' as p;
+import 'package:sqflite_common/sqflite_logger.dart';
+import 'package:sqflite_common_ffi/sqflite_ffi.dart';
+import 'package:openstrap_edge/data/db.dart';
+import 'package:openstrap_edge/data/models.dart';
+
+void main() {
+  // Every `synchronous=…` statement executed on any connection, in order.
+  final syncStmts = <String>[];
+
+  setUpAll(() async {
+    sqfliteFfiInit();
+    // ignore: experimental_member_use — stable enough to spy SQL in a test.
+    databaseFactory = SqfliteDatabaseFactoryLogger(
+      databaseFactoryFfi,
+      options: SqfliteLoggerOptions(
+        log: (event) {
+          if (event is SqfliteLoggerSqlEvent) {
+            final sql = event.sql.toLowerCase();
+            if (sql.contains('pragma synchronous=')) syncStmts.add(sql);
+          }
+        },
+      ),
+    );
+    LocalDb.dbName = 'openstrap_ack_sync_full_test.db';
+    final dir = await databaseFactory.getDatabasesPath();
+    await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName));
+    // Force the open now so its onConfigure PRAGMAs aren't counted in per-test
+    // windows — each test clears syncStmts against an already-open connection.
+    await LocalDb.instance;
+  });
+
+  tearDownAll(() async {
+    await LocalDb.close();
+    final dir = await databaseFactory.getDatabasesPath();
+    await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName));
+  });
+
+  Future<int> restingSynchronous() async {
+    final db = await LocalDb.instance;
+    final rows = await db.rawQuery('PRAGMA synchronous');
+    return rows.first.values.first as int; // FULL=2, NORMAL=1
+  }
+
+  RawRecord recAt(int counter) => RawRecord(
+        counter: counter,
+        packetType: 0x2F,
+        hex: '2f18aabbccdd',
+        capturedAt: 1750000000000 + counter,
+        recTs: 1750000000 + counter,
+      );
+
+  test('commitSyncBatch brackets synchronous=FULL and restores NORMAL', () async {
+    expect(await restingSynchronous(), 1, reason: 'connection opens at NORMAL');
+
+    syncStmts.clear();
+    await LocalDb.commitSyncBatch(
+      [recAt(5001)],
+      <Sample?>[Sample(tsEpoch: 1750005001, counter: 5001, hr: 60)],
+      trimToken: 'deadbeef',
+    );
+
+    expect(syncStmts, ['pragma synchronous=full', 'pragma synchronous=normal'],
+        reason: 'FULL is set before the commit and NORMAL restored right after');
+    expect(await restingSynchronous(), 1, reason: 'connection left at NORMAL');
+  });
+
+  test('synchronous is restored to NORMAL even when the commit throws', () async {
+    syncStmts.clear();
+    // raws non-empty but samples empty → samples[i] throws RangeError INSIDE the
+    // db.transaction, after FULL is set. The finally must still restore NORMAL.
+    await expectLater(
+      LocalDb.commitSyncBatch([recAt(6001)], const <Sample?>[]),
+      throwsA(isA<RangeError>()),
+    );
+
+    expect(syncStmts, ['pragma synchronous=full', 'pragma synchronous=normal'],
+        reason: 'a thrown commit must not leak FULL');
+    expect(await restingSynchronous(), 1,
+        reason: 'FULL did not leak past the throwing commit');
+  });
+}