From d2ccec4d3580d4f2caea56ed634d21c0f91260c8 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:37:16 +0530 Subject: [PATCH 1/2] feat(sync): log-only burst-completeness shortfall diagnostic (correct received-total signal) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Emit an honest, observation-only frame-loss signal at HISTORY_END without touching the commit/ACK decision. The band's num_packets counts every frame it transmitted (all types); the correct completeness comparison is against totalTrafficPacketCount — the all-types received total — not the banked R24 subset (which fabricates a shortfall whenever console/event frames ride along un-banked). This is type-agnostic and interleaving-immune. New pure helper burstPacketShortfall() = expected - (received_all_types + dropped_this_burst): a POSITIVE result is frames the band sent that never reached us (true loss); zero is complete; negative is retries/dupes, not loss. Gate-dropped (RecordGate) records are added back so plausibility rejections never read as radio loss. At burst end we now log a "would-flag" line and stamp burst_shortfall into the existing mismatch ledger entry — LOG-ONLY. Commit-before-ACK, the verbatim token echo, and the OK/FAIL decision are all unchanged. This is groundwork so we can SEE true frame loss in telemetry before ever wiring a field-validated FAIL gate; a hard FAIL/re-flood path is deliberately NOT included here. Rejected alternative: gating on the per-revision counter gap — the counter is a GLOBAL flash-log index sliced per revision, so gaps are the normal state and would false-positive constantly. Adds pure unit tests covering benign interleaving (no false positive), true loss, the gate-dropped add-back, negative/retry case, and shortfall==0 == burstPacketCountMatches. --- lib/ble/ble_engine.dart | 53 ++++++++++++++++++++++ test/ble_engine_test.dart | 95 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index f6184d9..81a7fbd 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -155,6 +155,31 @@ bool burstPacketCountMatches({ }) => expectedPacketCount == actualBurstPacketCount + droppedThisBurst; +/// Honest burst-completeness signal for TELEMETRY ONLY — this NEVER gates the +/// commit/ACK decision (see the log-only call site). +/// +/// [receivedTrafficCount] is every frame we actually received this burst, ALL +/// types (historical R24 data + interleaved console/event/unknown) — i.e. +/// [BurstStats.totalTrafficPacketCount], NOT the banked historical subset. The +/// band's [expectedPacketCount] (num_packets) likewise counts every frame it +/// transmitted, so comparing the two all-types totals is type-agnostic and +/// interleaving-immune: benign console/event frames riding along cannot fake a +/// shortfall the way comparing against the R24-only subset did. +/// +/// [droppedThisBurst] (RecordGate plausibility rejections this burst) is added +/// back because the band counted those frames but they never entered +/// [receivedTrafficCount] — so subtracting them isolates frames the band sent +/// that NEVER reached us at all. A POSITIVE result is that true frame loss +/// (would-flag); zero is complete; negative just means we tallied more than +/// expected (retried/duplicate frames), which is not loss. +@visibleForTesting +int burstPacketShortfall({ + required int expectedPacketCount, + required int receivedTrafficCount, + int droppedThisBurst = 0, +}) => + expectedPacketCount - (receivedTrafficCount + droppedThisBurst); + /// Fired for every LIVE high-rate frame (0x28/0x2B/0x33). These are EPHEMERAL — /// they are NOT persisted to raw_records (that bloated storage ~50x and stalled /// derivation). The caller routes them to an in-memory sink for the live UI / @@ -2596,6 +2621,19 @@ class BleEngine { expectedPacketCount: expected, droppedThisBurst: droppedThisBurst, ); + // Honest, LOG-ONLY completeness signal (never gates the ACK). Compares + // num_packets against the ALL-TYPES received total (currentBurstTrafficCount), + // not the banked R24 subset — see burstPacketShortfall. Only a POSITIVE + // shortfall means frames the band sent never reached us (true loss); this + // is the signal we want visible in telemetry BEFORE ever wiring a FAIL + // gate (which needs its own design + field validation to avoid re-flood). + final shortfall = expected == null + ? 0 + : burstPacketShortfall( + expectedPacketCount: expected, + receivedTrafficCount: d.currentBurstTrafficCount, + droppedThisBurst: droppedThisBurst, + ); // ADVISORY ONLY, never a gate: `expectedPacketCount`'s exact semantics // (which transport packet types the band itself counts — command // responses interleaved with the burst? retried/duplicate frames?) are @@ -2635,11 +2673,26 @@ class BleEngine { 'traffic_burst_packets': d.currentBurstTrafficCount, 'burst_validation_failures': d.consecutiveValidationFailures, 'burst_breakdown': d.currentBurstBreakdown, + 'burst_shortfall': shortfall, }, )); } else { _burstMismatchStreak = 0; } + // Would-flag: the correct-signal completeness diagnostic. LOG-ONLY — the + // commit + verbatim-token ACK below are unchanged. A positive shortfall + // is the honest "true frame loss" telemetry we want to watch before a + // later, field-validated FAIL gate ever acts on it. + if (shortfall > 0) { + _log( + '[SYNC] burst completeness would-flag (LOG-ONLY, commit+ACK ' + 'unchanged): expected=$expected ' + 'received=${d.currentBurstTrafficCount} ' + 'dropped_this_burst=$droppedThisBurst shortfall=$shortfall ' + '(all-types received total — true frame loss; groundwork for a ' + 'future FAIL gate, NOT gating today)', + ); + } final r = d.bufferedRecTsRange; final droppedThisBurstForLog = droppedThisBurst; final hadDurableRows = diff --git a/test/ble_engine_test.dart b/test/ble_engine_test.dart index 6acce80..af99175 100644 --- a/test/ble_engine_test.dart +++ b/test/ble_engine_test.dart @@ -122,6 +122,101 @@ void main() { }); }); + group('burst completeness shortfall (log-only would-flag signal)', () { + test('no shortfall when all-types received total equals num_packets', () { + // Band sent 49 frames (30 R24 + 17 console + 2 event); we received all. + final received = countBurstTrafficPackets( + dataPacketCountsByRevision: const {24: 30}, + consoleCount: 17, + eventCount: 2, + ); + expect( + burstPacketShortfall( + expectedPacketCount: 49, + receivedTrafficCount: received, + ), + 0, + ); + }); + + test( + 'interleaved console/event frames do NOT false-positive: comparing ' + 'against the all-types received total (not the banked R24 subset) ' + 'keeps shortfall at zero', + () { + final received = countBurstTrafficPackets( + dataPacketCountsByRevision: const {24: 15}, + consoleCount: 37, + eventCount: 2, + ); + // Banked R24 subset alone is 15 — comparing THAT to num_packets=54 + // would fabricate a 39-frame "loss". The correct all-types total is 54. + expect(received, 54); + expect( + burstPacketShortfall( + expectedPacketCount: 54, + receivedTrafficCount: received, + ), + 0, + ); + }, + ); + + test('positive shortfall flags true frame loss (band sent more than we got)', + () { + final received = countBurstTrafficPackets( + dataPacketCountsByRevision: const {24: 20}, + consoleCount: 3, + ); + // Band reported 30, we received 23 all-types, nothing gate-dropped → 7 lost. + expect( + burstPacketShortfall( + expectedPacketCount: 30, + receivedTrafficCount: received, + ), + 7, + ); + }); + + test('gate-dropped records are added back so they never read as loss', () { + // 26 all-types received, 24 legitimately gate-dropped, band expected 50 → + // fully explained, no true loss. + expect( + burstPacketShortfall( + expectedPacketCount: 50, + receivedTrafficCount: 26, + droppedThisBurst: 24, + ), + 0, + ); + }); + + test('negative shortfall (retries/dupes counted extra) is not loss', () { + expect( + burstPacketShortfall( + expectedPacketCount: 26, + receivedTrafficCount: 28, + ), + lessThan(0), + ); + }); + + test('shortfall==0 is exactly burstPacketCountMatches', () { + const expected = 50, received = 26, dropped = 24; + final matches = burstPacketCountMatches( + expectedPacketCount: expected, + actualBurstPacketCount: received, + droppedThisBurst: dropped, + ); + final shortfall = burstPacketShortfall( + expectedPacketCount: expected, + receivedTrafficCount: received, + droppedThisBurst: dropped, + ); + expect(matches, (shortfall == 0)); + }); + }); + group('maintenance traffic gating', () { test('maintenance traffic is paused while offload is active', () { expect(shouldPauseMaintenanceTraffic(offloadActive: true), isTrue); From a974918dfd97e77ec8dd4c8421b28600187c7180 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:43:58 +0530 Subject: [PATCH 2/2] docs(sync): don't overclaim shortfall as confirmed frame loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A positive burst shortfall means frames the band counted that we did not count as valid received traffic. CRC-failed frames also never enter currentBurstTrafficCount, so a positive shortfall can be missing OR corrupted traffic — it cannot by itself prove a frame never arrived. Soften the helper doc, the would-flag log text, and the test name accordingly. Wording-only; no behavior change (still log-only). --- lib/ble/ble_engine.dart | 24 ++++++++++++++---------- test/ble_engine_test.dart | 8 +++++--- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index 81a7fbd..59c9917 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -168,10 +168,12 @@ bool burstPacketCountMatches({ /// /// [droppedThisBurst] (RecordGate plausibility rejections this burst) is added /// back because the band counted those frames but they never entered -/// [receivedTrafficCount] — so subtracting them isolates frames the band sent -/// that NEVER reached us at all. A POSITIVE result is that true frame loss -/// (would-flag); zero is complete; negative just means we tallied more than -/// expected (retried/duplicate frames), which is not loss. +/// [receivedTrafficCount]. A POSITIVE result is frames the band counted that we +/// did NOT count as valid received traffic — i.e. missing OR corrupted traffic +/// (would-flag / potential loss): CRC-failed frames also never enter +/// [receivedTrafficCount], so a positive shortfall cannot by itself prove a +/// frame never arrived. Zero is complete; negative just means we tallied more +/// than expected (retried/duplicate frames), which is not loss. @visibleForTesting int burstPacketShortfall({ required int expectedPacketCount, @@ -2624,9 +2626,10 @@ class BleEngine { // Honest, LOG-ONLY completeness signal (never gates the ACK). Compares // num_packets against the ALL-TYPES received total (currentBurstTrafficCount), // not the banked R24 subset — see burstPacketShortfall. Only a POSITIVE - // shortfall means frames the band sent never reached us (true loss); this - // is the signal we want visible in telemetry BEFORE ever wiring a FAIL - // gate (which needs its own design + field validation to avoid re-flood). + // shortfall means frames the band counted that we did not count as valid + // received traffic (missing OR CRC-corrupted — potential loss); this is + // the signal we want visible in telemetry BEFORE ever wiring a FAIL gate + // (which needs its own design + field validation to avoid re-flood). final shortfall = expected == null ? 0 : burstPacketShortfall( @@ -2681,15 +2684,16 @@ class BleEngine { } // Would-flag: the correct-signal completeness diagnostic. LOG-ONLY — the // commit + verbatim-token ACK below are unchanged. A positive shortfall - // is the honest "true frame loss" telemetry we want to watch before a - // later, field-validated FAIL gate ever acts on it. + // is the honest missing/corrupted-traffic telemetry we want to watch + // before a later, field-validated FAIL gate ever acts on it. if (shortfall > 0) { _log( '[SYNC] burst completeness would-flag (LOG-ONLY, commit+ACK ' 'unchanged): expected=$expected ' 'received=${d.currentBurstTrafficCount} ' 'dropped_this_burst=$droppedThisBurst shortfall=$shortfall ' - '(all-types received total — true frame loss; groundwork for a ' + '(all-types received total — frames the band counted that we did ' + 'not; missing or CRC-corrupted, potential loss; groundwork for a ' 'future FAIL gate, NOT gating today)', ); } diff --git a/test/ble_engine_test.dart b/test/ble_engine_test.dart index af99175..ba961d7 100644 --- a/test/ble_engine_test.dart +++ b/test/ble_engine_test.dart @@ -162,13 +162,15 @@ void main() { }, ); - test('positive shortfall flags true frame loss (band sent more than we got)', - () { + test( + 'positive shortfall flags missing-or-corrupted traffic (band counted ' + 'more than we did)', () { final received = countBurstTrafficPackets( dataPacketCountsByRevision: const {24: 20}, consoleCount: 3, ); - // Band reported 30, we received 23 all-types, nothing gate-dropped → 7 lost. + // Band reported 30, we counted 23 all-types, nothing gate-dropped → 7 + // frames the band sent that we did not count (never arrived or CRC-failed). expect( burstPacketShortfall( expectedPacketCount: 30,