feat(reliability): retry, circuit breaker, graceful degradation, single-flight (LAB-518)

README.md

@@ -39,6 +39,7 @@
 | `cachekitio` | ✅ | HTTP backend for [api.cachekit.io](https://api.cachekit.io) via [reqwest](https://crates.io/crates/reqwest) + rustls |
 | `encryption` | ✅ | Zero-knowledge AES-256-GCM via [cachekit-core](https://crates.io/crates/cachekit-core) |
 | `l1` | ✅ | In-process L1 cache via [moka](https://crates.io/crates/moka) |
+| `reliability` | ✅ | Retry with backoff + jitter, circuit breaker, distributed fill locks (native only) |
 | `redis` | ❌ | Redis backend via [fred](https://crates.io/crates/fred) (native only) |
 | `memcached` | ❌ | Memcached backend via [rust-memcache](https://crates.io/crates/memcache) (native only) |
 | `file` | ❌ | Local filesystem backend, byte-compatible with cachekit-py's File backend (native only) |
@@ -63,6 +64,7 @@ cachekit-rs = { version = "0.2", default-features = false, features = ["workers"
 > **Mutually exclusive features:**
 > - `workers` + `redis` — Workers runtime cannot use fred
 > - `workers` + `l1` — moka requires std threads unavailable in wasm32
+> - `workers` + `reliability` — retry/breaker timers need tokio `time`, unavailable in wasm32
 > - `workers` + `memcached` — Workers runtime has no TCP sockets
 > - `workers` + `file` — Workers runtime has no filesystem
 
@@ -324,6 +326,41 @@ When the `l1` feature is enabled (default), CacheKit maintains an in-process [mo
 
 ---
 
+## Reliability
+
+With the `reliability` feature (default, native only), the `production`, `encrypted`, and `io` presets wrap every backend operation in a reliability stack; `minimal` stays bare for maximum throughput:
+
+| Layer | What it does | Defaults |
+|:------|:-------------|:---------|
+| **Retry** | Truncated exponential backoff + jitter on transient/timeout errors (`BackendErrorKind::is_retryable`); permanent and auth errors propagate immediately | 3 attempts, 100 ms base, 5 s cap, jitter ×[0.5, 1.5) |
+| **Circuit breaker** | closed → open after N retryable failures in a rolling window; fails fast (`BackendErrorKind::CircuitOpen`) while open; half-open probes recovery | threshold 5, window 60 s, open 5 s, 3 probes, close after 3 successes |
+| **Graceful degradation** | On outage-class backend failure (transient, timeout, open breaker), `#[cachekit]`-wrapped functions run uncached (fail-open); permanent/auth errors propagate — a wrong API key fails loudly. `secure` paths fail **closed** on everything — encrypted workloads never silently degrade | built into the macro |
+| **Single-flight** | Concurrent misses of one key collapse to a single execution: per-key in-process lock, plus a distributed fill lock across processes on lock-capable backends (cachekit.io, Redis) | in-process always on; cross-process 5 s lock, 100 ms polls |
+
+Retry sits *inside* the breaker (one exhausted retry sequence = one breaker failure), degradation and single-flight sit in the `#[cachekit]` macro around the miss path — the same composition as the TypeScript SDK's `ReliabilityExecutor` and the Python decorator.
+
+```rust,ignore
+use std::time::Duration;
+use cachekit::{CacheKit, ReliabilityConfig, RetryConfig};
+
+// Presets enable it — override or disable per client:
+let cache = CacheKit::production("redis://localhost:6379").await?
+    .reliability(ReliabilityConfig {
+        retry: Some(RetryConfig { max_attempts: 5, ..RetryConfig::default() }),
+        ..ReliabilityConfig::default()
+    })
+    .build()?;
+
+// Opt a preset out: a config with both layers `None` applies no wrapping.
+let bare = CacheKit::production("redis://localhost:6379").await?
+    .reliability(ReliabilityConfig { retry: None, circuit_breaker: None })
+    .build()?;
+```
+
+Requires a tokio runtime for backoff timers (the `redis` and `cachekitio` backends already do).
+
+---
+
 ## Environment Variables
 
 | Variable | Required | Description |

crates/cachekit-macros/src/lib.rs

@@ -203,6 +203,22 @@ fn extract_ok_type(ret: &ReturnType) -> syn::Result<Type> {
 /// - A stored entry that cannot be decoded as the return type is treated as
 ///   a miss and overwritten (self-healing), never an error loop.
 ///
+/// # Reliability behaviour
+///
+/// - **Graceful degradation**: on an outage-class backend failure —
+///   transient, timeout, or an open circuit breaker — the plain path fails
+///   *open*: the function executes uncached and its result is returned.
+///   Permanent and authentication errors propagate even on the plain path
+///   (a wrong API key must fail loudly, not silently disable caching
+///   forever). With `secure`, *every* backend and decryption error fails
+///   *closed* and propagates: an encrypted workload never silently
+///   degrades.
+/// - **Cold-miss single-flight**: concurrent calls that miss on the same
+///   key are collapsed to one execution per process (and per fleet, when
+///   the backend supports distributed fill locks — CachekitIO and Redis do,
+///   with the `reliability` feature). Waiters re-read the filled entry
+///   instead of recomputing. See `cachekit::flight`.
+///
 /// # Example
 ///
 /// ```ignore
@@ -297,6 +313,28 @@ fn expand(args: &MacroArgs, mut func: ItemFn) -> syn::Result<TokenStream2> {
         )
     };
 
+    // Graceful degradation (LAB-518): on an OUTAGE-class backend failure —
+    // retryable (transient/timeout) or a fast-failing open circuit breaker —
+    // the plain path fails OPEN: the wrapped function runs uncached, so a
+    // cache outage costs performance, not availability. Permanent and
+    // authentication errors PROPAGATE even on the plain path: a wrong API
+    // key that silently fell open would run uncached forever with zero
+    // signal while looking healthy (expert-panel finding). The `secure`
+    // path stays fail-CLOSED on everything: backend and decryption errors
+    // reach the caller, so an encrypted workload never silently degrades.
+    let fail_open_arm = if args.secure {
+        quote! {}
+    } else {
+        quote! {
+            Err(cachekit::error::CachekitError::Backend(__ck_be))
+                if __ck_be.kind.is_retryable()
+                    || matches!(
+                        __ck_be.kind,
+                        cachekit::error::BackendErrorKind::CircuitOpen
+                    ) => {}
+        }
+    };
+
     // Capture the original function body.
     let original_body = &func.block;
 
@@ -329,15 +367,37 @@ fn expand(args: &MacroArgs, mut func: ItemFn) -> syn::Result<TokenStream2> {
             // cannot be decoded as #ok_type (poisoned, foreign-shaped, or a
             // Python-internal CK frame) — treat it as a miss so the fresh
             // result OVERWRITES it, instead of hard-failing every call until
-            // TTL expiry. All other errors (namespaced-client Config, backend
-            // failures) propagate.
+            // TTL expiry. Backend errors fail open on the plain path (run
+            // the function uncached) and fail closed on the secure path.
+            // Other errors (namespaced-client Config, decryption) propagate.
             match #get_expr {
                 Ok(Some(__ck_cached)) => return Ok(__ck_cached),
                 Ok(None) => {}
                 Err(cachekit::error::CachekitError::Serialization(_)) => {}
+                #fail_open_arm
                 Err(__ck_err) => return Err(__ck_err.into()),
             }
 
+            // Cold-miss single-flight: collapse concurrent fills of this key
+            // to one execution (misses are billable). While another worker is
+            // filling, re-check the cache instead of recomputing.
+            let mut __ck_flight = #client_ident.single_flight(&__ck_key).await;
+            while __ck_flight.wait_for_fill().await {
+                match #get_expr {
+                    Ok(Some(__ck_cached)) => {
+                        __ck_flight.release().await;
+                        return Ok(__ck_cached);
+                    }
+                    Ok(None) => {}
+                    Err(cachekit::error::CachekitError::Serialization(_)) => {}
+                    #fail_open_arm
+                    Err(__ck_err) => {
+                        __ck_flight.release().await;
+                        return Err(__ck_err.into());
+                    }
+                }
+            }
+
             // Execute original function body
             let __ck_result: #ret_ty = (async #original_body).await;
 
@@ -346,6 +406,7 @@ fn expand(args: &MacroArgs, mut func: ItemFn) -> syn::Result<TokenStream2> {
                 #set_expr
             }
 
+            __ck_flight.release().await;
             __ck_result
         }
     };

crates/cachekit/Cargo.toml

@@ -15,23 +15,26 @@ categories = ["caching", "web-programming", "cryptography"]
 name = "cachekit"
 
 [features]
-default = ["cachekitio", "encryption", "l1"]
+default = ["cachekitio", "encryption", "l1", "reliability"]
 cachekitio = ["dep:reqwest", "dep:serde_json"]
 redis = ["dep:fred"]
 # Memcached backend (native only). rust-memcache is sync (r2d2-pooled, socket
 # read/write timeouts); ops run on tokio's blocking pool via spawn_blocking,
 # each under a tokio/time budget so a wedged server can't hang callers.
-memcached = ["dep:memcache", "dep:tokio", "tokio/rt", "tokio/time"]
+memcached = ["dep:memcache", "tokio/rt", "tokio/time"]
 # Local filesystem backend (native only). tokio/rt provides spawn_blocking so
 # disk I/O never blocks the async executor; libc provides O_NOFOLLOW/ELOOP
 # constants and rustix provides safe flock/geteuid on unix.
-file = ["dep:tokio", "tokio/rt", "tokio/sync", "dep:libc", "dep:rustix"]
+file = ["tokio/rt", "tokio/sync", "dep:libc", "dep:rustix"]
 workers = ["dep:worker", "dep:js-sys", "dep:getrandom", "dep:serde_json"]
 # cachekit-core 0.2 folds aes-gcm into the `encryption` feature automatically on wasm32.
 # No separate `cachekit-core/wasm` activation needed.
 encryption = ["cachekit-core/encryption"]
 l1 = ["dep:moka"]
 macros = ["dep:cachekit-macros"]
+# Retry with backoff + circuit breaker around backend ops, and cross-process
+# cold-miss suppression via LockableBackend. Native only (needs tokio timers).
+reliability = ["tokio/time"]
 # Opt into ?Send / Rc on native targets (for tokio::task::LocalSet, single-threaded
 # runtimes, etc.). Same code paths that wasm32 already uses.
 unsync = []
@@ -49,6 +52,11 @@ urlencoding = "2"
 hex = "0.4"
 url = "2"
 uuid = { version = "1", features = ["v4", "js"] }
+# `sync` only: runtime-independent primitives for cold-miss single-flight.
+# Compiles on wasm32. Features stack per consumer: `reliability` adds `time`
+# (native only); the `memcached`/`file` backends add `rt` (spawn_blocking) —
+# see the feature declarations above.
+tokio = { version = "1", default-features = false, features = ["sync"] }
 
 # Optional: HTTP backend (native)
 reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls", "json"] }
@@ -65,10 +73,6 @@ fred = { version = "9", optional = true, features = ["i-scripts"] }
 # uses `is_multiple_of`/let-chains and breaks this workspace's 1.85 MSRV.
 memcache = { version = "0.19", optional = true, default-features = false }
 
-# Optional: shared by the memcached and file backends for spawn_blocking.
-# Feature lists live on the `memcached`/`file` feature declarations above.
-tokio = { version = "1", optional = true, default-features = false }
-
 # Optional: L1 in-process cache
 moka = { version = "0.12", optional = true, features = ["sync"] }
 
@@ -88,7 +92,7 @@ libc = { version = "0.2", optional = true }
 rustix = { version = "1", optional = true, default-features = false, features = ["fs", "process", "std"] }
 
 [dev-dependencies]
-tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] }
 serde_json = "1"
 serial_test = "3"
 tempfile = "3"

crates/cachekit/src/backend/cachekitio.rs

@@ -4,7 +4,7 @@ use std::time::Duration;
 use async_trait::async_trait;
 use zeroize::Zeroizing;
 
-use crate::backend::{Backend, HealthStatus};
+use crate::backend::{Backend, HealthStatus, LockableBackend};
 use crate::error::{BackendError, BackendErrorKind};
 use crate::metrics::{metrics_headers, MetricsProvider};
 use crate::session::session_headers;
@@ -247,6 +247,10 @@ impl Backend for CachekitIO {
             Err(self.error_from_response(resp).await)
         }
     }
+
+    fn as_lockable(&self) -> Option<&dyn LockableBackend> {
+        Some(self)
+    }
 }
 
 // ── Builder ───────────────────────────────────────────────────────────────────

crates/cachekit/src/backend/mod.rs

@@ -49,6 +49,16 @@ pub trait Backend: Send + Sync {
 
     /// Return health/status information for this backend.
     async fn health(&self) -> Result<HealthStatus, BackendError>;
+
+    /// Expose this backend's [`LockableBackend`] capability, if it has one.
+    ///
+    /// Trait objects (`dyn Backend`) cannot be cross-cast to a sibling trait,
+    /// so backends that support distributed locking opt in by overriding this
+    /// to return `Some(self)`. Used by the client's cold-miss single-flight
+    /// for cross-process fill suppression. Default: `None`.
+    fn as_lockable(&self) -> Option<&dyn LockableBackend> {
+        None
+    }
 }
 
 /// Async cache backend abstraction (`?Send` variant).
@@ -77,6 +87,16 @@ pub trait Backend {
 
     /// Return health/status information for this backend.
     async fn health(&self) -> Result<HealthStatus, BackendError>;
+
+    /// Expose this backend's [`LockableBackend`] capability, if it has one.
+    ///
+    /// Trait objects (`dyn Backend`) cannot be cross-cast to a sibling trait,
+    /// so backends that support distributed locking opt in by overriding this
+    /// to return `Some(self)`. Used by the client's cold-miss single-flight
+    /// for cross-process fill suppression. Default: `None`.
+    fn as_lockable(&self) -> Option<&dyn LockableBackend> {
+        None
+    }
 }
 
 // ── TtlInspectable ───────────────────────────────────────────────────────────

crates/cachekit/src/backend/redis.rs

@@ -124,6 +124,10 @@ impl Backend for RedisBackend {
             details,
         })
     }
+
+    fn as_lockable(&self) -> Option<&dyn LockableBackend> {
+        Some(self)
+    }
 }
 
 // ── TtlInspectable impl ───────────────────────────────────────────────────────

crates/cachekit/src/client.rs

@@ -77,6 +77,7 @@ pub struct CacheKit {
     default_ttl: Duration,
     namespace: Option<String>,
     max_payload_bytes: usize,
+    flight: crate::flight::FlightMap,
 
     #[cfg(feature = "l1")]
     l1: Option<crate::l1::L1Cache>,
@@ -344,6 +345,24 @@ impl CacheKit {
         Ok(self.backend.exists(&full_key).await?)
     }
 
+    // ── Single-flight ─────────────────────────────────────────────────────────
+
+    /// Begin a cold-miss single-flight for `key` (see [`crate::flight`]).
+    ///
+    /// Call after a cache miss, before computing the value. Concurrent
+    /// in-process fills of the same key are collapsed to one; with the
+    /// `reliability` feature and a lock-capable backend (CachekitIO, Redis),
+    /// fills are also suppressed across processes via a distributed fill
+    /// lock. The `#[cachekit]` macro does this automatically.
+    ///
+    /// The key is namespaced like every cache operation but not validated —
+    /// this call is infallible; an invalid key simply fails later at the
+    /// actual cache operation.
+    pub async fn single_flight(&self, key: &str) -> crate::flight::SingleFlight {
+        let full_key = self.namespaced_key(key);
+        crate::flight::SingleFlight::acquire(&self.flight, &self.backend, &full_key).await
+    }
+
     // ── Secure cache ─────────────────────────────────────────────────────────
 
     /// Return a [`SecureCache`] handle that encrypts all values before storage.
@@ -526,6 +545,9 @@ pub struct CacheKitBuilder {
 
     #[cfg(feature = "encryption")]
     encryption: Option<SharedEncryption>,
+
+    #[cfg(all(feature = "reliability", not(target_arch = "wasm32")))]
+    reliability: Option<crate::reliability::ReliabilityConfig>,
 }
 
 impl CacheKitBuilder {
@@ -578,6 +600,19 @@ impl CacheKitBuilder {
         self
     }
 
+    /// Wrap the backend in the reliability stack (retry with exponential
+    /// backoff + jitter, circuit breaker) — see [`crate::reliability`].
+    ///
+    /// Enabled by default with production settings by the `production`,
+    /// `encrypted`, and `io` intent presets; off for `minimal` and for
+    /// manually-built clients. To opt a preset out, pass a config with both
+    /// layers `None` — an empty config applies no wrapping at all.
+    #[cfg(all(feature = "reliability", not(target_arch = "wasm32")))]
+    pub fn reliability(mut self, config: crate::reliability::ReliabilityConfig) -> Self {
+        self.reliability = Some(config);
+        self
+    }
+
     /// Configure encryption from raw master key bytes and tenant ID.
     ///
     /// The master key must be at least 16 bytes (32 recommended).
@@ -650,11 +685,23 @@ impl CacheKitBuilder {
             Some(crate::l1::L1Cache::new(capacity))
         };
 
+        // Apply the reliability stack last so it decorates the final backend.
+        // A config with neither layer set is the documented opt-out: skip the
+        // (no-op) decorator entirely.
+        #[cfg(all(feature = "reliability", not(target_arch = "wasm32")))]
+        let backend = match self.reliability {
+            Some(config) if config.retry.is_some() || config.circuit_breaker.is_some() => {
+                crate::reliability::wrap_reliable(backend, config)
+            }
+            _ => backend,
+        };
+
         Ok(CacheKit {
             backend,
             default_ttl: self.default_ttl.unwrap_or(Duration::from_secs(300)),
             namespace: self.namespace,
             max_payload_bytes: self.max_payload_bytes.unwrap_or(5 * 1024 * 1024),
+            flight: crate::flight::FlightMap::default(),
 
             #[cfg(feature = "l1")]
             l1,