From 8d7e9d347f0687493fd45e13ada1b0ff85dae1ab Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Fri, 24 Jul 2026 01:13:46 +1000 Subject: [PATCH] feat(cache): cold-miss single-flight + opt-in cross-process locking (LAB-519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cold L1+L2 miss previously executed the wrapped function once per concurrent caller — N cold callers meant N upstream calls, N L2 GETs (N billed misses under metered-misses), and N L2 writes. In-process (always on): concurrent wrap() calls for one cache key share a single in-flight promise covering the L2 read, compute, and write. The L2 read must be inside the flight — the GET-miss is the billed event, so deduping only the compute would still bill N misses. Cross-process (opt-in, stampede.distributedLock): wires the previously caller-less LockableBackend capability around the miss path, mirroring cachekit-py's acquire_lock flow — acquire, double-check L2, compute, write, release. Contested waiters retry the lock on an interval rather than polling get() (a poll GET against a still-cold key is itself a billed miss), then fall through to computing after lockWaitMs: the lease is best-effort mitigation, never a correctness gate. Lock calls stay outside the reliability executor so lock failures neither stack retry latency nor open the circuit breaker for data operations. Deliberately no general admission-control cap beyond L1's maxConcurrentRefreshes: on Node's single-threaded event loop concurrent misses don't compete for threads, single-flight collapses the per-key herd, and distinct-key floods are bounded by backend timeouts plus the circuit breaker. Decision recorded on StampedeConfig. Co-authored-by: multica-agent --- packages/cachekit/README.md | 45 +++ .../cachekit/src/cache.single-flight.test.ts | 293 ++++++++++++++++++ packages/cachekit/src/cache.ts | 211 ++++++++++++- packages/cachekit/src/constants.ts | 13 + packages/cachekit/src/index.ts | 1 + packages/cachekit/src/types/cache.ts | 58 ++++ 6 files changed, 607 insertions(+), 14 deletions(-) create mode 100644 packages/cachekit/src/cache.single-flight.test.ts diff --git a/packages/cachekit/README.md b/packages/cachekit/README.md index d270f3b..2935f23 100644 --- a/packages/cachekit/README.md +++ b/packages/cachekit/README.md @@ -8,6 +8,7 @@ Production-ready Redis caching for TypeScript/Node.js. Hybrid TypeScript-Rust de - **Dual-layer caching**: L1 in-memory (~50ns) + L2 Redis (~2-50ms) - **Stale-while-revalidate**: Serve stale data while refreshing in background +- **Stampede protection**: Cold-miss single-flight per process (always on) + opt-in cross-process distributed locks - **Zero-knowledge encryption**: Optional AES-256-GCM client-side encryption - **Circuit breaker**: Automatic failure isolation with exponential backoff - **TypeScript-first**: Full type safety with strict mode @@ -133,6 +134,50 @@ const cache = createCache({ }); ``` +## Stampede Protection + +A cold cache key hit by N concurrent callers would normally execute the wrapped +function N times and issue N backend reads — on the metered-misses SaaS backend +that is N billed misses for one key. + +**In-process single-flight is always on**: concurrent `wrap()` calls for the +same cache key share one in-flight promise, so the herd costs one L2 read, one +compute, and one write. If the flight fails, every waiting caller receives the +same rejection and the next call retries fresh. + +**Cross-process locking is opt-in** via `stampede.distributedLock`, for fleets +where many processes can go cold on the same key simultaneously. It mirrors +cachekit-py's flow: acquire the backend lock, double-check L2, compute, write, +release. Contested processes retry the lock on an interval (never polling +`get()` — on metered-misses backends a poll against a still-cold key is itself +a billed miss) and fall through to computing after `lockWaitMs`; the lock is +best-effort mitigation, never a correctness gate, so lock-endpoint failures +degrade to an unlocked compute. + +```typescript +const cache = createCache({ + backend: { url: 'redis://localhost:6379' }, // Redis and CachekitIO backends support locks + stampede: { + distributedLock: true, // default false + lockTimeoutMs: 30000, // lock lease; size at/above expected recompute time + lockWaitMs: 5000, // max wait for the lock holder before computing anyway + lockPollMs: 100, // lock retry interval while contested + }, +}); +``` + +Requires a lock-capable backend: Redis, a `CachekitIOBackendConfig` (the SaaS +lock endpoint is selected automatically), `cachekitioWithLocking()`, or +`cachekitioFull()`. `createCache` throws `ConfigurationError` if +`distributedLock` is requested on a backend without `acquireLock`/`releaseLock`. + +There is deliberately no general admission-control cap beyond L1's +`maxConcurrentRefreshes`: on Node's single-threaded event loop concurrent +misses don't compete for threads, single-flight collapses the per-key herd, +and distinct-key miss floods are already bounded by backend timeouts plus the +circuit breaker. A global semaphore would add queueing latency without a +failure mode it prevents. + ## API Reference ### createCache(options) diff --git a/packages/cachekit/src/cache.single-flight.test.ts b/packages/cachekit/src/cache.single-flight.test.ts new file mode 100644 index 0000000..7ac3959 --- /dev/null +++ b/packages/cachekit/src/cache.single-flight.test.ts @@ -0,0 +1,293 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { createCache } from './cache.js'; +import { ConfigurationError } from './errors.js'; +import type { SecureCache } from './types/cache.js'; +import type { Backend, LockableBackend } from './backends/types.js'; + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * In-memory backend with operation counters. `gets` on a cold key is the + * billed-miss count under metered-misses — the number these tests exist to + * pin down (LAB-519). + */ +class CountingBackend implements Backend { + store = new Map(); + gets = 0; + sets = 0; + + async get(key: string): Promise { + this.gets++; + return this.store.get(key) ?? null; + } + + async set(key: string, value: Uint8Array, _ttl?: number): Promise { + this.sets++; + this.store.set(key, value); + } + + async delete(key: string): Promise { + return this.store.delete(key); + } + + async exists(key: string): Promise { + return this.store.has(key); + } + + async close(): Promise { + // Shared between cache instances in cross-process tests — keep the store. + } +} + +/** Lockable variant: single-holder lock table, compare-and-delete release. */ +class CountingLockableBackend extends CountingBackend implements LockableBackend { + acquires = 0; + releases = 0; + private readonly locks = new Map(); + + async acquireLock(key: string, _timeoutMs?: number): Promise { + this.acquires++; + if (this.locks.has(key)) return null; + const lockId = `lock-${this.acquires}`; + this.locks.set(key, lockId); + return lockId; + } + + async releaseLock(key: string, lockId: string): Promise { + this.releases++; + if (this.locks.get(key) !== lockId) return false; + this.locks.delete(key); + return true; + } + + heldLocks(): number { + return this.locks.size; + } +} + +describe('Cold-miss single-flight (LAB-519)', () => { + const caches: SecureCache[] = []; + + const make = (options: Parameters[0]): SecureCache => { + const cache = createCache(options); + caches.push(cache); + return cache; + }; + + afterEach(async () => { + await Promise.all(caches.splice(0).map((c) => c.close())); + }); + + describe('in-process', () => { + it('shares one L2 read, one compute, and one write across a concurrent cold herd', async () => { + const backend = new CountingBackend(); + const cache = make({ backend, l1: { enabled: true } }); + + let calls = 0; + const wrapped = cache.wrap( + async (id: number) => { + calls++; + await sleep(20); + return `value-${id}`; + }, + { namespace: 'sf:herd', ttl: 60 } + ); + + const results = await Promise.all(Array.from({ length: 10 }, () => wrapped(1))); + + expect(results).toEqual(Array.from({ length: 10 }, () => 'value-1')); + expect(calls).toBe(1); + expect(backend.gets).toBe(1); // 1 billed miss, not 10 + expect(backend.sets).toBe(1); + }); + + it('does not share flights across distinct keys', async () => { + const backend = new CountingBackend(); + const cache = make({ backend, l1: { enabled: true } }); + + let calls = 0; + const wrapped = cache.wrap( + async (id: number) => { + calls++; + await sleep(10); + return id; + }, + { namespace: 'sf:keys', ttl: 60 } + ); + + const [a, b] = await Promise.all([wrapped(1), wrapped(2)]); + + expect(a).toBe(1); + expect(b).toBe(2); + expect(calls).toBe(2); + }); + + it('evicts the flight after it settles so later calls read the cache', async () => { + const backend = new CountingBackend(); + const cache = make({ backend, l1: { enabled: false } }); + + let calls = 0; + const wrapped = cache.wrap( + async () => { + calls++; + return 'once'; + }, + { namespace: 'sf:settle', ttl: 60 } + ); + + await wrapped(); + const second = await wrapped(); + + expect(second).toBe('once'); + expect(calls).toBe(1); + expect(backend.gets).toBe(2); // second call is its own flight: L2 hit + }); + + it('rejects every herd member on failure, then retries fresh on the next call', async () => { + const backend = new CountingBackend(); + const cache = make({ backend, l1: { enabled: true } }); + + let calls = 0; + const wrapped = cache.wrap( + async () => { + calls++; + await sleep(10); + if (calls === 1) throw new Error('upstream down'); + return 'recovered'; + }, + { namespace: 'sf:err', ttl: 60 } + ); + + const herd = await Promise.allSettled(Array.from({ length: 5 }, () => wrapped())); + + expect(herd.every((r) => r.status === 'rejected')).toBe(true); + expect(calls).toBe(1); // one shared failure, not five + + // Rejected flight must not be cached: next call recomputes. + await expect(wrapped()).resolves.toBe('recovered'); + expect(calls).toBe(2); + }); + }); + + describe('cross-process distributed lock (opt-in)', () => { + const lockOptions = { distributedLock: true, lockWaitMs: 2000, lockPollMs: 10 }; + + it('contested process waits on the lock and picks up the winner’s write instead of recomputing', async () => { + // Two cache instances sharing one backend = two processes sharing L2. + const backend = new CountingLockableBackend(); + const cache1 = make({ backend, l1: { enabled: false }, stampede: lockOptions }); + const cache2 = make({ backend, l1: { enabled: false }, stampede: lockOptions }); + + let calls = 0; + const slow = async (id: number): Promise => { + calls++; + await sleep(100); + return `result-${id}`; + }; + const w1 = cache1.wrap(slow, { namespace: 'sf:fleet', ttl: 60 }); + const w2 = cache2.wrap(slow, { namespace: 'sf:fleet', ttl: 60 }); + + const [a, b] = await Promise.all([w1(7), w2(7)]); + + expect(a).toBe('result-7'); + expect(b).toBe('result-7'); + expect(calls).toBe(1); // loser's post-grant double-check hit the winner's write + expect(backend.sets).toBe(1); + expect(backend.heldLocks()).toBe(0); // both sides released + expect(backend.releases).toBeGreaterThanOrEqual(1); + }); + + it('computes anyway when the holder never fills the cache (stampede fallthrough)', async () => { + class NeverGrantsBackend extends CountingBackend implements LockableBackend { + async acquireLock(): Promise { + return null; // permanently contested (e.g. holder crashed, lease outlives our wait) + } + async releaseLock(): Promise { + return false; + } + } + + const backend = new NeverGrantsBackend(); + const cache = make({ + backend, + l1: { enabled: false }, + stampede: { distributedLock: true, lockWaitMs: 100, lockPollMs: 20 }, + }); + + let calls = 0; + const wrapped = cache.wrap( + async () => { + calls++; + return 'computed-anyway'; + }, + { namespace: 'sf:fallthrough', ttl: 60 } + ); + + await expect(wrapped()).resolves.toBe('computed-anyway'); + expect(calls).toBe(1); + }); + + it('degrades to computing without the lock when acquireLock throws', async () => { + class BrokenLockBackend extends CountingBackend implements LockableBackend { + async acquireLock(): Promise { + throw new Error('lock endpoint unreachable'); + } + async releaseLock(): Promise { + return false; + } + } + + const backend = new BrokenLockBackend(); + const cache = make({ backend, l1: { enabled: false }, stampede: lockOptions }); + + let calls = 0; + const wrapped = cache.wrap( + async () => { + calls++; + return 'still-works'; + }, + { namespace: 'sf:broken-lock', ttl: 60 } + ); + + await expect(wrapped()).resolves.toBe('still-works'); + expect(calls).toBe(1); + expect(backend.sets).toBe(1); + }); + + it('releases the lock even when the compute throws', async () => { + const backend = new CountingLockableBackend(); + const cache = make({ backend, l1: { enabled: false }, stampede: lockOptions }); + + const wrapped = cache.wrap( + async () => { + throw new Error('compute failed'); + }, + { namespace: 'sf:release-on-error', ttl: 60 } + ); + + await expect(wrapped()).rejects.toThrow('compute failed'); + expect(backend.heldLocks()).toBe(0); + }); + + it('rejects distributedLock on a backend without lock capability', () => { + expect(() => + createCache({ + backend: new CountingBackend(), + stampede: { distributedLock: true }, + }) + ).toThrow(ConfigurationError); + }); + + it('rejects invalid stampede timing config', () => { + expect(() => + createCache({ backend: new CountingBackend(), stampede: { lockPollMs: 0 } }) + ).toThrow(ConfigurationError); + expect(() => + createCache({ backend: new CountingBackend(), stampede: { lockTimeoutMs: -1 } }) + ).toThrow(ConfigurationError); + expect(() => + createCache({ backend: new CountingBackend(), stampede: { lockWaitMs: Infinity } }) + ).toThrow(ConfigurationError); + }); + }); +}); diff --git a/packages/cachekit/src/cache.ts b/packages/cachekit/src/cache.ts index 1d26ce8..e09fdd0 100644 --- a/packages/cachekit/src/cache.ts +++ b/packages/cachekit/src/cache.ts @@ -1,13 +1,20 @@ import type { CacheOptions, SetOptions, + StampedeConfig, WrapOptions, + WrapOptionsBase, SecureCache, InvalidationConfig, } from './types/cache.js'; -import type { Backend, RedisBackendConfig, CachekitIOBackendConfig } from './backends/types.js'; +import type { + Backend, + LockableBackend, + RedisBackendConfig, + CachekitIOBackendConfig, +} from './backends/types.js'; import { redis } from './backends/redis.js'; -import { cachekitio } from './backends/cachekitio-factory.js'; +import { cachekitio, cachekitioWithLocking } from './backends/cachekitio-factory.js'; import { L1Cache } from './l1/lru-cache.js'; import { ReliabilityExecutor } from './reliability/executor.js'; import { BackgroundRefreshManager } from './cache/background-refresh.js'; @@ -28,7 +35,21 @@ import { ByteStorage } from '@cachekit-io/cachekit-core-ts'; import { RedisInvalidationChannel } from './invalidation/redis-channel.js'; import { createInvalidationEvent } from './invalidation/event.js'; import { BackendError, ConfigurationError } from './errors.js'; -import { DEFAULT_TTL_SECONDS } from './constants.js'; +import { + DEFAULT_TTL_SECONDS, + DEFAULT_LOCK_TIMEOUT_MS, + DEFAULT_LOCK_WAIT_MS, + DEFAULT_LOCK_POLL_MS, +} from './constants.js'; + +/** + * Sentinel for "the lock path did not resolve the miss — compute without + * it". Distinct from null: the wrapped function may legitimately resolve + * null, and conflating the two would compute twice under a held lock. + */ +const LOCK_FALLTHROUGH = Symbol('cachekit.lock-fallthrough'); + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); /** * Internal cache implementation. @@ -44,17 +65,64 @@ class CacheImpl implements SecureCache { private readonly defaultTtl: number; private readonly invalidationChannel: RedisInvalidationChannel | null = null; private closed = false; + /** One in-flight cold-miss resolution per cache key (single-flight, LAB-519). */ + private readonly inflight = new Map>(); + private readonly stampede: Required; + private readonly lockable: LockableBackend | null; constructor(options: CacheOptions) { // Initialize backend if ('get' in options.backend) { this.backend = options.backend; } else if ('apiKey' in options.backend) { - this.backend = cachekitio(options.backend as CachekitIOBackendConfig); + // The plain SaaS factory has no lock capability; the distributedLock + // opt-in selects the lockable wrapper so config-based users aren't + // forced to construct the backend by hand. + this.backend = options.stampede?.distributedLock + ? cachekitioWithLocking(options.backend as CachekitIOBackendConfig) + : cachekitio(options.backend as CachekitIOBackendConfig); } else { this.backend = redis(options.backend as RedisBackendConfig); } + // Stampede config + lock capability. Duck-typed like cachekit-py's + // hasattr check: user-supplied Backend instances aren't required to + // declare the LockableBackend interface, only to implement it. + this.stampede = { + distributedLock: options.stampede?.distributedLock ?? false, + lockTimeoutMs: options.stampede?.lockTimeoutMs ?? DEFAULT_LOCK_TIMEOUT_MS, + lockWaitMs: options.stampede?.lockWaitMs ?? DEFAULT_LOCK_WAIT_MS, + lockPollMs: options.stampede?.lockPollMs ?? DEFAULT_LOCK_POLL_MS, + }; + if (!Number.isFinite(this.stampede.lockTimeoutMs) || this.stampede.lockTimeoutMs <= 0) { + throw new ConfigurationError( + `stampede.lockTimeoutMs must be > 0, got ${this.stampede.lockTimeoutMs}` + ); + } + if (!Number.isFinite(this.stampede.lockPollMs) || this.stampede.lockPollMs <= 0) { + throw new ConfigurationError( + `stampede.lockPollMs must be > 0, got ${this.stampede.lockPollMs}` + ); + } + if (!Number.isFinite(this.stampede.lockWaitMs) || this.stampede.lockWaitMs < 0) { + throw new ConfigurationError( + `stampede.lockWaitMs must be >= 0, got ${this.stampede.lockWaitMs}` + ); + } + const maybeLockable = this.backend as Partial; + this.lockable = + typeof maybeLockable.acquireLock === 'function' && + typeof maybeLockable.releaseLock === 'function' + ? (this.backend as LockableBackend) + : null; + if (this.stampede.distributedLock && !this.lockable) { + throw new ConfigurationError( + 'stampede.distributedLock requires a backend with lock capability ' + + '(Redis, cachekitioWithLocking, or cachekitioFull) — the configured backend ' + + 'has no acquireLock/releaseLock' + ); + } + // Initialize L1 cache if (options.l1?.enabled !== false) { this.l1 = new L1Cache(options.l1); @@ -374,7 +442,12 @@ class CacheImpl implements SecureCache { swrResult.versionToken, this.l1, async (key, value, opts) => { - await this.setEntry(key, value, { ttl: opts.ttl, namespace: opts.namespace }, interop); + await this.setEntry( + key, + value, + { ttl: opts.ttl, namespace: opts.namespace }, + interop + ); } ); } @@ -382,18 +455,123 @@ class CacheImpl implements SecureCache { } } - // Check L2 - const cached = await this.getEntry(cacheKey, interop, options.ttl); - if (cached !== null) { - return cached; + // Cold path (L1 miss): single-flight per key per process. The flight + // covers the L2 read too, not just the compute — the L2 GET-miss is + // the billed event under metered-misses, so N concurrent cold callers + // must share one read, one compute, and one write (LAB-519). + const existing = this.inflight.get(cacheKey); + if (existing) { + return existing as Promise; + } + const flight = this.resolveMiss(cacheKey, interop, () => fn(...args), options); + this.inflight.set(cacheKey, flight); + try { + return await flight; + } finally { + this.inflight.delete(cacheKey); + } + }; + } + + /** + * Cold-path resolution shared by every concurrent caller of one key: + * L2 read, then compute + write, optionally bracketed by a distributed + * lock when the backend supports it and stampede.distributedLock is on. + */ + private async resolveMiss( + cacheKey: string, + interop: boolean, + compute: () => Promise, + options: WrapOptionsBase + ): Promise { + const cached = await this.getEntry(cacheKey, interop, options.ttl); + if (cached !== null) { + return cached; + } + + if (this.lockable && this.stampede.distributedLock) { + const locked = await this.resolveUnderLock(cacheKey, interop, compute, options); + if (locked !== LOCK_FALLTHROUGH) { + return locked; } + } - // Compute and cache - const result = await fn(...args); - await this.setEntry(cacheKey, result, { ttl: options.ttl, namespace: options.namespace }, interop); + return this.computeAndStore(cacheKey, interop, compute, options); + } - return result; - }; + /** + * Cross-process miss arbitration, mirroring cachekit-py's acquire_lock + * flow (wrapper.py): acquire → double-check L2 → compute → write → + * release. acquireLock never blocks on contention (LAB-240), so + * "waiting" is retrying the lock on an interval bounded by lockWaitMs — + * deliberately NOT polling get(), because on a metered-misses backend + * every poll GET against a still-cold key is itself a billed miss, + * while contested lock calls are not. + * + * Returns LOCK_FALLTHROUGH when the lock never resolved the miss + * (acquire error, or contested past the wait budget): the lease is + * best-effort stampede mitigation, never a correctness gate, so lock + * failure degrades to computing without it. + */ + private async resolveUnderLock( + cacheKey: string, + interop: boolean, + compute: () => Promise, + options: WrapOptionsBase + ): Promise { + const lockable = this.lockable!; + const { lockTimeoutMs, lockWaitMs, lockPollMs } = this.stampede; + const deadline = Date.now() + lockWaitMs; + + for (;;) { + let lockId: string | null; + try { + // Deliberately outside the reliability executor: retry would stack + // latency onto a best-effort call, and counting lock failures + // against the circuit breaker could open it for data operations. + lockId = await lockable.acquireLock(cacheKey, lockTimeoutMs); + } catch { + return LOCK_FALLTHROUGH; + } + + if (lockId !== null) { + try { + // Double-check: the holder we waited on (or a racing process) + // may have written between our miss and this grant — one GET + // that hits, instead of a duplicate compute + write. + const filled = await this.getEntry(cacheKey, interop, options.ttl); + if (filled !== null) { + return filled; + } + return await this.computeAndStore(cacheKey, interop, compute, options); + } finally { + // Best-effort: the lease auto-expires, and a failed release must + // not mask the compute result. + lockable.releaseLock(cacheKey, lockId).catch(() => {}); + } + } + + if (Date.now() + lockPollMs > deadline) { + return LOCK_FALLTHROUGH; + } + await sleep(lockPollMs); + } + } + + private async computeAndStore( + cacheKey: string, + interop: boolean, + compute: () => Promise, + options: WrapOptionsBase + ): Promise { + const result = await compute(); + await this.setEntry( + cacheKey, + result, + { ttl: options.ttl, namespace: options.namespace }, + interop + ); + return result; } with( @@ -466,6 +644,11 @@ class CacheImpl implements SecureCache { // Stop background refresh manager (clears in-flight refreshes) this.backgroundRefresh.close(); + // Drop single-flight registrations (in-flight promises settle on their + // own; callers already awaiting them get the result or a closed-backend + // error) + this.inflight.clear(); + // Stop invalidation channel if (this.invalidationChannel) { await this.invalidationChannel.stop(); diff --git a/packages/cachekit/src/constants.ts b/packages/cachekit/src/constants.ts index 4cd6452..99c53ea 100644 --- a/packages/cachekit/src/constants.ts +++ b/packages/cachekit/src/constants.ts @@ -126,6 +126,19 @@ export const MIN_MASTER_KEY_BYTES = 32; /** Minimum master key length in hex characters */ export const MIN_MASTER_KEY_HEX_LENGTH = 64; +// ============================================================================ +// Stampede / Single-Flight Constants +// ============================================================================ + +/** Default distributed lock lease in milliseconds (matches cachekit-py's lock_timeout=30s) */ +export const DEFAULT_LOCK_TIMEOUT_MS = 30000; + +/** Default wait for a contested lock holder to fill the cache (matches cachekit-py's blocking_timeout=5s) */ +export const DEFAULT_LOCK_WAIT_MS = 5000; + +/** Default lock retry interval while contested, in milliseconds */ +export const DEFAULT_LOCK_POLL_MS = 100; + // ============================================================================ // SWR (Stale-While-Revalidate) Constants // ============================================================================ diff --git a/packages/cachekit/src/index.ts b/packages/cachekit/src/index.ts index 2e1559f..26cc988 100644 --- a/packages/cachekit/src/index.ts +++ b/packages/cachekit/src/index.ts @@ -25,6 +25,7 @@ export type { WrapOptions, EncryptionConfig, ReliabilityConfig, + StampedeConfig, InvalidationConfig, } from './types/cache.js'; diff --git a/packages/cachekit/src/types/cache.ts b/packages/cachekit/src/types/cache.ts index 9850cee..918454b 100644 --- a/packages/cachekit/src/types/cache.ts +++ b/packages/cachekit/src/types/cache.ts @@ -100,6 +100,61 @@ export interface EncryptionConfig { tenantId?: string; } +/** + * Cold-miss stampede protection. + * + * In-process single-flight is always on: concurrent wrap() calls for the + * same cache key share one in-flight promise, so a herd of N cold callers + * costs one L2 read, one compute, and one L2 write instead of N of each — + * under metered-misses billing that is 1 billed miss per cold key instead + * of N. + * + * This config only controls the cross-process extension of that posture. + * + * Deliberately absent — general admission control (a global concurrent-miss + * cap beyond L1's maxConcurrentRefreshes): on Node's single-threaded event + * loop concurrent misses don't compete for threads, single-flight already + * collapses the per-key herd (the amplification vector metered-misses + * punishes), and distinct-key miss floods are bounded by backend timeouts + * plus the circuit breaker. A global semaphore would add queueing latency + * and a tuning knob without a failure mode it prevents; revisit only with + * evidence of backend connection exhaustion. (LAB-519) + */ +export interface StampedeConfig { + /** + * Hold a backend distributed lock around cold-miss compute, mirroring + * cachekit-py's acquire_lock flow: acquire → double-check L2 → compute → + * write → release. When contested, re-try the lock on an interval up to + * `lockWaitMs` (acquireLock never blocks — LAB-240), then compute anyway: + * the lease is best-effort stampede mitigation, never a correctness gate. + * + * Contested waiters deliberately retry the LOCK rather than polling + * get(): on a metered-misses backend every poll GET against a still-cold + * key would itself be a billed miss. + * + * Requires a LockableBackend (Redis, cachekitioWithLocking, + * cachekitioFull, or a CachekitIO config — the SaaS lock endpoint is + * selected automatically). createCache() throws ConfigurationError when + * the backend has no lock capability — an explicit opt-in that silently + * does nothing would be a lie. Default: false. + */ + distributedLock?: boolean; + /** + * Lock lease in milliseconds before auto-release. Size at or above the + * expected recompute time (default 30000, matching cachekit-py's + * lock_timeout). + */ + lockTimeoutMs?: number; + /** + * Contested: max milliseconds to wait for the lock holder to fill the + * cache before computing anyway (default 5000, matching cachekit-py's + * blocking_timeout). + */ + lockWaitMs?: number; + /** Contested: lock retry interval in milliseconds (default 100). */ + lockPollMs?: number; +} + /** * Reliability configuration for cache. */ @@ -131,6 +186,9 @@ export interface CacheOptions { /** Reliability configuration (circuit breaker, retry) */ reliability?: ReliabilityConfig; + /** Cold-miss stampede protection (cross-process lock opt-in) */ + stampede?: StampedeConfig; + /** Serializer configuration */ serializer?: Partial;