Mark KeyViz fanout design implemented

adapter/grpc_test.go

@@ -16,6 +16,19 @@ import (
 	_ "google.golang.org/grpc/health"
 )
 
+const (
+	grpcSequenceFullIterations  = 9999
+	grpcSequenceShortIterations = 256
+)
+
+func grpcSequenceIterations(t testing.TB) int {
+	t.Helper()
+	if testing.Short() {
+		return grpcSequenceShortIterations
+	}
+	return grpcSequenceFullIterations
+}
+
 func Test_value_can_be_deleted(t *testing.T) {
 	t.Parallel()
 	nodes, adders, _ := createNode(t, 3)
@@ -277,7 +290,7 @@ func Test_consistency_satisfy_write_after_read_sequence(t *testing.T) {
 	// not abort the test. The post-RPC assert.Equal still pins the
 	// consistency invariant: once Put eventually succeeds, the
 	// subsequent Get must return the same value, otherwise we fail.
-	for i := range 9999 {
+	for i := range grpcSequenceIterations(t) {
 		want := []byte("sequence" + strconv.Itoa(i))
 		err := retryNotLeader(ctx, func() error {
 			_, perr := c.RawPut(ctx, &pb.RawPutRequest{Key: key, Value: want})
@@ -332,7 +345,7 @@ func Test_grpc_transaction(t *testing.T) {
 	// _sequence: tolerate transient leader churn (purely availability,
 	// not consistency) while keeping the Put → Get → Delete → Get
 	// invariants strict.
-	for i := range 9999 {
+	for i := range grpcSequenceIterations(t) {
 		want := []byte("sequence" + strconv.Itoa(i))
 		err := retryNotLeader(ctx, func() error {
 			_, perr := c.Put(ctx, &pb.PutRequest{Key: key, Value: want})

adapter/test_util.go

@@ -757,7 +757,7 @@ func eventuallyExpired(t *testing.T, ttl time.Duration, condition func() bool, m
 // it safe to call from worker goroutines in parallel tests.
 //
 // Use cases: mid-test leader churn under CI load (e.g. grpc_test.go's
-// 9999-iteration consistency loops). The startup window — leader-churn
+// sequence consistency loops). The startup window — leader-churn
 // between createNode returning and the first write — is closed at the
 // readiness layer instead (waitForWriteableLeader, PR #898), so first-
 // write callers should NOT wrap in retryNotLeader; direct calls suffice

docs/design/2026_04_28_implemented_keyviz_adapter_labels.md

@@ -587,20 +587,24 @@ from PR #694: Claude bot critical, Gemini high.)
 The fan-out aggregator's per-cell merge key gains the label:
 
 - Phase 2-C (current): `(bucketID, raftGroupID, leaderTerm,
-  windowStart)` per design `2026_04_27_proposed_keyviz_cluster_fanout.md`
+  windowStart)` per design `2026_04_27_implemented_keyviz_cluster_fanout.md`
   §4.
 - With labels: same tuple — but `bucketID` itself now carries the
   label via the §5 composite (`route:1:dynamo`). The merge key
   width does **not** change; the new label dimension is
   encoded into `bucketID` so the aggregator already separates
   same-route different-label rows correctly.
 
-Reads still sum, writes still max-with-conflict; nothing about
-the merge **rules** changes other than the wire shape of
-`bucketID`. The merge **key** logic is unchanged: composite
-`bucketID` already separates same-route different-label rows
-correctly, so the aggregator's bucketing/grouping path needs no
-edits.
+Reads still sum. Writes inherit the implemented fan-out rule from
+`2026_04_27_implemented_keyviz_cluster_fanout.md` §4: when all
+non-zero contributors carry identity, writes are deduped by
+`(bucketID, raftGroupID, leaderTerm, windowStart)`, maxed within one
+identity, and summed across distinct leader terms for the same group
+and window. Missing identity falls back to the legacy max-with-conflict
+rule. Labels do not add another merge dimension beyond `bucketID`:
+the composite `bucketID` already separates same-route different-label
+rows correctly, so the aggregator's bucketing/grouping path needs no
+extra label-specific edit.
 
 The merge **value** path, however, still needs one explicit
 field copy: `mergeRowInto` (`internal/admin/keyviz_fanout.go:509`)

docs/design/2026_04_29_proposed_logical_backup.md

@@ -1570,7 +1570,7 @@ Scope: out of this proposal; mentioned only to draw the boundary.
 - Concurrent multi-cluster fan-out (one logical backup spanning shards
   on different physical clusters) — depends on the `Distribution`
   control plane being fan-out-aware (see
-  `2026_04_27_proposed_keyviz_cluster_fanout.md`).
+  `2026_04_27_implemented_keyviz_cluster_fanout.md`).
 
 ## Required Tests
 

docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md

@@ -161,7 +161,7 @@ Options considered:
 
 **Follower-forwarded write caveat (codex P2).** `ShardedCoordinator.observeMutation` runs on the node that received the client request before `ShardRouter.Commit` forwards (`kv/sharded_coordinator.go:1795-1844`), and the follower's `Internal.Forward` handler calls `transactionManager.Commit` directly (`adapter/internal.go:52`) rather than re-entering the leader's `ShardedCoordinator` observation hook. Therefore even a route whose shard group is led by the default-group leader can be **under-observed** if most clients connect through followers. M3 does not treat "missing local rows" as evidence of low load: local evidence can promote a split, but absence of local evidence only means "unknown" and is fail-closed for target selection (§6.2) and conservative for candidacy (fewer splits). Full coverage for follower-forwarded writes needs follower reports / fan-out into the same detector interface and is tracked under OQ-5.
 
-**Why not the keyviz cluster fan-out** (`docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md`): that aggregates across nodes for the admin heatmap UI and dedupes write samples by `(groupID, leaderTerm)`. Pulling cross-node data in would add latency and a dependency on the fan-out being configured, so M3 stays leader-local and fail-closed on unknown load. If a future milestone wants complete cluster-wide scoring, including follower-forwarded writes, it can read the same fan-out aggregate behind the same detector interface (§5) — recorded as OQ-5.
+**Why not the keyviz cluster fan-out** (`docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md`): that aggregates across nodes for the admin heatmap UI and dedupes write samples by `(groupID, leaderTerm)`. Pulling cross-node data in would add latency and a dependency on the fan-out being configured, so M3 stays leader-local and fail-closed on unknown load. If a future milestone wants complete cluster-wide scoring, including follower-forwarded writes, it can read the same fan-out aggregate behind the same detector interface (§5) — recorded as OQ-5.
 
 **Known limitation — multi-leader (multi-group) deployments.** Leader-local consumption is only authoritative for routes whose shard group is led by the **same node** that runs the detector — i.e. the default-group leader (where the scheduler must run, §6.1). In a multi-group cluster (`--raftGroups` with G1, G2, …), the default-group leader is **not necessarily** the leader of G1/G2/…: each node runs a single `ShardedCoordinator` and `MemSampler` and only `Observe`s traffic it *receives* (`kv/sharded_coordinator.go:1795-1824`). Concretely, in a 3-node cluster where node A leads the default group and node B leads G1, **node A's sampler has no data for routes in G1 served through node B**, so the detector on A cannot score or split them. This is **broader than "heavy follower-forwarded reads"**: it is a structural gap for any route whose group leader differs from the default-group leader, not just an edge case of read forwarding.
 

docs/design/2026_06_23_proposed_scaling_roadmap.md

@@ -258,15 +258,15 @@ memory each group's private cache/memtable pins.
 ### (f) Operational scaling
 
 - **keyviz** — the per-route load sampler is wired and allocation-free on the
-  write hot path (`observeMutation`, `kv/sharded_coordinator.go:1841-1846`),
-  with proposed extensions for cluster fan-out
-  (`2026_04_27_proposed_keyviz_cluster_fanout.md`),
-  subrange sampling (`2026_05_25_implemented_keyviz_subrange_sampling.md`),
-  hot-key top-K (`2026_05_28_implemented_keyviz_hot_key_topk.md`), and per-cell
-  conflict (implemented). It is the detection signal M3 reuses. Current
-  adapter-direct Redis/DynamoDB/S3 reads that hit `MVCCStore.GetAt` bypass this
-  coordinator sampler, so read-heavy hotspots remain invisible until read-path
-  sampling or an equivalent adapter read observation path is added.
+  write hot path (`observeMutation`, `kv/sharded_coordinator.go:1841-1846`).
+  Shipped extensions include cluster fan-out
+  (`2026_04_27_implemented_keyviz_cluster_fanout.md`), subrange sampling
+  (`2026_05_25_implemented_keyviz_subrange_sampling.md`), hot-key top-K
+  (`2026_05_28_implemented_keyviz_hot_key_topk.md`), and per-cell conflict. It
+  is the detection signal M3 reuses. Current adapter-direct
+  Redis/DynamoDB/S3 reads that hit `MVCCStore.GetAt` bypass this coordinator
+  sampler, so read-heavy hotspots remain invisible until read-path sampling or
+  an equivalent adapter read observation path is added.
 - **admin** — admin dashboard / data browser / purge-queue are implemented
   (`2026_04_24_implemented_admin_dashboard.md`,
   `2026_05_22_implemented_admin_data_browser.md`,

internal/admin/keyviz_fanout.go

@@ -71,7 +71,7 @@ const keyVizMergeBucketHint = 64
 // a stable row order; Responded counts ok=true entries; Expected is
 // the configured peer count plus self.
 //
-// See docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md 5.
+// See docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md 5.
 type FanoutResult struct {
 	Nodes     []FanoutNodeStatus `json:"nodes"`
 	Responded int                `json:"responded"`

internal/admin/keyviz_handler.go

@@ -56,7 +56,7 @@ const keyVizRowBudgetCap = 1024
 //
 // Fanout is non-nil when the handler is configured for cluster-wide
 // fan-out (Phase 2-C): it carries per-node status so the SPA can
-// surface degraded responses inline (see design 2026_04_27_proposed_keyviz_cluster_fanout.md).
+// surface degraded responses inline (see design 2026_04_27_implemented_keyviz_cluster_fanout.md).
 // The field is omitted from the wire form when fan-out is disabled
 // so old clients keep working unchanged.
 type KeyVizMatrix struct {

main.go

@@ -236,7 +236,7 @@ var (
 	// HTTP endpoints (host:port or scheme://host:port). When set,
 	// the admin keyviz handler aggregates the local matrix with
 	// peer responses; when empty, behaviour is unchanged
-	// (single-node view). See docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md.
+	// (single-node view). See docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md.
 	keyvizFanoutNodes   = flag.String("keyvizFanoutNodes", "", "Comma-separated peer admin endpoints (host:port) for keyviz cluster-wide fan-out; empty disables")
 	keyvizFanoutTimeout = flag.Duration("keyvizFanoutTimeout", keyvizFanoutDefaultTimeout, "Per-peer timeout for keyviz fan-out HTTP calls")
 )

scripts/rolling-update.env.example

@@ -122,7 +122,7 @@ ADMIN_ENABLED="false"
 # role allow-lists must be configured cluster-wide. Peers without
 # --adminEnabled expose an unauthenticated keyviz endpoint and
 # respond unconditionally.
-# See docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md for the
+# See docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md for the
 # full design.
 KEYVIZ_ENABLED="false"
 # KEYVIZ_FANOUT_NODES="10.0.0.1:8080,10.0.0.2:8080,10.0.0.3:8080"