From 030ee62a6fa473a1974b194fb779e2fc28267fb6 Mon Sep 17 00:00:00 2001
From: Jeff Larson <thejefflarson@gmail.com>
Date: Fri, 17 Jul 2026 23:14:21 -0700
Subject: [PATCH] fix(chart): agent DaemonSet updateStrategy maxSurge:1 so a
 stuck node can't stall the roll (JEF-423)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The in-repo agent DaemonSet declared no updateStrategy, inheriting k8s's
stall-prone default (maxUnavailable:1, maxSurge:0): each node's old pod must
terminate before the next starts, so a single stuck/cordoned node (disk-pressure,
a wedged kernel that won't unload the old eBPF probes) blocks the whole roll and
drops runtime coverage to zero fleet-wide until an operator intervenes.

Adds spec.updateStrategy.rollingUpdate.maxSurge, values-driven via a new
agent.maxSurge (default 1). A replacement pod now comes up on healthy nodes WITHOUT
first tearing down the old one, so the roll proceeds past a bad node. Surge — not a
higher maxUnavailable — keeps every reachable node observed throughout the upgrade,
so coverage never dips below the running fleet. Set agent.maxSurge=0 to restore the
in-place default.

Chart YAML — untestable beyond template render + lint:
  helm template charts/protector --set agent.enabled=true → renders maxSurge: 1
  ...--set agent.maxSurge=2 → renders maxSurge: 2 (override honoured)
  helm lint charts/protector → clean

Claude-Session: https://claude.ai/code/session_01VtjoJttCvBY4dzCoE4f9vP

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
---
 charts/protector/templates/agent-daemonset.yaml | 12 ++++++++++++
 charts/protector/values.yaml                    |  7 +++++++
 2 files changed, 19 insertions(+)

diff --git a/charts/protector/templates/agent-daemonset.yaml b/charts/protector/templates/agent-daemonset.yaml
index 617b1d7..611048b 100644
--- a/charts/protector/templates/agent-daemonset.yaml
+++ b/charts/protector/templates/agent-daemonset.yaml
@@ -26,6 +26,18 @@ metadata:
   labels:
     {{- include "protector.agentLabels" . | nindent 4 }}
 spec:
+  # A stuck node (cordoned, disk-pressure, a wedged kernel that won't unload the old
+  # eBPF probes) must NOT stall the whole roll. The DaemonSet default —
+  # maxUnavailable:1, maxSurge:0 — waits for each node's old pod to terminate before the
+  # next starts, so ONE unkillable pod drops runtime coverage to zero fleet-wide until an
+  # operator intervenes. maxSurge:1 lets a replacement pod come up on healthy nodes
+  # WITHOUT first tearing down the old one, so the roll proceeds past the bad node instead
+  # of blocking on it. Surge (not a higher maxUnavailable) keeps every reachable node
+  # observed throughout the upgrade — coverage never dips below the running fleet.
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: {{ .Values.agent.maxSurge }}
   selector:
     matchLabels:
       {{- include "protector.agentSelectorLabels" . | nindent 6 }}
diff --git a/charts/protector/values.yaml b/charts/protector/values.yaml
index 5ff98af..96c88d4 100644
--- a/charts/protector/values.yaml
+++ b/charts/protector/values.yaml
@@ -370,6 +370,13 @@ agent:
   #   podAnnotations:
   #     linkerd.io/inject: enabled
   podAnnotations: {}
+  # Rolling-update surge for the DaemonSet. The k8s default (maxUnavailable:1,
+  # maxSurge:0) waits for each node's old pod to terminate before starting the new one,
+  # so a single stuck/cordoned node stalls the whole roll and drops runtime coverage to
+  # zero fleet-wide. maxSurge:1 brings a replacement pod up on healthy nodes without first
+  # tearing down the old one, so the roll proceeds past a bad node — coverage never dips
+  # below the running fleet during an upgrade. Set 0 to restore the default in-place roll.
+  maxSurge: 1
   # RAM-tight for arm64: in-kernel aggregation is cheap; userspace is a thin batcher.
   resources:
     requests: