From fffe770ecb33d2c5501344ac9810d5c68648fa44 Mon Sep 17 00:00:00 2001 From: pinkneyj Date: Fri, 3 Jul 2026 12:02:28 +0100 Subject: [PATCH 1/4] update affinity on server and aggregator to be automatically set --- helm/templates/agent-deploy.yaml | 4 +++- helm/templates/aggregator-deploy.yaml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/helm/templates/agent-deploy.yaml b/helm/templates/agent-deploy.yaml index 59ec7c3a..f748b59e 100644 --- a/helm/templates/agent-deploy.yaml +++ b/helm/templates/agent-deploy.yaml @@ -347,7 +347,9 @@ spec: {{- include "cloudzero-agent.generateImagePullSecrets" (dict "root" . "image" .Values.components.prometheus.image) | nindent 6 }} {{- include "cloudzero-agent.generateNodeSelector" (dict "default" .Values.defaults.nodeSelector "nodeSelector" .Values.server.nodeSelector) | nindent 6 }} {{- include "cloudzero-agent.generateTolerations" (concat .Values.defaults.tolerations .Values.server.tolerations) | nindent 6 }} - {{- include "cloudzero-agent.generateAffinity" (dict "default" .Values.defaults.affinity "affinity" .Values.server.affinity) | nindent 6 }} + {{- $podAntiAffinity := dict "podAntiAffinity" (dict "preferredDuringSchedulingIgnoredDuringExecution" (list (dict "weight" 100 "podAffinityTerm" (dict "labelSelector" (dict "matchLabels" (dict "app.kubernetes.io/name" "server")) "topologyKey" "kubernetes.io/hostname")))) }} + {{- $userAffinity := deepCopy (.Values.server.affinity | default (dict)) }} + {{- include "cloudzero-agent.generateAffinity" (dict "default" .Values.defaults.affinity "affinity" (merge $userAffinity $podAntiAffinity)) | nindent 6 }} {{- with .Values.server.topologySpreadConstraints }} topologySpreadConstraints: {{- toYaml . | nindent 8 }} diff --git a/helm/templates/aggregator-deploy.yaml b/helm/templates/aggregator-deploy.yaml index 4edf7b43..e4411741 100644 --- a/helm/templates/aggregator-deploy.yaml +++ b/helm/templates/aggregator-deploy.yaml @@ -177,7 +177,9 @@ spec: {{- include "cloudzero-agent.generateDNSInfo" (dict "defaults" .Values.defaults.dns) | nindent 6 }} {{- include "cloudzero-agent.generateImagePullSecrets" (dict "root" . "image" .Values.components.agent.image) | nindent 6 }} {{- include "cloudzero-agent.generateNodeSelector" (dict "default" .Values.defaults.nodeSelector "nodeSelector" .Values.aggregator.nodeSelector) | nindent 6 }} - {{- include "cloudzero-agent.generateAffinity" (dict "default" .Values.defaults.affinity "affinity" .Values.aggregator.affinity) | nindent 6 }} + {{- $podAntiAffinity := dict "podAntiAffinity" (dict "preferredDuringSchedulingIgnoredDuringExecution" (list (dict "weight" 100 "podAffinityTerm" (dict "labelSelector" (dict "matchLabels" (dict "app.kubernetes.io/name" "aggregator")) "topologyKey" "kubernetes.io/hostname")))) }} + {{- $userAffinity := deepCopy (.Values.aggregator.affinity | default (dict)) }} + {{- include "cloudzero-agent.generateAffinity" (dict "default" .Values.defaults.affinity "affinity" (merge $userAffinity $podAntiAffinity)) | nindent 6 }} {{- include "cloudzero-agent.generateTolerations" (concat .Values.defaults.tolerations) | nindent 6 }} {{- with .Values.server.topologySpreadConstraints }} From d32537a5b52ef510ee3a093c253876ca9b7b38f1 Mon Sep 17 00:00:00 2001 From: pinkneyj Date: Wed, 8 Jul 2026 10:16:21 +0100 Subject: [PATCH 2/4] add helm hook/delete to backfill,config loader, helmless and init cert jobs --- helm/templates/backfill-job.yaml | 25 +++++++------------------ helm/templates/config-loader-job.yaml | 4 +++- helm/templates/helmless-job.yaml | 4 +++- helm/templates/init-cert-job.yaml | 4 +++- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/helm/templates/backfill-job.yaml b/helm/templates/backfill-job.yaml index 06f62476..97ba240a 100644 --- a/helm/templates/backfill-job.yaml +++ b/helm/templates/backfill-job.yaml @@ -16,9 +16,12 @@ immediately upon helm install to capture existing resources without waiting for the first scheduled run. - Helm hooks (post-install, hook-delete-policy) were tried, but rejected because - they require Helm-specific features that don't work with helm template, - kubectl apply, ArgoCD, Flux, or other GitOps tools + The one-time Job uses Helm hooks (pre-install, pre-upgrade) so that it is not + tracked as a managed resource by ArgoCD or Helm, preventing OutOfSync errors + when the job completes and is deleted. The CronJob is left as a standard + resource as it manages its own lifecycle. + + Note: ArgoCD must be configured with --enable-helm-hooks for hooks to run. The implementation uses a range loop with small if/else clauses to maximize code reuse while handling the minimal differences between CronJob and Job @@ -26,21 +29,6 @@ */ -}} {{- range $jobType := list "CronJob" "Job" }} {{- $jobCategory := ternary "cronjob" "onetime" (eq $jobType "CronJob") }} -{{- /* - Istio mTLS Port Exclusion - - When Istio is detected and cert-manager is not in use, exclude port 443 from - Envoy proxying. The backfill job connects to the webhook service on port 443 - (the service port, which Kubernetes routes to pod port 8443). Without this - exclusion, Istio wraps the connection in mTLS, but the webhook uses a - self-signed certificate that Istio doesn't recognize, causing TLS failures. - - Port 443 is excluded (not 8443) because Istio's sidecar intercepts outbound - traffic based on the destination port the application connects to - the - service port, not the pod's target port. - - See helm/docs/istio.md for full documentation. -*/ -}} {{- $istioAnnotations := dict -}} {{- if and (include "cloudzero-agent.Values.integrations.istio.enabled" $) (not $.Values.insightsController.tls.useCertManager) -}} {{- $istioAnnotations = dict "traffic.sidecar.istio.io/excludeOutboundPorts" "443" -}} @@ -57,6 +45,7 @@ metadata: $.Values.initBackfillJob.annotations $.Values.components.webhookServer.annotations $.Values.components.webhookServer.backfill.annotations + (if (eq $jobType "Job") (dict "helm.sh/hook" "pre-install,pre-upgrade" "helm.sh/hook-delete-policy" "before-hook-creation") (dict)) ) ) | nindent 2 }} {{- include "cloudzero-agent.generateLabels" (dict diff --git a/helm/templates/config-loader-job.yaml b/helm/templates/config-loader-job.yaml index 3d11a303..7467d595 100644 --- a/helm/templates/config-loader-job.yaml +++ b/helm/templates/config-loader-job.yaml @@ -9,8 +9,10 @@ metadata: .Values.defaults.annotations .Values.components.miscellaneous.configLoader.annotations (dict "checksum/values" (include "cloudzero-agent.configurationChecksum" .)) + (dict "helm.sh/hook" "pre-install,pre-upgrade") + (dict "helm.sh/hook-delete-policy" "before-hook-creation") ) - ) | nindent 2 }} + ) | indent 2 }} {{- include "cloudzero-agent.generateLabels" (dict "root" . "name" "validator" diff --git a/helm/templates/helmless-job.yaml b/helm/templates/helmless-job.yaml index 5ecef5af..c7bfd98f 100644 --- a/helm/templates/helmless-job.yaml +++ b/helm/templates/helmless-job.yaml @@ -8,8 +8,10 @@ metadata: "annotations" (list .Values.defaults.annotations .Values.components.miscellaneous.helmless.annotations + (dict "helm.sh/hook" "pre-install,pre-upgrade") + (dict "helm.sh/hook-delete-policy" "before-hook-creation") ) - ) | nindent 2 }} + ) | indent 2 }} {{- include "cloudzero-agent.generateLabels" (dict "root" . "name" "helmless" diff --git a/helm/templates/init-cert-job.yaml b/helm/templates/init-cert-job.yaml index d20e0ec1..4aaa6db1 100644 --- a/helm/templates/init-cert-job.yaml +++ b/helm/templates/init-cert-job.yaml @@ -11,8 +11,10 @@ metadata: .Values.defaults.annotations .Values.initCertJob.annotations .Values.components.miscellaneous.initCert.annotations + (dict "helm.sh/hook" "pre-install,pre-upgrade") + (dict "helm.sh/hook-delete-policy" "before-hook-creation") ) - ) | nindent 2 }} + ) | indent 2 }} {{- include "cloudzero-agent.generateLabels" (dict "root" . "name" "init-cert" From 41bcd8306802172de3562af6465118ea01f64adf Mon Sep 17 00:00:00 2001 From: pinkneyj Date: Wed, 8 Jul 2026 14:01:40 +0100 Subject: [PATCH 3/4] minor yaml changes --- helm/templates/backfill-job.yaml | 2 +- helm/templates/config-loader-job.yaml | 2 +- helm/templates/helmless-job.yaml | 2 +- helm/templates/init-cert-job.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/templates/backfill-job.yaml b/helm/templates/backfill-job.yaml index 97ba240a..b233a1d7 100644 --- a/helm/templates/backfill-job.yaml +++ b/helm/templates/backfill-job.yaml @@ -45,7 +45,7 @@ metadata: $.Values.initBackfillJob.annotations $.Values.components.webhookServer.annotations $.Values.components.webhookServer.backfill.annotations - (if (eq $jobType "Job") (dict "helm.sh/hook" "pre-install,pre-upgrade" "helm.sh/hook-delete-policy" "before-hook-creation") (dict)) + (ternary (dict "helm.sh/hook" "pre-install,pre-upgrade" "helm.sh/hook-delete-policy" "before-hook-creation") (dict) (eq $jobType "Job")) ) ) | nindent 2 }} {{- include "cloudzero-agent.generateLabels" (dict diff --git a/helm/templates/config-loader-job.yaml b/helm/templates/config-loader-job.yaml index 7467d595..228c0d83 100644 --- a/helm/templates/config-loader-job.yaml +++ b/helm/templates/config-loader-job.yaml @@ -12,7 +12,7 @@ metadata: (dict "helm.sh/hook" "pre-install,pre-upgrade") (dict "helm.sh/hook-delete-policy" "before-hook-creation") ) - ) | indent 2 }} + ) | nindent 2 }} {{- include "cloudzero-agent.generateLabels" (dict "root" . "name" "validator" diff --git a/helm/templates/helmless-job.yaml b/helm/templates/helmless-job.yaml index c7bfd98f..01213d66 100644 --- a/helm/templates/helmless-job.yaml +++ b/helm/templates/helmless-job.yaml @@ -11,7 +11,7 @@ metadata: (dict "helm.sh/hook" "pre-install,pre-upgrade") (dict "helm.sh/hook-delete-policy" "before-hook-creation") ) - ) | indent 2 }} + ) | nindent 2 }} {{- include "cloudzero-agent.generateLabels" (dict "root" . "name" "helmless" diff --git a/helm/templates/init-cert-job.yaml b/helm/templates/init-cert-job.yaml index 4aaa6db1..a6c45652 100644 --- a/helm/templates/init-cert-job.yaml +++ b/helm/templates/init-cert-job.yaml @@ -14,7 +14,7 @@ metadata: (dict "helm.sh/hook" "pre-install,pre-upgrade") (dict "helm.sh/hook-delete-policy" "before-hook-creation") ) - ) | indent 2 }} + ) | nindent 2 }} {{- include "cloudzero-agent.generateLabels" (dict "root" . "name" "init-cert" From b76d60905985ff91b785a9c3d8f98b96da042ba9 Mon Sep 17 00:00:00 2001 From: pinkneyj Date: Wed, 8 Jul 2026 14:57:58 +0100 Subject: [PATCH 4/4] ensure other changes no included in this PR --- helm/templates/agent-deploy.yaml | 4 +--- helm/templates/aggregator-deploy.yaml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/helm/templates/agent-deploy.yaml b/helm/templates/agent-deploy.yaml index f748b59e..59ec7c3a 100644 --- a/helm/templates/agent-deploy.yaml +++ b/helm/templates/agent-deploy.yaml @@ -347,9 +347,7 @@ spec: {{- include "cloudzero-agent.generateImagePullSecrets" (dict "root" . "image" .Values.components.prometheus.image) | nindent 6 }} {{- include "cloudzero-agent.generateNodeSelector" (dict "default" .Values.defaults.nodeSelector "nodeSelector" .Values.server.nodeSelector) | nindent 6 }} {{- include "cloudzero-agent.generateTolerations" (concat .Values.defaults.tolerations .Values.server.tolerations) | nindent 6 }} - {{- $podAntiAffinity := dict "podAntiAffinity" (dict "preferredDuringSchedulingIgnoredDuringExecution" (list (dict "weight" 100 "podAffinityTerm" (dict "labelSelector" (dict "matchLabels" (dict "app.kubernetes.io/name" "server")) "topologyKey" "kubernetes.io/hostname")))) }} - {{- $userAffinity := deepCopy (.Values.server.affinity | default (dict)) }} - {{- include "cloudzero-agent.generateAffinity" (dict "default" .Values.defaults.affinity "affinity" (merge $userAffinity $podAntiAffinity)) | nindent 6 }} + {{- include "cloudzero-agent.generateAffinity" (dict "default" .Values.defaults.affinity "affinity" .Values.server.affinity) | nindent 6 }} {{- with .Values.server.topologySpreadConstraints }} topologySpreadConstraints: {{- toYaml . | nindent 8 }} diff --git a/helm/templates/aggregator-deploy.yaml b/helm/templates/aggregator-deploy.yaml index e4411741..4edf7b43 100644 --- a/helm/templates/aggregator-deploy.yaml +++ b/helm/templates/aggregator-deploy.yaml @@ -177,9 +177,7 @@ spec: {{- include "cloudzero-agent.generateDNSInfo" (dict "defaults" .Values.defaults.dns) | nindent 6 }} {{- include "cloudzero-agent.generateImagePullSecrets" (dict "root" . "image" .Values.components.agent.image) | nindent 6 }} {{- include "cloudzero-agent.generateNodeSelector" (dict "default" .Values.defaults.nodeSelector "nodeSelector" .Values.aggregator.nodeSelector) | nindent 6 }} - {{- $podAntiAffinity := dict "podAntiAffinity" (dict "preferredDuringSchedulingIgnoredDuringExecution" (list (dict "weight" 100 "podAffinityTerm" (dict "labelSelector" (dict "matchLabels" (dict "app.kubernetes.io/name" "aggregator")) "topologyKey" "kubernetes.io/hostname")))) }} - {{- $userAffinity := deepCopy (.Values.aggregator.affinity | default (dict)) }} - {{- include "cloudzero-agent.generateAffinity" (dict "default" .Values.defaults.affinity "affinity" (merge $userAffinity $podAntiAffinity)) | nindent 6 }} + {{- include "cloudzero-agent.generateAffinity" (dict "default" .Values.defaults.affinity "affinity" .Values.aggregator.affinity) | nindent 6 }} {{- include "cloudzero-agent.generateTolerations" (concat .Values.defaults.tolerations) | nindent 6 }} {{- with .Values.server.topologySpreadConstraints }}