From a1b1bbfb77d9fda37a223e9e60e8401bd54bc913 Mon Sep 17 00:00:00 2001 From: Muhammad Waqas Date: Sun, 19 Jul 2026 08:57:44 +0500 Subject: [PATCH] Gate Layer 2 on healthy k3s networking --- docs/DEPENDENCY-AUDIT-2026-07-19.md | 45 +++++++++++++++++++++++++++++ docs/README.md | 27 ++++++++--------- go.mod | 2 +- go.sum | 4 +-- internal/stack/options_test.go | 12 ++++++++ internal/stack/workflows.go | 3 +- 6 files changed, 76 insertions(+), 17 deletions(-) create mode 100644 docs/DEPENDENCY-AUDIT-2026-07-19.md diff --git a/docs/DEPENDENCY-AUDIT-2026-07-19.md b/docs/DEPENDENCY-AUDIT-2026-07-19.md new file mode 100644 index 0000000..c6e87dc --- /dev/null +++ b/docs/DEPENDENCY-AUDIT-2026-07-19.md @@ -0,0 +1,45 @@ +# Dependency audit: 2026-07-19 + +The local reference repositories and GitHub releases were fetched before this +comparison. Layer 1 is current; several Layer 2 charts have newer upstream +releases. + +| Component | Repository version | Current upstream | Decision | +|---|---:|---:|---| +| k3s | v1.36.2+k3s1 | v1.36.2+k3s1 | Current | +| NVIDIA GPU Operator | v26.3.3 | v26.3.3 | Current | +| Ollama Helm chart | 1.50.0 | 1.67.0 | Migration required | +| Ollama image | 0.17.7 | 0.32.1 | Migration required | +| Open WebUI Helm chart | 12.10.0 | 15.2.0 | Migration required | +| Open WebUI image | 0.8.10 | 0.10.2 | Migration required | +| kube-prometheus-stack | 86.2.3 | 87.17.0 | Review CRD/rule changes | +| OpenTelemetry Collector chart | 0.158.1 | 0.165.0 | Migration required | +| OpenTelemetry Operator chart | 0.115.0 | 0.120.0 | Review CRD changes | + +Sources: [k3s releases](https://github.com/k3s-io/k3s/releases), +[GPU Operator releases](https://github.com/NVIDIA/gpu-operator/releases), +[Ollama releases](https://github.com/ollama/ollama/releases), +[Ollama chart](https://github.com/otwld/ollama-helm), +[Open WebUI releases](https://github.com/open-webui/open-webui/releases), +[Open WebUI charts](https://github.com/open-webui/helm-charts), +[Prometheus Community charts](https://github.com/prometheus-community/helm-charts), +and [OpenTelemetry charts](https://github.com/open-telemetry/opentelemetry-helm-charts). + +## Validation result + +The existing pins passed module verification, formatting, unit tests, race +tests, vet, `govulncheck`, builds, Helm lint, and default/CPU/local/GeForce/full +NVIDIA rendering with Helm 4.2.3. GitHub's current `main` workflow is also +green. + +The live cluster was not suitable for a safe multi-component upgrade because a +post-reboot stale k3s node address and Flannel interface were breaking pod +networking. Upgrading the charts during that fault would mix configuration and +version variables and would not produce trustworthy evidence. The versions +above are therefore recorded as a dedicated migration, not changed in place. + +Layer 2 now depends on the Layer 1 commit that detects stale node networking and +unhealthy GPU Operator pods. Once the host network is repaired, upgrade and +validate the components in this order: observability CRDs/operators, +Prometheus, Ollama, then Open WebUI. Preserve the GeForce 940M resource limits, +model-cleanup rejection, existing PVCs, and rollback values between steps. diff --git a/docs/README.md b/docs/README.md index 9c9d1cf..069f27d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,19 +6,20 @@ The documentation is organized around private LLM deployment and observability o ## Start Here -1. [QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md](QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md) -2. [LIVE-VALIDATION-GO-NATIVE-2026-07-18.md](LIVE-VALIDATION-GO-NATIVE-2026-07-18.md) -3. [LIVE-VALIDATION-2026-07-17.md](LIVE-VALIDATION-2026-07-17.md) -4. [QUICKSTART.md](QUICKSTART.md) -5. [cli.md](cli.md) -6. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) -7. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) -8. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) -9. [ARCHITECTURE.md](ARCHITECTURE.md) -10. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) -11. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) -12. [LANGUAGE-BOUNDARIES.md](LANGUAGE-BOUNDARIES.md) -13. [GO-NATIVE-MIGRATION-2026-07-18.md](GO-NATIVE-MIGRATION-2026-07-18.md) +1. [DEPENDENCY-AUDIT-2026-07-19.md](DEPENDENCY-AUDIT-2026-07-19.md) +2. [QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md](QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md) +3. [LIVE-VALIDATION-GO-NATIVE-2026-07-18.md](LIVE-VALIDATION-GO-NATIVE-2026-07-18.md) +4. [LIVE-VALIDATION-2026-07-17.md](LIVE-VALIDATION-2026-07-17.md) +5. [QUICKSTART.md](QUICKSTART.md) +6. [cli.md](cli.md) +7. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) +8. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) +9. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) +10. [ARCHITECTURE.md](ARCHITECTURE.md) +11. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) +12. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) +13. [LANGUAGE-BOUNDARIES.md](LANGUAGE-BOUNDARIES.md) +14. [GO-NATIVE-MIGRATION-2026-07-18.md](GO-NATIVE-MIGRATION-2026-07-18.md) External companion: diff --git a/go.mod b/go.mod index 4b8c764..2667da9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/Edge-Computing-LLM/llm-observability-stack go 1.25.0 require ( - github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013 + github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260719035336-95f5e6cb9c29 github.com/prometheus/client_golang v1.23.2 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 diff --git a/go.sum b/go.sum index 8dfbe66..611dbf1 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013 h1:yxsjOaobZcrVBnbF/Usz5HQ77HExRcOVkCi2o8dZhgs= -github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013/go.mod h1:+7CsSGQtUt/2vgTNKT6NFOjx5gkIhrNIK4YFw4/IVyA= +github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260719035336-95f5e6cb9c29 h1:apfG7YGso9zbX4y8ztOoAAeYUv/9ANP2cnUHBZxH76w= +github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260719035336-95f5e6cb9c29/go.mod h1:+7CsSGQtUt/2vgTNKT6NFOjx5gkIhrNIK4YFw4/IVyA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= diff --git a/internal/stack/options_test.go b/internal/stack/options_test.go index 2ffe2a7..1f86338 100644 --- a/internal/stack/options_test.go +++ b/internal/stack/options_test.go @@ -57,6 +57,18 @@ func TestHelmInstallDisablesBaseLayerChartsForGPUProfiles(t *testing.T) { } } +func TestBaseReadinessRejectsStaleNetworkAndUnhealthyOperator(t *testing.T) { + all := "" + for _, step := range baseReadySteps(DefaultOptions()) { + all += step.Name + "\n" + step.Command + "\n" + } + for _, want := range []string{"k3s node address", "InternalIP", "GPU Operator health", "unhealthy pods"} { + if !stringsContains(all, want) { + t.Fatalf("base readiness missing %q", want) + } + } +} + func TestInstallWithBaseIsDeprecated(t *testing.T) { opts := DefaultOptions() opts.WithBase = true diff --git a/internal/stack/workflows.go b/internal/stack/workflows.go index 173333e..4f46d83 100644 --- a/internal/stack/workflows.go +++ b/internal/stack/workflows.go @@ -180,13 +180,14 @@ func baseReady(ctx context.Context, r *edgebase.Runner, opts Options) error { func baseReadySteps(opts Options) []edgebase.Step { steps := []edgebase.Step{ {Name: "Kubernetes connectivity", Command: "kubectl cluster-info"}, + {Name: "k3s node address", Command: edgebase.NodeAddressHealthCheck()}, {Name: "k3s nodes", Command: "kubectl get nodes -o wide"}, } if GPUProfile(opts.Profile) { steps = append(steps, edgebase.Step{Name: "NVIDIA RuntimeClass", Command: "kubectl get runtimeclass nvidia"}, edgebase.Step{Name: "NVIDIA GPU allocatable", Command: edgebase.GPUCapacityCheck()}, - edgebase.Step{Name: "GPU Operator namespace", Command: "kubectl get pods -n gpu-operator -o wide"}, + edgebase.Step{Name: "GPU Operator health", Command: edgebase.GPUOperatorHealthCheck()}, ) } return steps