From 2b327a8210dfd690bc12377f05f663efb25f6cb6 Mon Sep 17 00:00:00 2001 From: Muhammad Waqas Date: Wed, 8 Jul 2026 19:47:19 +0500 Subject: [PATCH 1/6] Validate observability as Layer 2 --- .gitignore | 6 +++ .helmignore | 3 ++ Chart.lock | 13 +----- Chart.yaml | 15 ------- README.md | 43 +++++++++++++------ cmd/llm-observability/main.go | 6 +-- docs/ARCHITECTURE.md | 15 ++++--- docs/LIVE-VALIDATION-2026-07-08.md | 31 +++++++++++++ docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md | 4 +- docs/LOCAL-K3S-NVIDIA-RUNBOOK.md | 12 ++++-- docs/SINGLE-NODE-K3S-GEFORCE-940M.md | 8 ++-- docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md | 5 ++- docs/cli.md | 19 ++++---- hack/detect-runtime-profile.sh | 2 +- internal/stack/options_test.go | 15 ++++++- internal/stack/workflows.go | 17 ++------ ...bility-stack-run-instructions-with-git.txt | 8 ++-- templates/NOTES.txt | 8 ++-- templates/validate-config.yaml | 9 ++++ tests/test_helm_smoke.py | 36 ++++++++++++++-- values.enterprise-pilot-k3s.yaml | 6 +-- values.full-stack-nvidia.example.yaml | 4 +- values.yaml | 9 ++-- 23 files changed, 191 insertions(+), 103 deletions(-) create mode 100644 docs/LIVE-VALIDATION-2026-07-08.md diff --git a/.gitignore b/.gitignore index 6e4c582..2f0edf6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ values.local-*.yaml *.key *.crt *.kubeconfig +kubeconfig +kubeconfig* # Generated render/debug artifacts .generated/ @@ -43,6 +45,9 @@ pictures/ charts/*.tgz charts/**/*.tgz monitoring/charts/*.tgz +llm-observability-stack-*.tgz +*.tar.gz +*.zip # Large model/data artifacts (keep out of git) *.gguf *.safetensors @@ -84,6 +89,7 @@ rendered-with-etcd.yaml *.private.yaml jupyter-notebooks/python*-pip-list.txt local-run-output/ +validation-output/ *.kube-report.json *.kubeconfig.bak evidence-local-run/ diff --git a/.helmignore b/.helmignore index 3aef344..e3f6971 100644 --- a/.helmignore +++ b/.helmignore @@ -61,6 +61,9 @@ python-toolbox/examples/service_dns_check.py # Vendored chart docs/assets are not needed at install time charts/*.tgz +charts/gpu-operator/ +charts/nvidia-device-plugin/ +charts/dcgm-exporter/ charts/ollama/README.md charts/ollama/LICENSE charts/ollama/artifacthub-repo.yml diff --git a/Chart.lock b/Chart.lock index 9aa76b9..127ceb1 100644 --- a/Chart.lock +++ b/Chart.lock @@ -1,13 +1,4 @@ dependencies: -- name: gpu-operator - repository: file://charts/gpu-operator - version: v1.0.0-devel -- name: nvidia-device-plugin - repository: file://charts/nvidia-device-plugin - version: 0.19.2 -- name: dcgm-exporter - repository: file://charts/dcgm-exporter - version: 4.8.2 - name: ollama repository: file://charts/ollama version: 1.50.0 @@ -23,5 +14,5 @@ dependencies: - name: opentelemetry-operator repository: file://charts/opentelemetry-operator version: 0.115.0 -digest: sha256:7a8de8212f1051cbddc881e9d7d865424f83be907ccd7d5da91491038579eba5 -generated: "2026-06-16T14:22:44.167855506+05:00" +digest: sha256:9a3e5dc71273d49fd4f3e8a46e8142ca5c83ffb8163c9666c97f4a558af01480 +generated: "2026-07-08T19:18:56.195921069+05:00" diff --git a/Chart.yaml b/Chart.yaml index fd26760..daef376 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -13,9 +13,6 @@ kubeVersion: ">=1.27.0-0" home: https://github.com/Edge-Computing-LLM/llm-observability-stack sources: - https://github.com/prometheus-community/helm-charts - - https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/ - - https://github.com/NVIDIA/k8s-device-plugin - - https://github.com/NVIDIA/dcgm-exporter - https://github.com/open-telemetry/opentelemetry-helm-charts - https://github.com/otwld/ollama-helm - https://github.com/open-webui/helm-charts @@ -37,18 +34,6 @@ keywords: - opentelemetry - genai dependencies: - - name: gpu-operator - version: v1.0.0-devel - repository: file://charts/gpu-operator - condition: gpu-operator.enabled - - name: nvidia-device-plugin - version: 0.19.2 - repository: file://charts/nvidia-device-plugin - condition: nvidia-device-plugin.enabled - - name: dcgm-exporter - version: 4.8.2 - repository: file://charts/dcgm-exporter - condition: dcgm-exporter.enabled - name: ollama version: 1.50.0 repository: file://charts/ollama diff --git a/README.md b/README.md index 6791d66..69ee2ee 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,12 @@ Kubernetes-native observability, benchmarking, and operations tooling for private LLM inference on local edge systems. -Go CLI documentation: [docs/cli.md](docs/cli.md). +Preferred organization CLI: [`edge-cli`](https://github.com/Edge-Computing-LLM/edge-cli). +Repo-local legacy/helper CLI documentation: [docs/cli.md](docs/cli.md). -This repository packages a Helm-based stack for k3s and Kubernetes with Ollama/GGUF model serving, Open WebUI, an OpenTelemetry GenAI-instrumented FastAPI proxy, Prometheus, Grafana, OpenTelemetry Collector, blackbox probes, benchmark metrics, and optional NVIDIA GPU monitoring through the NVIDIA runtime, device plugin, GPU Operator, and DCGM-compatible dashboards. +This repository packages a Helm-based application and observability stack for k3s and Kubernetes with Ollama/GGUF model serving, Open WebUI, an OpenTelemetry GenAI-instrumented FastAPI proxy, Prometheus, Grafana, OpenTelemetry Collector, blackbox probes, benchmark metrics, and NVIDIA/DCGM-compatible dashboards. -The repository also includes a Go CLI named `llm-observability`. It reuses `github.com/Edge-Computing-LLM/k3s-nvidia-edge/pkg/edgebase` for base-layer checks and keeps application-specific Helm, validation, and benchmark workflows in this repository. +The repository also includes a Go CLI named `llm-observability` for repo-local helper workflows. New end-to-end installs should use `edge-cli`, which deploys `k3s-nvidia-edge` first and then this chart. GitHub repository: @@ -16,6 +17,8 @@ For local NVIDIA GPU deployments, deploy [`k3s-nvidia-edge`](https://github.com/ `k3s-nvidia-edge` owns k3s, k3s containerd NVIDIA runtime wiring, GPU Operator, NVIDIA device plugin, DCGM exporter, Node Feature Discovery, `RuntimeClass/nvidia`, and the allocatable `nvidia.com/gpu` resource. `llm-observability-stack` then deploys Ollama, Open WebUI, OpenTelemetry, dashboards, benchmarks, and application-level observability on top of that base layer. +An empty local k3s cluster with only CoreDNS, local-path-provisioner, and other default k3s system components is a valid starting point before the base layer is deployed. Run `edge install infra` or validate `k3s-nvidia-edge` before installing GPU profiles from this repository. + Read the full dependency guide before installing GPU profiles: - [k3s-nvidia-edge dependency](docs/K3S-NVIDIA-EDGE-DEPENDENCY.md) @@ -25,7 +28,7 @@ Read the full dependency guide before installing GPU profiles: - Local private LLM serving through Ollama and legally obtained GGUF models. - Kubernetes deployment through Helm with k3s-friendly profiles. - NVIDIA GPU scheduling with `runtimeClassName: nvidia` and `nvidia.com/gpu` when a GPU is available. -- CPU-only deployment profiles for MacOS/minikube and k3s systems without NVIDIA GPUs. +- Optional CPU validation profiles for development clusters without NVIDIA GPUs. - Open WebUI for browser-based interaction with local models. - A FastAPI proxy with LLM request metrics for TTFT, latency, tokens per second, prompt tokens, generated tokens, active requests, and errors. - Prometheus, Grafana, Alertmanager, kube-state-metrics, node exporter, ServiceMonitors, probes, and alert rules. @@ -58,6 +61,7 @@ Evidence and reproduction: - [Single-node k3s GeForce 940M guide](docs/SINGLE-NODE-K3S-GEFORCE-940M.md) - [Local k3s NVIDIA deployment report - 2026-07-02](docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md) +- [Live layered validation - 2026-07-08](docs/LIVE-VALIDATION-2026-07-08.md) - [Verified local GPU results](docs/VERIFIED-LOCAL-GPU-RESULTS.md) - [Xubuntu k3s NVIDIA runbook](docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md) - [Sanitized benchmark artifact](artifacts/geforce-940m-benchmark.json) @@ -82,7 +86,7 @@ These numbers prove constrained local edge feasibility. They do not claim enterp ## Platform Components -- Vendored Helm charts for Ollama, Open WebUI, NVIDIA GPU Operator, NVIDIA device plugin, DCGM exporter, kube-prometheus-stack, OpenTelemetry Collector, and OpenTelemetry Operator. +- Vendored Helm charts for Ollama, Open WebUI, kube-prometheus-stack, OpenTelemetry Collector, and OpenTelemetry Operator. - FastAPI OpenTelemetry GenAI-instrumented proxy with Prometheus metrics. - TTFT, latency, token, throughput, active-request, HTTP, and error telemetry. - Optional kube-prometheus-stack, Grafana, Alertmanager, node exporter, and kube-state-metrics from the root umbrella chart. @@ -115,7 +119,7 @@ Prometheus + Grafana + Alertmanager +-- ServiceMonitors, probes, benchmark Pushgateway, Kubernetes metrics ``` -The verified laptop profile uses Ollama/GGUF. The same observability contract can be used on larger RTX workstations, GPU Operator/DCGM clusters, NIM endpoints, or cloud GPU clusters. +The verified laptop profile uses Ollama/GGUF. The same observability contract can be used on larger local RTX workstations with the NVIDIA substrate prepared by `k3s-nvidia-edge`. ## Repository Layout @@ -149,7 +153,16 @@ Build the CLI: go build -o bin/llm-observability ./cmd/llm-observability ``` -Recommended local CLI flow when `k3s-nvidia-edge` is already healthy: +Preferred local CLI flow from the organization control plane: + +```bash +edge install infra --yes +edge validate infra +edge install observability --profile geforce-940m-k3s --yes +edge validate observability +``` + +Repo-local helper flow when `k3s-nvidia-edge` is already healthy: ```bash bin/llm-observability doctor @@ -163,7 +176,7 @@ bin/llm-observability validate - Helm 3 or 4. - For local NVIDIA k3s GPU profiles: `k3s-nvidia-edge` deployed and validated first. - NVIDIA driver and NVIDIA Container Toolkit for GPU profiles. -- NVIDIA device plugin or GPU Operator exposing `nvidia.com/gpu` for GPU mode. +- `RuntimeClass/nvidia` and `nvidia.com/gpu` provided by `k3s-nvidia-edge` for GPU mode. - A legally obtained GGUF model available on node storage. - Python 3.11 for tests and benchmark tooling. @@ -193,12 +206,11 @@ helm template llm-observability-stack . \ Review the machine-specific model host path before using this profile on another system. The profile schedules on nodes with `nvidia.com/gpu.present=true`, which supports a single-node k3s control-plane/worker laptop without requiring a separate worker label. -Deploy and validate `k3s-nvidia-edge` first: +Preferred: deploy and validate the base layer through `edge-cli` first: ```bash -cd /media/waqasm86/External1/Waqas-Projects/Project-Edge-Computing-LLM/k3s-nvidia-edge -bin/k3s-nvidia-edge install --yes --sudo=false --use-local-chart --skip-base-package-install --skip-toolkit-install --skip-k3s-install -bin/k3s-nvidia-edge validate --yes +edge install infra --yes +edge validate infra ``` Then deploy the LLM stack: @@ -213,7 +225,7 @@ helm upgrade --install llm-observability-stack . \ ./hack/test-geforce-940m-inference.sh ``` -### C. Full-stack NVIDIA profile +### C. Full observability NVIDIA profile ```bash helm upgrade --install llm-observability-stack . \ @@ -221,7 +233,10 @@ helm upgrade --install llm-observability-stack . \ -f values.full-stack-nvidia.example.yaml ``` -Use private values files or existing Kubernetes Secrets for OpenTelemetry and Open WebUI secrets. Never commit secrets. +This installs the LLM and observability layer only. It does not install GPU +Operator, NVIDIA device plugin, or DCGM exporter. Use private values files or +existing Kubernetes Secrets for OpenTelemetry and Open WebUI secrets. Never +commit secrets. ### D. Local full-stack k3s profile diff --git a/cmd/llm-observability/main.go b/cmd/llm-observability/main.go index 88f6a18..56a218f 100644 --- a/cmd/llm-observability/main.go +++ b/cmd/llm-observability/main.go @@ -31,7 +31,7 @@ Common flags: --release NAME Helm release name (default "llm-observability-stack") --values FILE additional values file; may be repeated --set KEY=VALUE additional Helm --set override; may be repeated - --with-base install or uninstall k3s-nvidia-edge base layer too + --with-base deprecated; use edge-cli to install or uninstall the base layer --skip-base do not install base, but still check it for GPU profiles --dry-run print mutating commands without executing them --yes execute mutating commands @@ -44,7 +44,7 @@ Examples: llm-observability validate llm-observability benchmark --model gemma3-1b-it-gguf-local --runs 3 llm-observability uninstall --yes - llm-observability uninstall --with-base --yes + edge uninstall all --yes ` func main() { @@ -63,7 +63,7 @@ func main() { fs.StringVar(&opts.Release, "release", opts.Release, "Helm release name") fs.Var(&values, "values", "additional Helm values file; may be repeated") fs.Var(&sets, "set", "additional Helm --set override; may be repeated") - fs.BoolVar(&opts.WithBase, "with-base", false, "include k3s-nvidia-edge base install/uninstall") + fs.BoolVar(&opts.WithBase, "with-base", false, "deprecated; use edge-cli for k3s-nvidia-edge base operations") fs.BoolVar(&opts.SkipBase, "skip-base", false, "skip base installation") fs.BoolVar(&opts.DryRun, "dry-run", false, "print mutating commands without executing them") fs.BoolVar(&opts.Yes, "yes", false, "execute mutating commands") diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a3c9e0f..b752af3 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -4,7 +4,9 @@ This document explains how `llm-observability-stack` is put together, which comp For local NVIDIA GPU deployments, this repository is not the cluster bootstrap layer. Deploy and validate `k3s-nvidia-edge` first, then deploy `llm-observability-stack` on top of the ready k3s/NVIDIA substrate. See [k3s-nvidia-edge dependency](K3S-NVIDIA-EDGE-DEPENDENCY.md). -The Go CLI in `cmd/llm-observability` follows the same boundary. It imports `github.com/Edge-Computing-LLM/k3s-nvidia-edge/pkg/edgebase` for base-layer workflows and keeps LLM stack workflows in `internal/stack`. +The preferred end-to-end CLI is `edge-cli`. The Go CLI in `cmd/llm-observability` +is a repo-local helper for this chart and should not be used as the primary +cross-repository installer. ## 1. Design Goals @@ -95,14 +97,17 @@ Optional root-level resource set. It supports Open WebUI websocket/state flows w Optional and disabled by default. These exist for troubleshooting and demo scenarios, not for the default happy path. -### 2.8 NVIDIA and observability operators +### 2.8 NVIDIA Metrics Integration -The root chart vendors NVIDIA-related charts for portability and non-local profiles, but the verified local k3s/NVIDIA path treats `k3s-nvidia-edge` as the owner of the GPU substrate. Keep GPU Operator, NVIDIA device plugin, and DCGM exporter disabled here when `k3s-nvidia-edge` is installed. +The root chart no longer vendors NVIDIA substrate charts. The verified local +k3s/NVIDIA path treats `k3s-nvidia-edge` as the owner of GPU Operator, NVIDIA +device plugin, Node Feature Discovery, RuntimeClass, allocatable GPU resources, +and DCGM exporter. Available integration points: -- `gpu-operator` for clusters that need NVIDIA driver/toolkit/device-plugin/DCGM lifecycle management. -- `nvidia-device-plugin` plus `dcgm-exporter` for lightweight workstation or k3s clusters where host drivers and container runtime are already installed. +- `monitoring.dcgmExporter.serviceMonitor` can observe an existing DCGM exporter + Service created by the base layer. - `kube-prometheus-stack` for Prometheus Operator, Prometheus, Grafana, Alertmanager, node exporter, and kube-state-metrics. - `opentelemetry-collector` for a directly managed OTLP collector Deployment and Service. - `opentelemetry-operator` remains available for clusters that need an operator-managed `OpenTelemetryCollector` custom resource. diff --git a/docs/LIVE-VALIDATION-2026-07-08.md b/docs/LIVE-VALIDATION-2026-07-08.md new file mode 100644 index 0000000..ce430ba --- /dev/null +++ b/docs/LIVE-VALIDATION-2026-07-08.md @@ -0,0 +1,31 @@ +# Live Validation - 2026-07-08 + +Validated as Layer 2 on local Xubuntu 24, single-node k3s, NVIDIA GeForce 940M. + +Prerequisite: + +```bash +edge validate infra +``` + +Tested commands: + +```bash +edge install observability --profile geforce-940m-k3s --yes +edge validate observability +edge status +edge logs --tail 30 +edge uninstall observability --yes +edge install observability --profile geforce-940m-k3s --yes +``` + +Results: + +- Deployed Ollama, Open WebUI, Open WebUI Redis, and OpenTelemetry Collector in `llm-observability`. +- Ollama used `runtimeClassName: nvidia` and `nvidia.com/gpu: 1`. +- Ollama loaded `gemma3-1b-it-gguf-local` and smoke validation returned `validation ok`. +- No GPU Operator, NVIDIA device plugin, or DCGM exporter resources were rendered by this chart. +- Helm package did not contain old `charts/gpu-operator`, `charts/nvidia-device-plugin`, or `charts/dcgm-exporter` dependencies. +- DCGM exporter remained owned by Layer 1 in namespace `gpu-operator`. + +The GeForce 940M profile is valid for the tested low-VRAM workstation flow when Layer 1 is ready first. diff --git a/docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md b/docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md index eabb596..4438d1b 100644 --- a/docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md +++ b/docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md @@ -53,8 +53,8 @@ The prompt path `/media/waqasm86/External1/Project-Llamatelemetry/Project-Llamat Observed alignment: -- `llm-observability-stack` keeps GPU Operator, NVIDIA device plugin, DCGM exporter, kube-prometheus-stack, OpenTelemetry, Ollama, and Open WebUI as vendored Helm dependencies. -- The local deployment correctly avoids installing duplicate NVIDIA device-plugin/operator components when the cluster already has GPU Operator. +- `llm-observability-stack` now keeps GPU Operator, NVIDIA device plugin, and DCGM exporter out of its Helm dependency list. Those remain owned by `k3s-nvidia-edge`. +- The local deployment avoids installing duplicate NVIDIA device-plugin/operator components when the cluster already has GPU Operator. - The local profile now follows the single-node k3s reality by using GPU capability detection instead of requiring a `node-role.kubernetes.io/worker=true` selector. ## Project Changes Applied diff --git a/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md b/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md index 6a00f4f..f8fe478 100644 --- a/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md +++ b/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md @@ -1,6 +1,9 @@ # Local k3s NVIDIA Runbook -This runbook deploys `llm-observability-stack` on a local Xubuntu 24 host with k3s, NVIDIA GPU support, local GGUF storage, Ollama, Open WebUI, Prometheus/Grafana, DCGM exporter, and the vendored OpenTelemetry Collector chart. +This runbook deploys `llm-observability-stack` on a local Xubuntu 24 host after +`k3s-nvidia-edge` has prepared k3s, the NVIDIA runtime, GPU Operator, device +plugin, RuntimeClass, and DCGM exporter. This chart installs Ollama, Open WebUI, +Prometheus/Grafana, OpenTelemetry Collector, and LLM observability resources. ## 1. Enter the Project @@ -28,10 +31,12 @@ kubectl get pods -A | grep -Ei 'nvidia|device-plugin|dcgm' kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" gpu="}{.status.allocatable.nvidia\.com/gpu}{"\n"}{end}' ``` -If the device plugin is not installed, install it using the local helper: +If `RuntimeClass/nvidia` or `nvidia.com/gpu` is missing, install or repair the +base layer first: ```bash -./hack/install-nvidia-device-plugin.sh +edge install infra --yes +edge validate infra ``` ## 4. Verify the Local GGUF Model @@ -77,7 +82,6 @@ docker.io/library/busybox:1.38.0 ghcr.io/jkroepke/kube-webhook-certgen:1.8.3 ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s:0.153.0 ghcr.io/open-webui/open-webui:0.8.10 -nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless ollama/ollama:0.17.7 quay.io/kiwigrid/k8s-sidecar:2.7.3 quay.io/prometheus-operator/prometheus-operator:v0.91.0 diff --git a/docs/SINGLE-NODE-K3S-GEFORCE-940M.md b/docs/SINGLE-NODE-K3S-GEFORCE-940M.md index a1551e4..d6f8e18 100644 --- a/docs/SINGLE-NODE-K3S-GEFORCE-940M.md +++ b/docs/SINGLE-NODE-K3S-GEFORCE-940M.md @@ -8,13 +8,13 @@ and host-path ownership would make that topology unreliable. ## Prepare the combined node ```bash -./hack/prepare-single-node-k3s.sh -./hack/install-nvidia-device-plugin.sh +edge install infra --yes +edge validate infra kubectl get nodes --show-labels ``` -The scripts retain the existing control-plane role and add worker, GPU-present, and model-host -labels. The device plugin must expose one `nvidia.com/gpu` before Ollama is installed. +The base layer must expose one `nvidia.com/gpu` and `RuntimeClass/nvidia` before +Ollama is installed. ## Deploy the exact local GGUF model diff --git a/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md b/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md index 3b1ea79..e81db37 100644 --- a/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md +++ b/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md @@ -107,7 +107,10 @@ kubectl exec -n llm-observability deploy/ollama -- ollama list ## Full Local k3s/NVIDIA Profile -This profile runs Ollama, Open WebUI, LangChain proxy, Python toolbox, OpenTelemetry Collector, Prometheus, Grafana, Alertmanager, node exporter, kube-state-metrics, and DCGM exporter. +This profile runs Ollama, Open WebUI, LangChain proxy, Python toolbox, +OpenTelemetry Collector, Prometheus, Grafana, Alertmanager, node exporter, and +kube-state-metrics. It may observe the DCGM exporter from the base layer through +ServiceMonitor resources, but it does not deploy DCGM exporter. ```bash MODEL_DIR=/media/waqasm86/External1/Waqas-Projects/repos-llamatelemetry/llamatelemetry-xubuntu24/models diff --git a/docs/cli.md b/docs/cli.md index 45594ab..9606e05 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,14 +1,14 @@ # llm-observability CLI -`llm-observability` is the Go CLI for this repository. It manages the LLM application and observability layer that runs on top of the `k3s-nvidia-edge` base layer. +`llm-observability` is the repo-local Go helper CLI. Use `edge-cli` as the +primary organization-level CLI for end-to-end installs and uninstalls. ## Architecture Boundary - `k3s-nvidia-edge` owns the local Linux, k3s, k3s containerd, NVIDIA runtime, GPU Operator, NVIDIA device plugin, DCGM exporter, Node Feature Discovery, `RuntimeClass/nvidia`, and `nvidia.com/gpu` validation layer. - `llm-observability-stack` owns Ollama, Open WebUI, Open WebUI Redis, OpenTelemetry Collector, optional Prometheus/Grafana, optional LangChain/FastAPI proxy, benchmark tooling, notebooks, and local model configuration. -- The `llm-observability` CLI imports reusable base workflows from `github.com/Edge-Computing-LLM/k3s-nvidia-edge/pkg/edgebase`. +- `edge-cli` owns cross-repository ordering: infra first, observability second. - The CLI does not import `k3s-nvidia-edge/internal/...`. -- This is an incremental step toward a future organization-level CLI spanning Edge-Computing-LLM repositories. ## Build @@ -35,7 +35,6 @@ bin/llm-observability status bin/llm-observability validate bin/llm-observability benchmark --model gemma3-1b-it-gguf-local --runs 3 bin/llm-observability uninstall --yes -bin/llm-observability uninstall --with-base --yes bin/llm-observability print-commands --profile geforce-940m-k3s ``` @@ -86,10 +85,12 @@ bin/llm-observability install --profile geforce-940m-k3s --skip-base --yes bin/llm-observability validate ``` -If the base layer is not installed yet: +If the base layer is not installed yet, use `edge-cli`: ```bash -bin/llm-observability install --profile geforce-940m-k3s --with-base --yes +edge install infra --yes +edge validate infra +edge install observability --profile geforce-940m-k3s --yes ``` By default, install and uninstall commands are dry-run unless `--yes` is provided. @@ -137,10 +138,10 @@ Keep the namespace: bin/llm-observability uninstall --keep-namespace --yes ``` -Also uninstall the base GPU layer: +Remove all layers through `edge-cli` when you need reverse-order cleanup: ```bash -bin/llm-observability uninstall --with-base --yes +edge uninstall all --yes ``` -The base layer is never removed unless `--with-base` is explicitly passed. +The repo-local `--with-base` flag is deprecated and returns an error. diff --git a/hack/detect-runtime-profile.sh b/hack/detect-runtime-profile.sh index 8ae5e23..1c619d2 100755 --- a/hack/detect-runtime-profile.sh +++ b/hack/detect-runtime-profile.sh @@ -104,7 +104,7 @@ ollama: number: 1 nvidiaResource: ${GPU_RESOURCE} dcgm-exporter: - enabled: true + enabled: false monitoring: dcgmExporter: serviceMonitor: diff --git a/internal/stack/options_test.go b/internal/stack/options_test.go index 04c1f7c..c218e00 100644 --- a/internal/stack/options_test.go +++ b/internal/stack/options_test.go @@ -1,6 +1,9 @@ package stack -import "testing" +import ( + "context" + "testing" +) func TestProfileValuesFile(t *testing.T) { tests := map[string]string{ @@ -44,6 +47,16 @@ func TestHelmInstallDisablesBaseLayerChartsForGPUProfiles(t *testing.T) { } } +func TestInstallWithBaseIsDeprecated(t *testing.T) { + opts := DefaultOptions() + opts.WithBase = true + opts.DryRun = true + err := Install(context.Background(), opts) + if err == nil || !stringsContains(err.Error(), "edge-cli") { + t.Fatalf("Install with --with-base error = %v, want edge-cli deprecation", err) + } +} + func stringsContains(s, sub string) bool { for i := 0; i+len(sub) <= len(s); i++ { if s[i:i+len(sub)] == sub { diff --git a/internal/stack/workflows.go b/internal/stack/workflows.go index 729e381..01b5f83 100644 --- a/internal/stack/workflows.go +++ b/internal/stack/workflows.go @@ -27,17 +27,10 @@ func Install(ctx context.Context, opts Options) error { } r := runner(opts) if opts.WithBase { - baseOpts := baseOptions(opts) - baseRunner := edgebase.NewRunner(baseOpts) - if err := edgebase.Doctor(ctx, baseRunner, baseOpts); err != nil { - return err - } - if err := edgebase.Install(ctx, baseRunner, baseOpts); err != nil { - return err - } + return fmt.Errorf("--with-base is deprecated for llm-observability-stack. Use edge-cli and run edge install infra before edge install observability") } else if GPUProfile(opts.Profile) { if err := baseReady(ctx, r, opts); err != nil { - return fmt.Errorf("base NVIDIA layer is not ready; run with --with-base to install it or fix k3s-nvidia-edge first: %w", err) + return fmt.Errorf("base NVIDIA layer is not ready; run edge install infra or validate k3s-nvidia-edge first: %w", err) } } for _, step := range installSteps(opts) { @@ -91,11 +84,7 @@ func Uninstall(ctx context.Context, opts Options) error { } } if opts.WithBase { - baseOpts := baseOptions(opts) - baseOpts.UninstallK3s = false - if err := edgebase.Uninstall(ctx, edgebase.NewRunner(baseOpts), baseOpts); err != nil { - return err - } + return fmt.Errorf("--with-base is deprecated for llm-observability-stack. Use edge-cli uninstall all for reverse-order layer removal") } return nil } diff --git a/llm-observability-stack-run-instructions-with-git.txt b/llm-observability-stack-run-instructions-with-git.txt index 0daf896..54d61a9 100644 --- a/llm-observability-stack-run-instructions-with-git.txt +++ b/llm-observability-stack-run-instructions-with-git.txt @@ -3,7 +3,7 @@ LLM Observability Stack Run and Git Publish Instructions Target system: - Xubuntu 24 - local k3s Kubernetes -- NVIDIA GPU runtime and device plugin +- NVIDIA GPU runtime and device plugin from k3s-nvidia-edge - project path: /media/waqasm86/External1/Waqas-Projects/Project-Edge-Computing-LLM/llm-observability-stack @@ -27,10 +27,10 @@ Check that k3s sees the NVIDIA GPU: kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" gpu="}{.status.allocatable.nvidia\.com/gpu}{"\n"}{end}' -If gpu= is empty, prepare the node and install the NVIDIA device plugin: +If gpu= is empty, prepare and validate the base layer first: -./hack/prepare-single-node-k3s.sh -./hack/install-nvidia-device-plugin.sh +edge install infra --yes +edge validate infra 3. Verify the local GGUF model exists diff --git a/templates/NOTES.txt b/templates/NOTES.txt index d39fa5d..c669c41 100644 --- a/templates/NOTES.txt +++ b/templates/NOTES.txt @@ -1,13 +1,15 @@ {{- $runtime := default "nvidia" .Values.runtime.accelerator -}} 1. Runtime profile: {{ $runtime }} {{- if eq $runtime "nvidia" }} -2. NVIDIA mode requires the NVIDIA device plugin and an allocatable GPU resource. +2. NVIDIA mode requires Layer 1 (`k3s-nvidia-edge`) to provide RuntimeClass/nvidia and allocatable nvidia.com/gpu. {{- else }} 2. CPU mode disables NVIDIA runtime, GPU resource requests, and DCGM exporter. {{- end }} 3. Confirm the GGUF host path exists on every node eligible for Ollama scheduling. -4. Suggested install: - helm upgrade --install llm-observability-stack ./llm-observability-stack -n {{ include "llm-observability-stack.namespace" . }} --create-namespace -f values.enterprise-pilot-k3s.yaml --set kube-prometheus-stack.crds.enabled=false +4. Preferred install: + edge install infra --yes + edge validate infra + edge install observability --profile geforce-940m-k3s --yes Endpoints after install: {{- if .Values.ollama.enabled }} diff --git a/templates/validate-config.yaml b/templates/validate-config.yaml index b7f96ca..bbf6f40 100644 --- a/templates/validate-config.yaml +++ b/templates/validate-config.yaml @@ -29,6 +29,15 @@ Validate cross-chart secret wiring to avoid drift between: {{- $gpuOperator := (get .Values "gpu-operator") | default dict -}} {{- $devicePlugin := (get .Values "nvidia-device-plugin") | default dict -}} {{- $dcgmExporter := (get .Values "dcgm-exporter") | default dict -}} +{{- if (get $gpuOperator "enabled") -}} +{{- fail "gpu-operator.enabled is no longer supported in llm-observability-stack. Deploy the NVIDIA substrate with k3s-nvidia-edge first, then install this chart." -}} +{{- end -}} +{{- if (get $devicePlugin "enabled") -}} +{{- fail "nvidia-device-plugin.enabled is no longer supported in llm-observability-stack. Deploy the NVIDIA substrate with k3s-nvidia-edge first, then install this chart." -}} +{{- end -}} +{{- if (get $dcgmExporter "enabled") -}} +{{- fail "dcgm-exporter.enabled is no longer supported in llm-observability-stack. Observe the DCGM exporter provided by k3s-nvidia-edge/GPU Operator with monitoring.dcgmExporter.serviceMonitor instead." -}} +{{- end -}} {{- if and (get $gpuOperator "enabled") (get $devicePlugin "enabled") -}} {{- fail "gpu-operator.enabled and nvidia-device-plugin.enabled must not both be true. gpu-operator already manages the NVIDIA device plugin." -}} {{- end -}} diff --git a/tests/test_helm_smoke.py b/tests/test_helm_smoke.py index b36748e..99f93fb 100644 --- a/tests/test_helm_smoke.py +++ b/tests/test_helm_smoke.py @@ -107,6 +107,9 @@ def test_full_stack_nvidia_profile_renders_observability_resources() -> None: assert "kind: PrometheusRule" in manifest assert "name: llm-observability-dashboards" in manifest assert "llm_observability_time_to_first_token_seconds" in manifest + assert "name: nvidia-device-plugin" not in manifest + assert "name: dcgm-exporter" not in manifest + assert "kind: ClusterPolicy" not in manifest @pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") @@ -139,6 +142,9 @@ def test_geforce_profile_uses_repository_modelfile_and_gpu_label() -> None: assert "http://langchain-demo:8000/ollama" not in manifest assert "name: langchain-demo" not in manifest assert "/bin/ollama rm" not in manifest + assert "name: nvidia-device-plugin" not in manifest + assert "name: dcgm-exporter" not in manifest + assert "kind: ClusterPolicy" not in manifest default_render = _run(["helm", "template", "llm-observability-stack", "."]) assert default_render.returncode == 0, _combined_output(default_render) @@ -239,6 +245,8 @@ def test_generated_nvidia_overlay_uses_gpu_resource_without_static_node_label() assert "runtimeClassName: \"nvidia\"" in manifest assert "nvidia.com/gpu: 1" in manifest assert "nvidia.com/gpu.present" not in manifest + assert "name: nvidia-device-plugin" not in manifest + assert "name: dcgm-exporter" not in manifest @pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") @@ -292,6 +300,23 @@ def test_nvidia_profile_rejects_missing_startup_model_residency() -> None: assert "OLLAMA_KEEP_ALIVE" in _combined_output(finite_keep_alive) +@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") +def test_base_layer_chart_enables_are_rejected() -> None: + for key in ("gpu-operator.enabled", "nvidia-device-plugin.enabled", "dcgm-exporter.enabled"): + render = _run( + [ + "helm", + "template", + "llm-observability-stack", + ".", + "--set", + f"{key}=true", + ] + ) + assert render.returncode != 0 + assert "k3s-nvidia-edge" in _combined_output(render) + + @pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") def test_helm_package_stays_below_secret_limit_budget(tmp_path: Path) -> None: package = _run(["helm", "package", ".", "-d", str(tmp_path)]) @@ -320,9 +345,6 @@ def test_helm_package_stays_below_secret_limit_budget(tmp_path: Path) -> None: "llm-observability-stack/dashboards/benchmark-results.json", "llm-observability-stack/python-toolbox/examples/otel_genai_inference_traces.py", "llm-observability-stack/python-toolbox/examples/otel_genai_trace_seed_every_5m.py", - "llm-observability-stack/charts/gpu-operator/Chart.yaml", - "llm-observability-stack/charts/nvidia-device-plugin/Chart.yaml", - "llm-observability-stack/charts/dcgm-exporter/Chart.yaml", "llm-observability-stack/charts/kube-prometheus-stack/Chart.yaml", "llm-observability-stack/charts/opentelemetry-collector/Chart.yaml", "llm-observability-stack/charts/opentelemetry-operator/Chart.yaml", @@ -330,6 +352,14 @@ def test_helm_package_stays_below_secret_limit_budget(tmp_path: Path) -> None: for required_file in required_files: assert required_file in names, required_file + removed_substrate_files = [ + "llm-observability-stack/charts/gpu-operator/Chart.yaml", + "llm-observability-stack/charts/nvidia-device-plugin/Chart.yaml", + "llm-observability-stack/charts/dcgm-exporter/Chart.yaml", + ] + for removed_file in removed_substrate_files: + assert removed_file not in names, removed_file + @pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") def test_secret_wiring_validation_fails_on_mismatched_legacy_and_subchart_values() -> None: diff --git a/values.enterprise-pilot-k3s.yaml b/values.enterprise-pilot-k3s.yaml index 5ef7f0a..638be05 100644 --- a/values.enterprise-pilot-k3s.yaml +++ b/values.enterprise-pilot-k3s.yaml @@ -38,14 +38,14 @@ opentelemetry: genAI: enabled: true -# This local k3s node already has nvidia-device-plugin installed as a separate release. -# Keep the vendored NVIDIA installer charts available but disabled to avoid duplicate DaemonSets. +# This local k3s node already has the NVIDIA substrate from k3s-nvidia-edge. +# Do not install GPU Operator, NVIDIA device plugin, or DCGM exporter from this chart. gpu-operator: enabled: false nvidia-device-plugin: enabled: false dcgm-exporter: - enabled: true + enabled: false networkPolicy: enabled: false diff --git a/values.full-stack-nvidia.example.yaml b/values.full-stack-nvidia.example.yaml index 3f7178c..fca6997 100644 --- a/values.full-stack-nvidia.example.yaml +++ b/values.full-stack-nvidia.example.yaml @@ -34,11 +34,11 @@ gpu-operator: enabled: false nvidia-device-plugin: - enabled: true + enabled: false migStrategy: none dcgm-exporter: - enabled: true + enabled: false serviceMonitor: additionalLabels: release: kube-prometheus-stack diff --git a/values.yaml b/values.yaml index c843c7c..1320004 100644 --- a/values.yaml +++ b/values.yaml @@ -76,8 +76,8 @@ nvidia: gpuCount: 1 gpu-operator: - # Cluster-level installer. Enable this when the cluster does not already have NVIDIA - # drivers, container toolkit, device plugin, and DCGM exporter managed elsewhere. + # Legacy compatibility key only. The chart no longer vendors or installs GPU Operator. + # Use k3s-nvidia-edge, normally through edge-cli, to own the NVIDIA substrate. enabled: false nfd: enabled: true @@ -93,7 +93,7 @@ gpu-operator: enabled: true nvidia-device-plugin: - # Lightweight alternative to gpu-operator when host drivers/toolkit already exist. + # Legacy compatibility key only. The chart no longer installs the NVIDIA device plugin. enabled: false allowDefaultNamespace: true fullnameOverride: nvidia-device-plugin @@ -114,7 +114,8 @@ nvidia-device-plugin: memory: 256Mi dcgm-exporter: - # Enable with nvidia-device-plugin, or leave disabled when gpu-operator manages DCGM. + # Legacy compatibility key only. The chart no longer installs DCGM exporter. + # Use monitoring.dcgmExporter.serviceMonitor to observe an existing exporter. enabled: false fullnameOverride: dcgm-exporter runtimeClassName: nvidia From f71bfecf22c56f65063a7a0b2711fee11ca7308c Mon Sep 17 00:00:00 2001 From: Muhammad Waqas Date: Fri, 17 Jul 2026 23:25:38 +0500 Subject: [PATCH 2/6] Align CPU fallback CI and deployment docs --- .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/workflows/helm-validate.yaml | 10 ------- README.md | 9 ++++-- SUPPORT.md | 4 +-- docs/LIVE-VALIDATION-2026-07-17.md | 43 ++++++++++++++++++++++++++++ docs/README.md | 17 +++++------ docs/cli.md | 4 +-- templates/NOTES.txt | 4 +-- templates/prometheus-rules.yaml | 2 +- 9 files changed, 64 insertions(+), 31 deletions(-) create mode 100644 docs/LIVE-VALIDATION-2026-07-17.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d4cf26d..fcbedf8 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: true contact_links: - name: Support Guide - url: https://github.com/waqasm86/llm-observability-stack/blob/main/SUPPORT.md + url: https://github.com/Edge-Computing-LLM/llm-observability-stack/blob/main/SUPPORT.md about: Check support and troubleshooting guidance before opening an issue. diff --git a/.github/workflows/helm-validate.yaml b/.github/workflows/helm-validate.yaml index c37ea51..078d3a1 100644 --- a/.github/workflows/helm-validate.yaml +++ b/.github/workflows/helm-validate.yaml @@ -39,16 +39,6 @@ jobs: --set open-webui.webuiSecret.existingSecretName= \ >/tmp/rendered-full-stack-nvidia.yaml - - name: Build and validate monitoring chart - run: | - if [ ! -d monitoring ]; then - echo "monitoring chart is not present in this repository; skipping optional monitoring chart validation" - exit 0 - fi - helm dependency build monitoring - helm lint monitoring - helm template llm-monitoring monitoring >/tmp/rendered-monitoring.yaml - - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/README.md b/README.md index 69ee2ee..0a2775f 100644 --- a/README.md +++ b/README.md @@ -156,9 +156,7 @@ go build -o bin/llm-observability ./cmd/llm-observability Preferred local CLI flow from the organization control plane: ```bash -edge install infra --yes -edge validate infra -edge install observability --profile geforce-940m-k3s --yes +edge install all --accelerator auto --yes edge validate observability ``` @@ -193,6 +191,11 @@ helm version The local bootstrap helper detects the Kubernetes runtime before installing. It uses NVIDIA mode when Kubernetes advertises `nvidia.com/gpu`; otherwise it writes a CPU-only overlay and runs the same edge LLM observability path without NVIDIA runtime or GPU resource requests. +The organization CLI exposes the same policy directly. The full installer uses +host detection, while `edge install observability --accelerator auto --yes` uses +Kubernetes allocatable GPU capacity. Explicit `cpu` and `nvidia` modes are also +available for deterministic automation. + ## Quick Start ### A. Minimal validation profile diff --git a/SUPPORT.md b/SUPPORT.md index e7cf991..deee104 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -4,8 +4,8 @@ Use GitHub Issues for setup, debugging, and enhancement questions: -- Repository: https://github.com/waqasm86/llm-observability-stack -- New issue: https://github.com/waqasm86/llm-observability-stack/issues/new +- Repository: https://github.com/Edge-Computing-LLM/llm-observability-stack +- New issue: https://github.com/Edge-Computing-LLM/llm-observability-stack/issues/new ## What to Include diff --git a/docs/LIVE-VALIDATION-2026-07-17.md b/docs/LIVE-VALIDATION-2026-07-17.md new file mode 100644 index 0000000..263a93e --- /dev/null +++ b/docs/LIVE-VALIDATION-2026-07-17.md @@ -0,0 +1,43 @@ +# Live stack validation — 2026-07-17 + +The chart was rendered and installed against a live single-node k3s cluster after +`k3s-nvidia-edge` advertised one `nvidia.com/gpu` resource. + +Verified before rollout: + +- NVIDIA automatic profile selection +- GPU infrastructure dependency validation +- Prometheus Operator CRD server-side application +- chart dependency build +- PVC binding through `local-path` +- GGUF host model presence (`gemma-3-1b-it-Q4_K_M.gguf`, approximately 769 MiB) +- Redis and OpenTelemetry Collector readiness +- Ollama GPU scheduling and Open WebUI scheduling +- Open WebUI health endpoint +- live Ollama inference through the locally mounted GGUF model + +The first Helm client reached its 15-minute timeout while the first-time +multi-gigabyte images were still downloading. Ollama pulled in approximately 25 +minutes and Open WebUI in approximately 33 minutes. No chart, storage, scheduler, +secret, or service error occurred. The interrupted revision was reconciled with a +rollback to its own revision after all resources became ready; the subsequent +unchanged idempotent CLI installs completed in under 20 seconds and final Helm +revision 4 is `deployed`. + +Final runtime evidence: + +- Ollama, Open WebUI, Redis, and OpenTelemetry Collector were Ready with no + container restarts. +- Open WebUI `/health` returned `{"status":true}`. +- `ollama list` reported the local `gemma3-1b-it-gguf-local` model at 806 MB. +- The validation prompt returned exactly `validation ok`. +- Ollama detected the GeForce 940M through CUDA, offloaded 23/27 model layers, + placed 400.8 MiB of weights on CUDA, and reported a 57% CPU / 43% GPU processor + split for the loaded model. + +Static and application validation passed independently: + +- Helm lint +- default, CPU, and GeForce 940M template rendering +- Go tests and vet +- 17 Python smoke tests under `/usr/local/bin/python3.11` diff --git a/docs/README.md b/docs/README.md index b1bb69c..2ddec05 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,14 +6,15 @@ The documentation is organized around private LLM deployment and observability o ## Start Here -1. [QUICKSTART.md](QUICKSTART.md) -2. [cli.md](cli.md) -3. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) -4. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) -5. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) -6. [ARCHITECTURE.md](ARCHITECTURE.md) -7. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) -8. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) +1. [LIVE-VALIDATION-2026-07-17.md](LIVE-VALIDATION-2026-07-17.md) +2. [QUICKSTART.md](QUICKSTART.md) +3. [cli.md](cli.md) +4. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) +5. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) +6. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) +7. [ARCHITECTURE.md](ARCHITECTURE.md) +8. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) +9. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) ## Core Guides diff --git a/docs/cli.md b/docs/cli.md index 9606e05..c1680a4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -88,9 +88,7 @@ bin/llm-observability validate If the base layer is not installed yet, use `edge-cli`: ```bash -edge install infra --yes -edge validate infra -edge install observability --profile geforce-940m-k3s --yes +edge install all --accelerator auto --yes ``` By default, install and uninstall commands are dry-run unless `--yes` is provided. diff --git a/templates/NOTES.txt b/templates/NOTES.txt index c669c41..a62854d 100644 --- a/templates/NOTES.txt +++ b/templates/NOTES.txt @@ -7,9 +7,7 @@ {{- end }} 3. Confirm the GGUF host path exists on every node eligible for Ollama scheduling. 4. Preferred install: - edge install infra --yes - edge validate infra - edge install observability --profile geforce-940m-k3s --yes + edge install all --accelerator auto --yes Endpoints after install: {{- if .Values.ollama.enabled }} diff --git a/templates/prometheus-rules.yaml b/templates/prometheus-rules.yaml index 8e50254..8b4ad53 100644 --- a/templates/prometheus-rules.yaml +++ b/templates/prometheus-rules.yaml @@ -20,7 +20,7 @@ spec: severity: warning annotations: summary: LLM request error rate is above the configured threshold - runbook_url: https://github.com/waqasm86/llm-observability-stack/blob/main/docs/OPERATIONS-RUNBOOK.md + runbook_url: https://github.com/Edge-Computing-LLM/llm-observability-stack/blob/main/docs/OPERATIONS-RUNBOOK.md - alert: LLMHighP95Latency expr: | histogram_quantile(0.95, From 4314795a576b325ed86026379e62d15a25074372 Mon Sep 17 00:00:00 2001 From: Muhammad Waqas Date: Sat, 18 Jul 2026 00:21:32 +0500 Subject: [PATCH 3/6] Deploy Qwen 1.8B within GeForce 940M VRAM --- Modelfile.qwen-1.8b-chat-q4_K_M | 27 ++++++ README.md | 9 +- docs/LOCAL-K3S-NVIDIA-RUNBOOK.md | 13 ++- docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md | 91 ++++++++++++++++++++ docs/README.md | 19 ++-- docs/SINGLE-NODE-K3S-GEFORCE-940M.md | 19 ++-- hack/test-geforce-940m-inference.sh | 2 +- tests/test_helm_smoke.py | 9 +- values.geforce-940m-k3s.yaml | 14 +-- 9 files changed, 171 insertions(+), 32 deletions(-) create mode 100644 Modelfile.qwen-1.8b-chat-q4_K_M create mode 100644 docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md diff --git a/Modelfile.qwen-1.8b-chat-q4_K_M b/Modelfile.qwen-1.8b-chat-q4_K_M new file mode 100644 index 0000000..54a70b4 --- /dev/null +++ b/Modelfile.qwen-1.8b-chat-q4_K_M @@ -0,0 +1,27 @@ +# Modelfile for the locally retained Ollama Qwen 1.8B Chat Q4_K_M GGUF. +# +# Upstream: https://ollama.com/library/qwen:1.8b-chat-q4_K_M +# GGUF SHA-256: ef0125bcc77278420b64229fc29e235435c517f67ab7aa8546a9f5f7be644cef +# License: Tongyi Qianwen Research License (non-commercial use unless separately licensed). +# +# The GeForce 940M has 1 GiB VRAM. Live tuning on this host showed that 24 GPU +# layers use 860 MiB, while 23 GPU layers use 824 MiB. Keep num_batch at 1: +# larger batches make this Qwen architecture reserve a CUDA graph buffer that +# does not fit on this GPU. + +FROM /models/gguf/qwen-1.8b-chat-q4_K_M.gguf + +TEMPLATE """<|im_start|>system +{{ .System }}<|im_end|> +<|im_start|>user +{{ .Prompt }}<|im_end|> +<|im_start|>assistant +""" + +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_end|>" +PARAMETER num_gpu 23 +PARAMETER num_ctx 256 +PARAMETER num_batch 1 +PARAMETER num_predict 128 +PARAMETER temperature 0 diff --git a/README.md b/README.md index 0a2775f..88cc109 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,11 @@ helm template llm-observability-stack . \ Review the machine-specific model host path before using this profile on another system. The profile schedules on nodes with `nvidia.com/gpu.present=true`, which supports a single-node k3s control-plane/worker laptop without requiring a separate worker label. +This profile uses the locally retained Qwen 1.8B Chat Q4_K_M GGUF and +`Modelfile.qwen-1.8b-chat-q4_K_M`. On the 1 GiB GeForce 940M it pins 23/25 +layers to CUDA, limits batch size to 1, uses a 256-token context, and keeps the +model loaded indefinitely. The measured steady-state allocation is 824 MiB VRAM. + Preferred: deploy and validate the base layer through `edge-cli` first: ```bash @@ -219,7 +224,7 @@ edge validate infra Then deploy the LLM stack: ```bash -cd /media/waqasm86/External1/Waqas-Projects/Project-Edge-Computing-LLM/llm-observability-stack +cd /media/waqasm86/External1/Waqas-Projects/Project-Linux-Kubernetes-Nvidia/Project-Edge-Computing-LLM/llm-observability-stack helm upgrade --install llm-observability-stack . \ -n llm-observability --create-namespace \ @@ -290,7 +295,7 @@ Run the public benchmark from another terminal: ```bash ./benchmarks/ollama_benchmark.py \ - --model gemma3-1b-it-gguf-local \ + --model qwen-1-8b-chat-q4-k-m-local \ --warmup-runs 1 \ --runs 10 \ --output artifacts/benchmark-local.json diff --git a/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md b/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md index f8fe478..917c928 100644 --- a/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md +++ b/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md @@ -41,12 +41,17 @@ edge validate infra ## 4. Verify the Local GGUF Model -The default local enterprise profile expects: +The GeForce 940M profile expects: ```bash -ls -lh /media/waqasm86/External1/Waqas-Projects/repos-llamatelemetry/llamatelemetry-xubuntu24/models/gemma-3-1b-it-Q4_K_M.gguf +ls -lh /media/waqasm86/External1/Waqas-Projects/repos-llamatelemetry/llamatelemetry-xubuntu24/models/qwen-1.8b-chat-q4_K_M.gguf +sha256sum /media/waqasm86/External1/Waqas-Projects/repos-llamatelemetry/llamatelemetry-xubuntu24/models/qwen-1.8b-chat-q4_K_M.gguf ``` +The expected SHA-256 is +`ef0125bcc77278420b64229fc29e235435c517f67ab7aa8546a9f5f7be644cef`. +Other profiles may retain their Gemma defaults. + The chart mounts that host directory read-only into Ollama at `/models/gguf`, and the Ollama PVC is annotated with `helm.sh/resource-policy: keep`. ## 5. Build and Import Local Images @@ -149,7 +154,7 @@ The NVIDIA profiles render Ollama with: ```text OLLAMA_KEEP_ALIVE=-1 ollama.ollama.models.run: - - gemma3-1b-it-gguf-local + - qwen-1-8b-chat-q4-k-m-local ``` At container startup, the chart creates the local GGUF-backed model and then runs a small warmup prompt so Ollama loads the model into its GPU-enabled runner. `OLLAMA_KEEP_ALIVE=-1` keeps that loaded runner resident instead of unloading it after an idle timeout. @@ -174,7 +179,7 @@ From another terminal: curl -s http://127.0.0.1:11434/api/tags | jq curl -s http://127.0.0.1:11434/api/generate \ -H 'Content-Type: application/json' \ - -d '{"model":"gemma3-1b-it-gguf-local","prompt":"Reply with one short sentence.","stream":false}' | jq + -d '{"model":"qwen-1-8b-chat-q4-k-m-local","prompt":"Reply with one short sentence.","stream":false}' | jq ``` ## 10. Verify OpenTelemetry Collector diff --git a/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md b/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md new file mode 100644 index 0000000..2e54ba6 --- /dev/null +++ b/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md @@ -0,0 +1,91 @@ +# Qwen 1.8B live validation — 2026-07-18 + +## Scope + +This report records the migration of the live single-node Ubuntu 24.04 + k3s +cluster from the local Gemma model to Ollama +`qwen:1.8b-chat-q4_K_M`. The node uses an NVIDIA GeForce 940M with 1 GiB VRAM. + +The upstream tag is a Qwen 1.5 1.84B-parameter Q4_K_M model with an approximately +1.2 GB GGUF payload. Its Tongyi Qianwen Research License permits non-commercial +research/evaluation; commercial use requires a separate license. + +## Local GGUF + +The official Ollama tag was pulled into the retained Ollama PVC. Its model layer +was copied to the host model directory as: + +```text +/media/waqasm86/External1/Waqas-Projects/repos-llamatelemetry/llamatelemetry-xubuntu24/models/qwen-1.8b-chat-q4_K_M.gguf +``` + +Verified payload: + +- size: approximately 1.2 GB; +- SHA-256: `ef0125bcc77278420b64229fc29e235435c517f67ab7aa8546a9f5f7be644cef`; +- hash matched Ollama's `sha256-ef0125...` model layer exactly. + +The chart mounts the host directory read-only and creates the runtime alias +`qwen-1-8b-chat-q4-k-m-local` from +`Modelfile.qwen-1.8b-chat-q4_K_M`. + +## VRAM tuning + +The requested 850 GB limit was interpreted as 850 MiB because the physical GPU +has only 1,024 MiB. The upstream model cannot fit fully in VRAM and must use +CPU/GPU split inference. + +| Configuration | Result | +|---|---| +| Automatic, batch 512, context 1024 | Failed safely: CUDA graph allocation exceeded 1 GiB | +| 6 GPU layers, batch 512 | Failed safely: graph buffer still too large | +| 13 GPU layers, batch 64 | Failed safely: graph buffer still too large | +| 1 GPU layer, batch 1, context 256 | Passed; 80 MiB VRAM | +| 24 GPU layers, batch 1, context 256 | Passed; 860 MiB VRAM, above ceiling | +| 23 GPU layers, batch 1, context 256 | Passed; 824 MiB VRAM, selected | + +Final Ollama evidence: + +```text +PROCESSOR: 27%/73% CPU/GPU +CONTEXT: 256 +UNTIL: Forever +GPU VRAM: 824 MiB used, 152 MiB free +layers: 23/25 repeating layers offloaded to CUDA +weights: 734.9 MiB CUDA, 277.5 MiB CPU +KV cache: 2.9 MiB CUDA +graph: 8.0 MiB CUDA +``` + +`OLLAMA_KEEP_ALIVE=-1` and `OLLAMA_MAX_LOADED_MODELS=1` keep this Qwen runner +resident and prevent a second model from competing for the GPU. + +## Deployment results + +- Gemma was stopped and removed from the Ollama registry; the original read-only + host GGUF was not deleted. +- Helm revision 6 is `deployed`. +- Ollama was recreated with zero restarts and automatically created/warmed the + local Qwen alias. +- Ollama, Open WebUI, Redis, and OpenTelemetry Collector were Ready. +- Open WebUI `/health` returned `{"status":true}`. +- `edge install observability --accelerator auto --yes` completed, including its + Qwen smoke prompt. +- `ollama ps` reported `Forever` after all tests. + +## Inference tests + +| Test | Response | Throughput | +|---|---|---:| +| Exact validation | `validation ok` | 15.78 tokens/s | +| Arithmetic (`2 + 2`) | `2 plus 2 equals 4.` | 9.75 tokens/s | +| English-to-Chinese translation | `边缘计算的中文翻译是“边缘计算”。` | 11.47 tokens/s | + +Static validation also passed: + +- Helm dependency build and lint; +- GeForce profile render with `num_gpu=23`, `num_batch=1`, and `num_ctx=256`; +- Python 3.11 Helm smoke tests; +- edge-cli Go tests. + +The Qwen model was deliberately left loaded in GPU memory after validation. diff --git a/docs/README.md b/docs/README.md index 2ddec05..a5bdfb8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,15 +6,16 @@ The documentation is organized around private LLM deployment and observability o ## Start Here -1. [LIVE-VALIDATION-2026-07-17.md](LIVE-VALIDATION-2026-07-17.md) -2. [QUICKSTART.md](QUICKSTART.md) -3. [cli.md](cli.md) -4. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) -5. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) -6. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) -7. [ARCHITECTURE.md](ARCHITECTURE.md) -8. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) -9. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) +1. [QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md](QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md) +2. [LIVE-VALIDATION-2026-07-17.md](LIVE-VALIDATION-2026-07-17.md) +3. [QUICKSTART.md](QUICKSTART.md) +4. [cli.md](cli.md) +5. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) +6. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) +7. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) +8. [ARCHITECTURE.md](ARCHITECTURE.md) +9. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) +10. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) ## Core Guides diff --git a/docs/SINGLE-NODE-K3S-GEFORCE-940M.md b/docs/SINGLE-NODE-K3S-GEFORCE-940M.md index d6f8e18..a855330 100644 --- a/docs/SINGLE-NODE-K3S-GEFORCE-940M.md +++ b/docs/SINGLE-NODE-K3S-GEFORCE-940M.md @@ -35,7 +35,7 @@ In another terminal: curl -s http://127.0.0.1:11434/api/tags | jq curl -s http://127.0.0.1:11434/api/generate \ -H 'Content-Type: application/json' \ - -d '{"model":"gemma3-1b-it-gguf-local","prompt":"Reply with one short sentence.","stream":false}' | jq + -d '{"model":"qwen-1-8b-chat-q4-k-m-local","prompt":"Reply with one short sentence.","stream":false}' | jq kubectl logs -n llm-observability deployment/ollama | grep -Ei 'cuda|gpu|offload|memory' ``` @@ -44,16 +44,23 @@ profile points Open WebUI directly at the in-cluster Ollama service (`http://oll not deploy GPU Operator, NVIDIA device plugin, or DCGM exporter workloads. Those remain owned by the base `k3s-nvidia-edge` layer. -The profile reads `Modelfile.gemma-3-1b-it-gguf` directly and mounts the verified host directory. +The profile reads `Modelfile.qwen-1.8b-chat-q4_K_M` directly and mounts the verified host directory. It starts Ollama plus Open WebUI for local browser inference once the GPU baseline is healthy. -Verified on June 15, 2026: +The official Qwen GGUF is approximately 1.2 GB and cannot fit entirely in the +GeForce 940M's 1 GiB VRAM. The profile pins 23 of 25 repeating layers to CUDA, +uses a batch size of 1 and context of 256, and keeps the model resident with +`OLLAMA_KEEP_ALIVE=-1`. This measured configuration uses 824 MiB VRAM; 24 layers +used 860 MiB and exceeded the configured 850 MiB ceiling. + +Verified on July 18, 2026: - k3s node role: `control-plane,worker` - device resource: one allocatable `nvidia.com/gpu` -- model: `gemma3-1b-it-gguf-local:latest`, 806 MB -- Ollama detection: CUDA compute 5.0, NVIDIA GeForce 940M, approximately 969 MiB free VRAM -- observed inference: 6.1 seconds, up to 52% GPU utilization and 554 MiB VRAM +- model: `qwen-1-8b-chat-q4-k-m-local:latest`, approximately 1.2 GB +- Ollama detection: CUDA compute 5.0, NVIDIA GeForce 940M, 1,024 MiB total VRAM +- selected runtime: 23/25 layers on CUDA, 824 MiB used, 152 MiB free +- observed inference: 9.75–15.78 generated tokens/s across arithmetic, translation, and exact-response tests Run `./hack/test-geforce-940m-inference.sh` to repeat the smoke test. diff --git a/hack/test-geforce-940m-inference.sh b/hack/test-geforce-940m-inference.sh index c8d7e7e..1b5ccb4 100755 --- a/hack/test-geforce-940m-inference.sh +++ b/hack/test-geforce-940m-inference.sh @@ -2,7 +2,7 @@ set -euo pipefail NAMESPACE="${NAMESPACE:-llm-observability}" -MODEL="${MODEL:-gemma3-1b-it-gguf-local}" +MODEL="${MODEL:-qwen-1-8b-chat-q4-k-m-local}" PROMPT="${PROMPT:-In one short sentence, explain why GPU observability matters.}" gpu="$(kubectl get nodes -o jsonpath='{range .items[*]}{.status.allocatable.nvidia\.com/gpu}{end}')" diff --git a/tests/test_helm_smoke.py b/tests/test_helm_smoke.py index 99f93fb..7610a20 100644 --- a/tests/test_helm_smoke.py +++ b/tests/test_helm_smoke.py @@ -126,14 +126,16 @@ def test_geforce_profile_uses_repository_modelfile_and_gpu_label() -> None: ) assert render.returncode == 0, _combined_output(render) manifest = render.stdout - assert "gemma-3-1b-it-Q4_K_M.gguf" in manifest + assert "qwen-1.8b-chat-q4_K_M.gguf" in manifest assert "nvidia.com/gpu.present: \"true\"" in manifest assert "node-role.kubernetes.io/worker" not in manifest assert "nvidia.com/gpu: 1" in manifest assert "OLLAMA_KEEP_ALIVE" in manifest assert 'value: "-1"' in manifest - assert '/bin/ollama run gemma3-1b-it-gguf-local "Reply with exactly: model ready"' in manifest - assert "PARAMETER num_ctx 1024" in manifest + assert '/bin/ollama run qwen-1-8b-chat-q4-k-m-local "Reply with exactly: model ready"' in manifest + assert "PARAMETER num_gpu 23" in manifest + assert "PARAMETER num_ctx 256" in manifest + assert "PARAMETER num_batch 1" in manifest assert 'helm.sh/resource-policy: keep' in manifest assert "readOnly: true" in manifest assert "name: open-webui" in manifest @@ -340,6 +342,7 @@ def test_helm_package_stays_below_secret_limit_budget(tmp_path: Path) -> None: required_files = [ "llm-observability-stack/langchain-demo/app.py", "llm-observability-stack/Modelfile.gemma-3-1b-it-gguf", + "llm-observability-stack/Modelfile.qwen-1.8b-chat-q4_K_M", "llm-observability-stack/dashboards/llm-overview.json", "llm-observability-stack/dashboards/nvidia-gpu.json", "llm-observability-stack/dashboards/benchmark-results.json", diff --git a/values.geforce-940m-k3s.yaml b/values.geforce-940m-k3s.yaml index f2edec2..2e34b0c 100644 --- a/values.geforce-940m-k3s.yaml +++ b/values.geforce-940m-k3s.yaml @@ -14,16 +14,16 @@ nvidia: ollamaModel: enabled: true - name: gemma3-1b-it-gguf-local + name: qwen-1-8b-chat-q4-k-m-local createAtStartup: true warmupPrompt: Say hello in one short sentence. gguf: enabled: true hostPath: /media/waqasm86/External1/Waqas-Projects/repos-llamatelemetry/llamatelemetry-xubuntu24/models - filename: gemma-3-1b-it-Q4_K_M.gguf + filename: qwen-1.8b-chat-q4_K_M.gguf mountPath: /models/gguf modelfile: - file: Modelfile.gemma-3-1b-it-gguf + file: Modelfile.qwen-1.8b-chat-q4_K_M ollama: enabled: true @@ -43,10 +43,10 @@ ollama: models: pull: [] run: - - gemma3-1b-it-gguf-local + - qwen-1-8b-chat-q4-k-m-local runPrompt: "Reply with exactly: model ready" create: - - name: gemma3-1b-it-gguf-local + - name: qwen-1-8b-chat-q4-k-m-local configMapRef: ollama-local-modelfile configMapKeyRef: Modelfile persistentVolume: @@ -82,7 +82,7 @@ ollama: - name: OLLAMA_MAX_LOADED_MODELS value: "1" - name: OLLAMA_CONTEXT_LENGTH - value: "1024" + value: "256" openWebUI: enabled: true @@ -156,7 +156,7 @@ open-webui: - name: RAG_EMBEDDING_ENGINE value: ollama - name: RAG_EMBEDDING_MODEL - value: gemma3-1b-it-gguf-local + value: qwen-1-8b-chat-q4-k-m-local - name: RAG_EMBEDDING_MODEL_AUTO_UPDATE value: "False" - name: RAG_RERANKING_MODEL_AUTO_UPDATE From 8bda5c23b8f12d406c80f65c21093e03a490195b Mon Sep 17 00:00:00 2001 From: Muhammad Waqas Date: Sat, 18 Jul 2026 00:44:30 +0500 Subject: [PATCH 4/6] Align current Qwen documentation --- README.md | 24 +++++++++++++------- docs/ARCHITECTURE.md | 13 +++++++++++ docs/KUBECTL-COMMAND-REFERENCE.md | 2 +- docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md | 5 ++++ docs/README.md | 10 ++++++-- docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md | 4 ++-- docs/cli.md | 6 +++-- 7 files changed, 49 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 88cc109..8e64797 100644 --- a/README.md +++ b/README.md @@ -44,18 +44,21 @@ The current local deployment target is a single-node Xubuntu 24 system running k - k3s node: combined control-plane and worker. - NVIDIA device plugin resource: `nvidia.com/gpu: 1`. - RuntimeClass: `nvidia`. -- Model: Gemma 3 1B IT Q4_K_M GGUF. +- Model: Qwen 1.8B Chat Q4_K_M GGUF through the local alias + `qwen-1-8b-chat-q4-k-m-local`. -Measured after one warmup and three streaming benchmark requests: +Measured after deployment, warmup, and exact-response, arithmetic, and +translation inference checks: | Metric | Result | |---|---:| -| TTFT p50 | 0.377 s | -| TTFT p95 | 0.381 s | -| Mean throughput | 11.69 tokens/s | -| End-to-end p95 | 6.97 s | -| Peak GPU utilization | 52% | -| VRAM usage | 554 MiB | +| Model size | Approximately 1.2 GB | +| CUDA layers | 23/25 | +| Processor split | 27% CPU / 73% GPU | +| Context / batch | 256 / 1 | +| Observed throughput | 9.75-15.78 tokens/s | +| VRAM usage | 824 MiB used / 152 MiB free | +| Residency | `Forever` | Evidence and reproduction: @@ -66,9 +69,14 @@ Evidence and reproduction: - [Xubuntu k3s NVIDIA runbook](docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md) - [Sanitized benchmark artifact](artifacts/geforce-940m-benchmark.json) - [GeForce 940M Helm profile](values.geforce-940m-k3s.yaml) +- [Qwen GGUF runtime evidence companion](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) These numbers prove constrained local edge feasibility. They do not claim enterprise load, concurrency, fleet reliability, or production readiness. +The companion repository performs read-only runtime contract checks and +captures sanitized evidence. This chart remains the source of truth for the +Modelfile, Helm values, model lifecycle, and workload configuration. + ## Who This Is For - Developers running private LLMs on local Linux systems. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b752af3..71faf52 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -8,6 +8,12 @@ The preferred end-to-end CLI is `edge-cli`. The Go CLI in `cmd/llm-observability is a repo-local helper for this chart and should not be used as the primary cross-repository installer. +The optional +[`qwen-gguf-observability`](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) +repository is a read-only evidence consumer. It observes selected Kubernetes, +Helm, Ollama, and GPU status after deployment; it owns no chart resources, +Modelfile, model lifecycle, or telemetry backend. + ## 1. Design Goals - Keep the stack understandable on a single local node @@ -207,3 +213,10 @@ The verified local NVIDIA flow depends on `k3s-nvidia-edge` for: - CUDA pod validation `llm-observability-stack` should be installed only after those checks pass. + +## 10. Downstream evidence companion + +After this stack is healthy, `qwen-gguf-observability` can validate the narrow +GeForce/Qwen runtime contract and write sanitized JSON or Markdown evidence. +Deployment changes discovered by those checks must be made here or in the +infrastructure repository, not copied into the evidence tool. diff --git a/docs/KUBECTL-COMMAND-REFERENCE.md b/docs/KUBECTL-COMMAND-REFERENCE.md index e27b17b..2036e3b 100644 --- a/docs/KUBECTL-COMMAND-REFERENCE.md +++ b/docs/KUBECTL-COMMAND-REFERENCE.md @@ -146,7 +146,7 @@ Ollama API test: curl -s http://localhost:11434/api/tags | jq curl -s http://localhost:11434/api/chat \ -H 'Content-Type: application/json' \ - -d '{"model":"gemma3-1b-it-gguf-local","stream":false,"messages":[{"role":"user","content":"health check"}]}' | jq + -d '{"model":"qwen-1-8b-chat-q4-k-m-local","stream":false,"messages":[{"role":"user","content":"health check"}]}' | jq ``` LangChain demo test: diff --git a/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md b/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md index 2e54ba6..5659ac6 100644 --- a/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md +++ b/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md @@ -89,3 +89,8 @@ Static validation also passed: - edge-cli Go tests. The Qwen model was deliberately left loaded in GPU memory after validation. + +Ongoing read-only contract validation and sanitized evidence capture are kept +in the complementary +[`qwen-gguf-observability`](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) +repository. This stack remains the deployment and model-configuration owner. diff --git a/docs/README.md b/docs/README.md index a5bdfb8..a4beb22 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,12 @@ The documentation is organized around private LLM deployment and observability o 9. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) 10. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) +External companion: + +- [`qwen-gguf-observability`](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) + provides read-only Qwen runtime contract checks and sanitized evidence. It + does not replace this repository's Helm, Modelfile, or benchmark assets. + ## Core Guides - [QUICKSTART.md](QUICKSTART.md) @@ -54,9 +60,9 @@ The documentation is organized around private LLM deployment and observability o ## Local Validation - [VERIFIED-LOCAL-GPU-RESULTS.md](VERIFIED-LOCAL-GPU-RESULTS.md) - - Sanitized local GPU benchmark and deployment results. + - Historical Gemma-era sanitized GPU benchmark and deployment results. - [LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md](LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md) - - Local k3s/NVIDIA status report for the Xubuntu 24 system. + - Historical Gemma-era k3s/NVIDIA status report for the Xubuntu 24 system. - [SINGLE-NODE-K3S-GEFORCE-940M.md](SINGLE-NODE-K3S-GEFORCE-940M.md) - Low-memory GeForce 940M profile notes and constraints. - [../validation/README.md](../validation/README.md) diff --git a/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md b/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md index e81db37..fa4de0b 100644 --- a/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md +++ b/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md @@ -187,7 +187,7 @@ Ollama generation: ```bash curl -s http://127.0.0.1:11434/api/generate \ -H 'Content-Type: application/json' \ - -d '{"model":"gemma3-1b-it-gguf-local:latest","prompt":"Reply with one short sentence.","stream":false}' | jq + -d '{"model":"qwen-1-8b-chat-q4-k-m-local:latest","prompt":"Reply with one short sentence.","stream":false}' | jq ``` LangChain proxy: @@ -229,7 +229,7 @@ Keep benchmark outputs under `artifacts/` and commit only sanitized evidence. ```bash python3.11 benchmarks/ollama_benchmark.py \ --url 'http://127.0.0.1:11434/api/generate' \ - --model gemma3-1b-it-gguf-local:latest \ + --model qwen-1-8b-chat-q4-k-m-local:latest \ --warmup-runs 1 \ --runs 3 \ --output artifacts/local-benchmark.json diff --git a/docs/cli.md b/docs/cli.md index c1680a4..d80597b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -8,6 +8,8 @@ primary organization-level CLI for end-to-end installs and uninstalls. - `k3s-nvidia-edge` owns the local Linux, k3s, k3s containerd, NVIDIA runtime, GPU Operator, NVIDIA device plugin, DCGM exporter, Node Feature Discovery, `RuntimeClass/nvidia`, and `nvidia.com/gpu` validation layer. - `llm-observability-stack` owns Ollama, Open WebUI, Open WebUI Redis, OpenTelemetry Collector, optional Prometheus/Grafana, optional LangChain/FastAPI proxy, benchmark tooling, notebooks, and local model configuration. - `edge-cli` owns cross-repository ordering: infra first, observability second. +- `qwen-gguf-observability` optionally consumes the deployed state for read-only + contract checks and evidence; it owns no install or uninstall workflow. - The CLI does not import `k3s-nvidia-edge/internal/...`. ## Build @@ -33,7 +35,7 @@ bin/llm-observability doctor bin/llm-observability install --profile geforce-940m-k3s --skip-base --yes bin/llm-observability status bin/llm-observability validate -bin/llm-observability benchmark --model gemma3-1b-it-gguf-local --runs 3 +bin/llm-observability benchmark --model qwen-1-8b-chat-q4-k-m-local --runs 3 bin/llm-observability uninstall --yes bin/llm-observability print-commands --profile geforce-940m-k3s ``` @@ -114,7 +116,7 @@ The benchmark command wraps the existing Python client: ```bash bin/llm-observability benchmark \ - --model gemma3-1b-it-gguf-local \ + --model qwen-1-8b-chat-q4-k-m-local \ --runs 3 \ --prompt "Explain edge GPU observability in one sentence." \ --output artifacts/benchmark-local.json From 16dfb24564974214e0eb94e5b7756772078b332f Mon Sep 17 00:00:00 2001 From: Muhammad Waqas Date: Sat, 18 Jul 2026 00:47:06 +0500 Subject: [PATCH 5/6] Refresh GitHub Actions runtimes --- .github/workflows/helm-validate.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm-validate.yaml b/.github/workflows/helm-validate.yaml index 078d3a1..7ee2f84 100644 --- a/.github/workflows/helm-validate.yaml +++ b/.github/workflows/helm-validate.yaml @@ -4,14 +4,17 @@ on: push: pull_request: +permissions: + contents: read + jobs: lint-and-render: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Helm - uses: azure/setup-helm@v4 + uses: azure/setup-helm@v5 with: version: v3.15.4 @@ -40,7 +43,7 @@ jobs: >/tmp/rendered-full-stack-nvidia.yaml - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" From 20c77b868853d08025393ec0208cdd061c2fae0c Mon Sep 17 00:00:00 2001 From: Muhammad Waqas Date: Sat, 18 Jul 2026 01:10:51 +0500 Subject: [PATCH 6/6] Standardize Qwen and Python 3.11 tooling --- README.md | 5 ++ benchmarks/ollama_benchmark.py | 2 +- cmd/llm-observability/main.go | 2 +- docs/LANGUAGE-BOUNDARIES.md | 50 +++++++++++++++++++ docs/PYTHON-KUBERNETES-AUTOMATION.md | 12 ++--- docs/README.md | 6 +++ docs/scripts/README.md | 14 +++--- docs/scripts/network_inventory.py | 8 +-- docs/scripts/service_path_inspector.py | 6 +-- docs/scripts/watch_endpoints.py | 6 +-- hack/README.md | 2 + hack/validate-local-stack.sh | 4 +- internal/stack/options.go | 2 +- internal/stack/options_test.go | 10 ++++ internal/stack/workflows.go | 6 +-- .../python/01_namespace_inventory.py | 2 +- .../python/02_service_path_inspector.py | 2 +- .../python/03_workload_health.py | 2 +- .../python/04_networking_report.py | 2 +- .../python/05_watch_events.py | 2 +- .../python/README.md | 4 +- .../runbooks/run_python_suite.sh | 2 +- 22 files changed, 109 insertions(+), 42 deletions(-) create mode 100644 docs/LANGUAGE-BOUNDARIES.md diff --git a/README.md b/README.md index 8e64797..7412150 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,11 @@ The companion repository performs read-only runtime contract checks and captures sanitized evidence. This chart remains the source of truth for the Modelfile, Helm values, model lifecycle, and workload configuration. +The separate +[`Frontend-Edge-LLM-Observability`](https://github.com/Edge-Computing-LLM/Frontend-Edge-LLM-Observability) +project provides the TypeScript/Vue dashboard. It consumes controlled DCGM and +future telemetry endpoints; frontend source is not duplicated in this chart. + ## Who This Is For - Developers running private LLMs on local Linux systems. diff --git a/benchmarks/ollama_benchmark.py b/benchmarks/ollama_benchmark.py index a58e2ff..bab28ce 100755 --- a/benchmarks/ollama_benchmark.py +++ b/benchmarks/ollama_benchmark.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 """Run repeatable Ollama streaming benchmarks and optionally publish summary metrics.""" from __future__ import annotations diff --git a/cmd/llm-observability/main.go b/cmd/llm-observability/main.go index 56a218f..99622eb 100644 --- a/cmd/llm-observability/main.go +++ b/cmd/llm-observability/main.go @@ -42,7 +42,7 @@ Examples: llm-observability install --profile geforce-940m-k3s --skip-base --yes llm-observability status llm-observability validate - llm-observability benchmark --model gemma3-1b-it-gguf-local --runs 3 + llm-observability benchmark --model qwen-1-8b-chat-q4-k-m-local --runs 3 llm-observability uninstall --yes edge uninstall all --yes ` diff --git a/docs/LANGUAGE-BOUNDARIES.md b/docs/LANGUAGE-BOUNDARIES.md new file mode 100644 index 0000000..67b85b1 --- /dev/null +++ b/docs/LANGUAGE-BOUNDARIES.md @@ -0,0 +1,50 @@ +# Programming language and script boundaries + +This repository deliberately uses several implementation languages because it +contains a Helm application, a compatibility CLI, an instrumented API, and +operator tooling. + +## Go + +Use Go in `cmd/llm-observability` and `internal/stack` for durable repo-level +CLI behavior, typed options, process execution, timeouts, errors, and lifecycle +operations. New cross-repository workflows belong in `edge-cli`. + +Do not create a second Go implementation of Helm templates, Python benchmark +math, or browser behavior. + +## Python 3.11 + +Use Python 3.11 for: + +- the instrumented FastAPI/LangChain service; +- benchmarks and structured result processing; +- Kubernetes API reports and notebook code; +- Helm/application tests; +- in-cluster diagnostics whose container image pins its own interpreter. + +Host commands use `python3.11` and install packages directly with +`python3.11 -m pip`. This local workflow does not create virtual environments. +Container commands may use `python` when that name is supplied by the pinned +container image. + +## Bash + +Keep Bash in `hack/` for short, transparent composition of existing tools such +as Helm, kubectl, Docker/nerdctl, and k3s containerd import. Scripts must use +`set -euo pipefail`, quote variables, expose safe defaults, and delegate complex +logic to Go or Python 3.11. + +Do not add growing state machines, JSON parsers, model policy, or duplicated +install/uninstall logic to Bash. + +## Helm and YAML + +Helm templates and values own Kubernetes desired state. Runtime detection may +write a generated values overlay, but it must not modify tracked defaults. + +## TypeScript + +Browser functionality belongs in the separate +`Frontend-Edge-LLM-Observability` repository. This chart may integrate that +frontend later, but it should not vendor a second frontend source tree. diff --git a/docs/PYTHON-KUBERNETES-AUTOMATION.md b/docs/PYTHON-KUBERNETES-AUTOMATION.md index 3496d9f..81b2ab4 100644 --- a/docs/PYTHON-KUBERNETES-AUTOMATION.md +++ b/docs/PYTHON-KUBERNETES-AUTOMATION.md @@ -11,15 +11,13 @@ Profile note: ## 1. Install and Environment ```bash -python3 -m venv .venv -source .venv/bin/activate -pip install kubernetes +/usr/local/bin/python3.11 -m pip install kubernetes ``` Or use project script requirements: ```bash -pip install -r docs/scripts/requirements.txt +/usr/local/bin/python3.11 -m pip install -r docs/scripts/requirements.txt ``` ## 2. Authentication Modes @@ -54,9 +52,9 @@ config.load_incluster_config() Quick usage: ```bash -python docs/scripts/network_inventory.py --namespace llm-observability -python docs/scripts/service_path_inspector.py --namespace llm-observability --service ollama -python docs/scripts/watch_endpoints.py --namespace llm-observability --service open-webui --timeout 600 +/usr/local/bin/python3.11 docs/scripts/network_inventory.py --namespace llm-observability +/usr/local/bin/python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service ollama +/usr/local/bin/python3.11 docs/scripts/watch_endpoints.py --namespace llm-observability --service open-webui --timeout 600 ``` ## 4. Practical Python Examples diff --git a/docs/README.md b/docs/README.md index a4beb22..d884ea6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,12 +16,16 @@ The documentation is organized around private LLM deployment and observability o 8. [ARCHITECTURE.md](ARCHITECTURE.md) 9. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) 10. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) +11. [LANGUAGE-BOUNDARIES.md](LANGUAGE-BOUNDARIES.md) External companion: - [`qwen-gguf-observability`](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) provides read-only Qwen runtime contract checks and sanitized evidence. It does not replace this repository's Helm, Modelfile, or benchmark assets. +- [`Frontend-Edge-LLM-Observability`](https://github.com/Edge-Computing-LLM/Frontend-Edge-LLM-Observability) + owns the TypeScript/Vue presentation layer and does not receive direct + Kubernetes credentials from this chart. ## Core Guides @@ -39,6 +43,8 @@ External companion: - Existing local k3s/NVIDIA command reference. - [ARCHITECTURE.md](ARCHITECTURE.md) - Component ownership, request paths, service exposure, and configuration boundaries. +- [LANGUAGE-BOUNDARIES.md](LANGUAGE-BOUNDARIES.md) + - Go, Python 3.11, Bash, Helm/YAML, and frontend ownership rules. - [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) - Day-0 and day-1 tasks: deploy, verify, port-forward, rebuild images, debug, and clean up. - [NOTEBOOKS-GUIDE.md](NOTEBOOKS-GUIDE.md) diff --git a/docs/scripts/README.md b/docs/scripts/README.md index 584d219..864def3 100644 --- a/docs/scripts/README.md +++ b/docs/scripts/README.md @@ -7,9 +7,7 @@ They are designed to run from host or CI and do not require `python-toolbox` to ## Install ```bash -python3 -m venv .venv -source .venv/bin/activate -pip install -r docs/scripts/requirements.txt +/usr/local/bin/python3.11 -m pip install -r docs/scripts/requirements.txt ``` ## Scripts @@ -23,7 +21,7 @@ Purpose: Usage: ```bash -python docs/scripts/network_inventory.py --namespace llm-observability +/usr/local/bin/python3.11 docs/scripts/network_inventory.py --namespace llm-observability ``` ### 2) `service_path_inspector.py` @@ -35,9 +33,9 @@ Purpose: Usage: ```bash -python docs/scripts/service_path_inspector.py --namespace llm-observability --service ollama -python docs/scripts/service_path_inspector.py --namespace llm-observability --service open-webui -python docs/scripts/service_path_inspector.py --namespace llm-observability --service langchain-demo +/usr/local/bin/python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service ollama +/usr/local/bin/python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service open-webui +/usr/local/bin/python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service langchain-demo ``` ### 3) `watch_endpoints.py` @@ -49,7 +47,7 @@ Purpose: Usage: ```bash -python docs/scripts/watch_endpoints.py --namespace llm-observability --service ollama --timeout 600 +/usr/local/bin/python3.11 docs/scripts/watch_endpoints.py --namespace llm-observability --service ollama --timeout 600 ``` ## Config options diff --git a/docs/scripts/network_inventory.py b/docs/scripts/network_inventory.py index 41c4c0c..46341b6 100755 --- a/docs/scripts/network_inventory.py +++ b/docs/scripts/network_inventory.py @@ -1,12 +1,12 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 """Print a networking inventory for a Kubernetes namespace. Requires: - pip install kubernetes + python3.11 -m pip install kubernetes Examples: - python docs/scripts/network_inventory.py --namespace llm-observability - python docs/scripts/network_inventory.py --namespace llm-observability --context my-k3s + python3.11 docs/scripts/network_inventory.py --namespace llm-observability + python3.11 docs/scripts/network_inventory.py --namespace llm-observability --context my-k3s """ from __future__ import annotations diff --git a/docs/scripts/service_path_inspector.py b/docs/scripts/service_path_inspector.py index ecd3f45..fc245af 100755 --- a/docs/scripts/service_path_inspector.py +++ b/docs/scripts/service_path_inspector.py @@ -1,11 +1,11 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 """Trace a Kubernetes Service to selectors, Pods, Endpoints, and EndpointSlices. Requires: - pip install kubernetes + python3.11 -m pip install kubernetes Example: - python docs/scripts/service_path_inspector.py --namespace llm-observability --service ollama + python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service ollama """ from __future__ import annotations diff --git a/docs/scripts/watch_endpoints.py b/docs/scripts/watch_endpoints.py index b6149fa..445d2cd 100755 --- a/docs/scripts/watch_endpoints.py +++ b/docs/scripts/watch_endpoints.py @@ -1,11 +1,11 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 """Watch endpoint changes for a single service. Requires: - pip install kubernetes + python3.11 -m pip install kubernetes Example: - python docs/scripts/watch_endpoints.py --namespace llm-observability --service ollama --timeout 600 + python3.11 docs/scripts/watch_endpoints.py --namespace llm-observability --service ollama --timeout 600 """ from __future__ import annotations diff --git a/hack/README.md b/hack/README.md index e513a35..8607809 100644 --- a/hack/README.md +++ b/hack/README.md @@ -25,6 +25,8 @@ This directory contains the local image workflow helpers for `llm-observability- - writes `.generated/values.runtime-detected.yaml` - enables NVIDIA scheduling when `nvidia.com/gpu` is advertised - disables NVIDIA runtime and GPU requests when only CPU capacity is available + - remains a thin repo-local Helm overlay helper; durable accelerator detection + and ordered installation are implemented in Go by `edge-cli` ## Typical Usage diff --git a/hack/validate-local-stack.sh b/hack/validate-local-stack.sh index 0f224d0..2e8e09d 100755 --- a/hack/validate-local-stack.sh +++ b/hack/validate-local-stack.sh @@ -7,7 +7,7 @@ if [[ "${1:-}" == "--strict-gpu" ]]; then STRICT_GPU=true fi -PYTHON_BIN="${PYTHON_BIN:-python3}" +PYTHON_BIN="${PYTHON_BIN:-python3.11}" for command in helm kubectl "$PYTHON_BIN"; do command -v "$command" >/dev/null || { echo "missing required command: $command" >&2; exit 1; } @@ -18,7 +18,7 @@ helm lint . helm template llm-observability-stack . >/dev/null helm template llm-observability-stack . \ -f values.full-stack-nvidia.example.yaml \ - --set opentelemetry.tracing.enabled="" \ + --set opentelemetry.tracing.enabled=false \ --set openWebUI.existingSecret="" \ --set open-webui.webuiSecret.existingSecretName="" >/dev/null "$PYTHON_BIN" -m pytest -q tests diff --git a/internal/stack/options.go b/internal/stack/options.go index 44e3564..7d4bfe0 100644 --- a/internal/stack/options.go +++ b/internal/stack/options.go @@ -31,7 +31,7 @@ func DefaultOptions() Options { Namespace: "llm-observability", Release: "llm-observability-stack", Timeout: "5m", - Model: "gemma3-1b-it-gguf-local", + Model: "qwen-1-8b-chat-q4-k-m-local", Runs: 3, Prompt: "Explain GPU observability in one concise sentence.", Output: "artifacts/benchmark-local.json", diff --git a/internal/stack/options_test.go b/internal/stack/options_test.go index c218e00..506a14c 100644 --- a/internal/stack/options_test.go +++ b/internal/stack/options_test.go @@ -33,6 +33,16 @@ func TestGPUProfile(t *testing.T) { } } +func TestGeForceDefaultsUseQwenAndPython311(t *testing.T) { + opts := DefaultOptions() + if opts.Model != "qwen-1-8b-chat-q4-k-m-local" { + t.Fatalf("default model = %q, want Qwen local alias", opts.Model) + } + if !stringsContains(benchmarkCommand(opts), "python3.11 benchmarks/ollama_benchmark.py") { + t.Fatalf("benchmark command must use Python 3.11") + } +} + func TestHelmInstallDisablesBaseLayerChartsForGPUProfiles(t *testing.T) { cmd := helmInstallCommand(DefaultOptions()) for _, want := range []string{ diff --git a/internal/stack/workflows.go b/internal/stack/workflows.go index 01b5f83..9474b7a 100644 --- a/internal/stack/workflows.go +++ b/internal/stack/workflows.go @@ -159,7 +159,7 @@ func baseReadySteps(opts Options) []edgebase.Step { func stackDoctorSteps(opts Options) []edgebase.Step { ns := shellQuote(opts.Namespace) steps := []edgebase.Step{ - {Name: "Required commands", Command: "missing=0; for c in kubectl helm python3; do command -v $c >/dev/null && echo \"$c: $(command -v $c)\" || { echo \"$c: missing\"; missing=1; }; done; exit $missing"}, + {Name: "Required commands", Command: "missing=0; for c in kubectl helm python3.11; do command -v $c >/dev/null && echo \"$c: $(command -v $c)\" || { echo \"$c: missing\"; missing=1; }; done; exit $missing"}, {Name: "Helm release", Command: fmt.Sprintf("helm status %s -n %s || true", shellQuote(opts.Release), ns)}, {Name: "LLM namespace", Command: fmt.Sprintf("kubectl get namespace %s || true", ns)}, {Name: "LLM workloads", Command: fmt.Sprintf("kubectl get pods,deploy,statefulset,svc,pvc -n %s -o wide || true", ns)}, @@ -266,7 +266,7 @@ func helmInstallCommand(opts Options) string { func benchmarkCommand(opts Options) string { return withRoot(fmt.Sprintf(`set -euo pipefail -command -v python3 >/dev/null +command -v python3.11 >/dev/null kubectl rollout status deploy/ollama -n %s --timeout=%s pf_log="$(mktemp)" kubectl port-forward -n %s svc/ollama 11434:11434 >"$pf_log" 2>&1 & @@ -274,7 +274,7 @@ pf_pid="$!" cleanup() { kill "$pf_pid" >/dev/null 2>&1 || true; rm -f "$pf_log"; } trap cleanup EXIT sleep 3 -python3 benchmarks/ollama_benchmark.py --url http://127.0.0.1:11434/api/generate --model %s --runs %d --warmup-runs 1 --prompt %s --output %s`, +python3.11 benchmarks/ollama_benchmark.py --url http://127.0.0.1:11434/api/generate --model %s --runs %d --warmup-runs 1 --prompt %s --output %s`, shellQuote(opts.Namespace), shellQuote(opts.Timeout), shellQuote(opts.Namespace), diff --git a/jupyter-notebooks/llm-observability-in-action/python/01_namespace_inventory.py b/jupyter-notebooks/llm-observability-in-action/python/01_namespace_inventory.py index 7e48031..c46378c 100755 --- a/jupyter-notebooks/llm-observability-in-action/python/01_namespace_inventory.py +++ b/jupyter-notebooks/llm-observability-in-action/python/01_namespace_inventory.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 """Comprehensive namespace inventory for llm-observability in-action.""" from __future__ import annotations diff --git a/jupyter-notebooks/llm-observability-in-action/python/02_service_path_inspector.py b/jupyter-notebooks/llm-observability-in-action/python/02_service_path_inspector.py index 4ee4437..3515af5 100755 --- a/jupyter-notebooks/llm-observability-in-action/python/02_service_path_inspector.py +++ b/jupyter-notebooks/llm-observability-in-action/python/02_service_path_inspector.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 """Trace service routing path: selector -> pods -> endpoints -> routes.""" from __future__ import annotations diff --git a/jupyter-notebooks/llm-observability-in-action/python/03_workload_health.py b/jupyter-notebooks/llm-observability-in-action/python/03_workload_health.py index 98f280f..3622001 100755 --- a/jupyter-notebooks/llm-observability-in-action/python/03_workload_health.py +++ b/jupyter-notebooks/llm-observability-in-action/python/03_workload_health.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 """Check rollout/availability health for namespace workloads.""" from __future__ import annotations diff --git a/jupyter-notebooks/llm-observability-in-action/python/04_networking_report.py b/jupyter-notebooks/llm-observability-in-action/python/04_networking_report.py index f3f6631..49649a8 100755 --- a/jupyter-notebooks/llm-observability-in-action/python/04_networking_report.py +++ b/jupyter-notebooks/llm-observability-in-action/python/04_networking_report.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 """Build a namespace networking report and optional JSON output.""" from __future__ import annotations diff --git a/jupyter-notebooks/llm-observability-in-action/python/05_watch_events.py b/jupyter-notebooks/llm-observability-in-action/python/05_watch_events.py index 74e3141..302dafc 100755 --- a/jupyter-notebooks/llm-observability-in-action/python/05_watch_events.py +++ b/jupyter-notebooks/llm-observability-in-action/python/05_watch_events.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 """Watch Kubernetes events in a namespace for troubleshooting.""" from __future__ import annotations diff --git a/jupyter-notebooks/llm-observability-in-action/python/README.md b/jupyter-notebooks/llm-observability-in-action/python/README.md index 7475aff..4f149ad 100644 --- a/jupyter-notebooks/llm-observability-in-action/python/README.md +++ b/jupyter-notebooks/llm-observability-in-action/python/README.md @@ -6,9 +6,7 @@ These scripts use the official Kubernetes Python client and are focused on your ```bash PYTHON_BIN="${PYTHON_BIN:-python3.11}" -"${PYTHON_BIN}" -m venv .venv -source .venv/bin/activate -pip install -r python/requirements.txt +"${PYTHON_BIN}" -m pip install -r python/requirements.txt ``` ## Scripts diff --git a/jupyter-notebooks/llm-observability-in-action/runbooks/run_python_suite.sh b/jupyter-notebooks/llm-observability-in-action/runbooks/run_python_suite.sh index 38b2dfd..475e81c 100755 --- a/jupyter-notebooks/llm-observability-in-action/runbooks/run_python_suite.sh +++ b/jupyter-notebooks/llm-observability-in-action/runbooks/run_python_suite.sh @@ -8,7 +8,7 @@ K8S_NAMESPACE="llm-observability" OPENWEBUI_SERVICE="open-webui" OLLAMA_SERVICE="ollama" LANGCHAIN_SERVICE="langchain-demo" -PYTHON_BIN="${PYTHON_BIN:-python3}" +PYTHON_BIN="${PYTHON_BIN:-python3.11}" if [[ -f "${ROOT_DIR}/config.env" ]]; then # shellcheck disable=SC1091