diff --git a/Modelfile.gemma-3-1b-it-gguf b/Modelfile.gemma-3-1b-it-gguf index c779db5..76cd00c 100644 --- a/Modelfile.gemma-3-1b-it-gguf +++ b/Modelfile.gemma-3-1b-it-gguf @@ -1,4 +1,10 @@ -# Modelfile for local GGUF (Gemma-3-1B-It Q4_K_M) +# Modelfile for local GGUF (Gemma-3-1B-It Q4_K_M). +# +# The 1 GiB GeForce 940M profile uses a small context and batch with an explicit +# partial layer offload. Ollama/llama.cpp keeps remaining layers in system RAM. +# num_gpu controls layers, not bytes; live validation enforces the 900 MiB total +# observed VRAM ceiling. This profile measured 450 MiB total device use with a +# 62% CPU / 38% GPU processor split on July 19, 2026. FROM /models/gguf/gemma-3-1b-it-Q4_K_M.gguf @@ -23,8 +29,10 @@ PARAMETER min_p 0.01 PARAMETER repeat_penalty 1 PARAMETER top_k 64 PARAMETER top_p 0.95 -PARAMETER num_predict 32768 +PARAMETER num_predict 128 PARAMETER stop PARAMETER stop PARAMETER temperature 1 -PARAMETER num_ctx 1024 +PARAMETER num_gpu 23 +PARAMETER num_ctx 256 +PARAMETER num_batch 1 diff --git a/Modelfile.llama3.2-1b b/Modelfile.llama3.2-1b new file mode 100644 index 0000000..17c1bde --- /dev/null +++ b/Modelfile.llama3.2-1b @@ -0,0 +1,20 @@ +# Low-VRAM derivative of the official Ollama Llama 3.2 1B model. +# +# Upstream: https://ollama.com/library/llama3.2:1b +# License: Llama 3.2 Community License; review the upstream license before use. +# +# FROM intentionally references the Ollama registry model. Ollama retains its +# official chat template and tokenizer metadata while this derivative pins the +# small-context settings used by the single-node GeForce 940M profile. +# num_gpu is an offload-layer control, not a hard VRAM quota. The repository's +# live validation records total device usage and rejects observations above the +# 900 MiB operational ceiling. Eight GPU layers measured 541 MiB total device +# use with a 52% CPU / 48% GPU processor split on July 19, 2026. + +FROM llama3.2:1b + +PARAMETER num_gpu 8 +PARAMETER num_ctx 256 +PARAMETER num_batch 1 +PARAMETER num_predict 128 +PARAMETER temperature 0 diff --git a/README.md b/README.md index 95ca7ad..51873ca 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,8 @@ The current local deployment target is a single-node Xubuntu 24 system running k - k3s node: combined control-plane and worker. - NVIDIA device plugin resource: `nvidia.com/gpu: 1`. - RuntimeClass: `nvidia`. -- Model: Qwen 1.8B Chat Q4_K_M GGUF through the local alias - `qwen-1-8b-chat-q4-k-m-local`. +- Model profiles: Qwen 1.8B Chat Q4_K_M, Gemma 3 1B IT Q4_K_M, and + Ollama `llama3.2:1b`, deployed one at a time with partial CUDA offload. Measured after deployment, warmup, and exact-response, arithmetic, and translation inference checks: @@ -69,7 +69,8 @@ Evidence and reproduction: - [Xubuntu k3s NVIDIA runbook](docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md) - [Sanitized benchmark artifact](artifacts/geforce-940m-benchmark.json) - [GeForce 940M Helm profile](values.geforce-940m-k3s.yaml) -- [Qwen GGUF runtime evidence companion](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) +- [Multi-model GGUF runtime evidence companion](https://github.com/Edge-Computing-LLM/gguf-observability) +- [Sequential three-model deployment and validation](docs/MULTI-MODEL-LOW-VRAM.md) These numbers prove constrained local edge feasibility. They do not claim enterprise load, concurrency, fleet reliability, or production readiness. @@ -249,6 +250,27 @@ helm upgrade --install llm-observability-stack . \ ./hack/test-geforce-940m-inference.sh ``` +Select Gemma or Llama by adding the matching overlay after the base GeForce +values file: + +```bash +# Gemma 3 1B IT Q4_K_M +helm upgrade --install llm-observability-stack . \ + -n llm-observability --create-namespace \ + -f values.geforce-940m-k3s.yaml \ + -f values.gemma-3-1b-geforce-940m-k3s.yaml + +# Llama 3.2 1B +helm upgrade --install llm-observability-stack . \ + -n llm-observability --create-namespace \ + -f values.geforce-940m-k3s.yaml \ + -f values.llama3.2-1b-geforce-940m-k3s.yaml +``` + +Use only one profile at a time on the 1 GiB GPU. The Modelfile `num_gpu` +setting controls partial layer offload; read-only runtime evidence enforces a +900 MiB total observed VRAM ceiling. + ### C. Full observability NVIDIA profile ```bash diff --git a/dashboards/edge-llm-observability.json b/dashboards/edge-llm-observability.json index 1987c6b..81664af 100644 --- a/dashboards/edge-llm-observability.json +++ b/dashboards/edge-llm-observability.json @@ -231,7 +231,7 @@ "gridPos": {"h": 11, "w": 12, "x": 0, "y": 41}, "options": { "mode": "markdown", - "content": "| Namespace | Service | Type | Ports | Purpose |\n|---|---|---|---|---|\n| gpu-operator | nvidia-dcgm-exporter | ClusterIP | 9400/TCP | GPU metrics scrape target |\n| llm-observability | ollama | ClusterIP | 11434/TCP | Qwen GGUF inference API |\n| llm-observability | open-webui | ClusterIP | 8080/TCP | Browser chat service |\n| llm-observability | open-webui-redis | ClusterIP | 6379/TCP | WebUI state and websocket support |\n| llm-observability | opentelemetry-collector | ClusterIP | 4317, 4318, 8888/TCP | OTLP and collector metrics |\n\nKubernetes remains the source of truth. This chart-owned table documents stable service contracts without publishing ClusterIPs." + "content": "| Namespace | Service | Type | Ports | Purpose |\n|---|---|---|---|---|\n| gpu-operator | nvidia-dcgm-exporter | ClusterIP | 9400/TCP | GPU metrics scrape target |\n| llm-observability | ollama | ClusterIP | 11434/TCP | Selected GGUF model inference API |\n| llm-observability | open-webui | ClusterIP | 8080/TCP | Browser chat service |\n| llm-observability | open-webui-redis | ClusterIP | 6379/TCP | WebUI state and websocket support |\n| llm-observability | opentelemetry-collector | ClusterIP | 4317, 4318, 8888/TCP | OTLP and collector metrics |\n\nKubernetes remains the source of truth. This chart-owned table documents stable service contracts without publishing ClusterIPs." } }, { @@ -315,7 +315,7 @@ "gridPos": {"h": 8, "w": 24, "x": 0, "y": 59}, "options": { "mode": "markdown", - "content": "### LLM observability readiness\n\n- **DCGM exporter:** live GPU utilization, framebuffer, temperatures, PCIe counters, and clocks come from the GPU Operator service.\n- **Qwen runtime:** the GeForce profile requests 23 GPU layers, context 256, batch 1, one parallel request, and infinite keep-alive.\n- **Runtime contract:** qwen-gguf-observability remains the authoritative read-only validator for node, GPU, Helm, pod, model-parameter, residency, and 850 MiB ceiling checks. Grafana visualizes metrics but does not replace that validator.\n- **OpenTelemetry:** the collector accepts OTLP metrics, traces, and logs on the chart-owned service.\n\nPrometheus is authoritative for live numeric and Kubernetes readiness panels. Static text records stable Helm-owned configuration and contains no ClusterIPs, pod identifiers, credentials, or public tunnel hostnames." + "content": "### LLM observability readiness\n\n- **DCGM exporter:** live GPU utilization, framebuffer, temperatures, PCIe counters, and clocks come from the GPU Operator service.\n- **Low-VRAM runtime:** model overlays request a small context, batch 1, one parallel request, infinite keep-alive, and model-specific partial GPU layer offload.\n- **Runtime contract:** gguf-observability remains the authoritative read-only validator for node, GPU, Helm, pod, model parameters, residency, and the 900 MiB observed ceiling. Grafana visualizes metrics but does not replace that validator.\n- **OpenTelemetry:** the collector accepts OTLP metrics, traces, and logs on the chart-owned service.\n\nPrometheus is authoritative for live numeric and Kubernetes readiness panels. Static text records stable Helm-owned configuration and contains no ClusterIPs, pod identifiers, credentials, or public tunnel hostnames." } } ], diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b64777e..5f561c9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -9,7 +9,7 @@ is a repo-local helper for this chart and should not be used as the primary cross-repository installer. The optional -[`qwen-gguf-observability`](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) +[`gguf-observability`](https://github.com/Edge-Computing-LLM/gguf-observability) repository is a read-only evidence consumer. It observes selected Kubernetes, Helm, Ollama, and GPU status after deployment; it owns no chart resources, Modelfile, model lifecycle, or telemetry backend. @@ -219,7 +219,7 @@ The verified local NVIDIA flow depends on `k3s-nvidia-edge` for: ## 10. Downstream evidence companion -After this stack is healthy, `qwen-gguf-observability` can validate the narrow -GeForce/Qwen runtime contract and write sanitized JSON or Markdown evidence. +After this stack is healthy, `gguf-observability` can validate a selected +GeForce/GGUF runtime contract and write sanitized JSON or Markdown evidence. Deployment changes discovered by those checks must be made here or in the infrastructure repository, not copied into the evidence tool. diff --git a/docs/DEPENDENCY-AUDIT-2026-07-19.md b/docs/DEPENDENCY-AUDIT-2026-07-19.md new file mode 100644 index 0000000..c6e87dc --- /dev/null +++ b/docs/DEPENDENCY-AUDIT-2026-07-19.md @@ -0,0 +1,45 @@ +# Dependency audit: 2026-07-19 + +The local reference repositories and GitHub releases were fetched before this +comparison. Layer 1 is current; several Layer 2 charts have newer upstream +releases. + +| Component | Repository version | Current upstream | Decision | +|---|---:|---:|---| +| k3s | v1.36.2+k3s1 | v1.36.2+k3s1 | Current | +| NVIDIA GPU Operator | v26.3.3 | v26.3.3 | Current | +| Ollama Helm chart | 1.50.0 | 1.67.0 | Migration required | +| Ollama image | 0.17.7 | 0.32.1 | Migration required | +| Open WebUI Helm chart | 12.10.0 | 15.2.0 | Migration required | +| Open WebUI image | 0.8.10 | 0.10.2 | Migration required | +| kube-prometheus-stack | 86.2.3 | 87.17.0 | Review CRD/rule changes | +| OpenTelemetry Collector chart | 0.158.1 | 0.165.0 | Migration required | +| OpenTelemetry Operator chart | 0.115.0 | 0.120.0 | Review CRD changes | + +Sources: [k3s releases](https://github.com/k3s-io/k3s/releases), +[GPU Operator releases](https://github.com/NVIDIA/gpu-operator/releases), +[Ollama releases](https://github.com/ollama/ollama/releases), +[Ollama chart](https://github.com/otwld/ollama-helm), +[Open WebUI releases](https://github.com/open-webui/open-webui/releases), +[Open WebUI charts](https://github.com/open-webui/helm-charts), +[Prometheus Community charts](https://github.com/prometheus-community/helm-charts), +and [OpenTelemetry charts](https://github.com/open-telemetry/opentelemetry-helm-charts). + +## Validation result + +The existing pins passed module verification, formatting, unit tests, race +tests, vet, `govulncheck`, builds, Helm lint, and default/CPU/local/GeForce/full +NVIDIA rendering with Helm 4.2.3. GitHub's current `main` workflow is also +green. + +The live cluster was not suitable for a safe multi-component upgrade because a +post-reboot stale k3s node address and Flannel interface were breaking pod +networking. Upgrading the charts during that fault would mix configuration and +version variables and would not produce trustworthy evidence. The versions +above are therefore recorded as a dedicated migration, not changed in place. + +Layer 2 now depends on the Layer 1 commit that detects stale node networking and +unhealthy GPU Operator pods. Once the host network is repaired, upgrade and +validate the components in this order: observability CRDs/operators, +Prometheus, Ollama, then Open WebUI. Preserve the GeForce 940M resource limits, +model-cleanup rejection, existing PVCs, and rollback values between steps. diff --git a/docs/LIVE-VALIDATION-GO-NATIVE-2026-07-18.md b/docs/LIVE-VALIDATION-GO-NATIVE-2026-07-18.md index 50dc2fe..7462d4f 100644 --- a/docs/LIVE-VALIDATION-GO-NATIVE-2026-07-18.md +++ b/docs/LIVE-VALIDATION-GO-NATIVE-2026-07-18.md @@ -43,7 +43,7 @@ expected k3s/containerd restart triggered by NVIDIA toolkit reconciliation. - `go test ./...` and `go vet ./...` for all four first-party repositories. - Go builds for `edge`, `k3s-nvidia-edge`, `llm-observability`, - `ollama-gateway`, `edge-toolbox`, and `qwen-observe`. + `ollama-gateway`, `edge-toolbox`, and the now-renamed `gguf-observe`. - `helm lint` for both project charts. - Helm rendering for default, CPU, enterprise, full-NVIDIA, local, and GeForce profiles. diff --git a/docs/MULTI-MODEL-LOW-VRAM.md b/docs/MULTI-MODEL-LOW-VRAM.md new file mode 100644 index 0000000..58ce2fe --- /dev/null +++ b/docs/MULTI-MODEL-LOW-VRAM.md @@ -0,0 +1,67 @@ +# Sequential GGUF validation on a 1 GiB NVIDIA GPU + +This runbook deploys one model at a time on the GeForce 940M reference system. +It uses partial CUDA layer offload and keeps the remaining model layers in +system RAM. + +## Why the limit is observational + +Kubernetes requests `nvidia.com/gpu: 1`, which assigns the whole GPU. The +GeForce 940M does not support MIG memory partitions, and Ollama does not expose +a per-container hard VRAM quota. Each Modelfile therefore pins `num_gpu`, +`num_ctx`, and `num_batch`, while `gguf-observability` rejects total +`nvidia-smi` usage above 900 MiB. + +This is a guardrail, not a hardware-enforced memory sandbox. Display-server and +other host allocations count toward the measured total. + +## Profiles + +| Model | Values files | Ollama alias | +|---|---|---| +| Qwen 1.8B Chat Q4_K_M | `values.geforce-940m-k3s.yaml` | `qwen-1-8b-chat-q4-k-m-local` | +| Gemma 3 1B IT Q4_K_M | base plus `values.gemma-3-1b-geforce-940m-k3s.yaml` | `gemma3-1b-it-gguf-local` | +| Llama 3.2 1B | base plus `values.llama3.2-1b-geforce-940m-k3s.yaml` | `llama3-2-1b-local` | + +Qwen and Gemma use read-only host-mounted GGUF files. Llama derives from the +official Ollama `llama3.2:1b` registry model and inherits its maintained chat +template. + +## Safe sequence + +1. Deploy and validate `k3s-nvidia-edge`. +2. Install one profile with `helm upgrade --install`. +3. Wait for `deployment/ollama` and confirm exactly one entry in `ollama ps`. +4. Run a fixed public smoke prompt. +5. Capture selected facts with `gguf-observe`; never persist the response. +6. Change to the next overlay. `OLLAMA_MAX_LOADED_MODELS=1` prevents a second + runner from remaining resident. +7. Confirm total observed VRAM remains at or below 900 MiB after every warmup. + +Example observer call: + +```bash +GGUF_MODEL=gemma3-1b-it-gguf-local \ +GGUF_VRAM_CEILING_MIB=900 \ +GGUF_EXPECTED_NUM_GPU=23 \ +GGUF_EXPECTED_NUM_CTX=256 \ +GGUF_EXPECTED_NUM_BATCH=1 \ +../gguf-observability/bin/gguf-observe validate +``` + +The Ollama volume is retained across Helm upgrades. Do not delete the PVC or +host GGUF directory during a profile change. + +## Validated results + +On July 19, 2026 all three aliases passed registration, residency, GPU-active, +keep-alive, effective-parameter, workload, and 900 MiB ceiling checks: + +| Model | CPU/GPU split | Total observed VRAM | Fixed smoke duration | +|---|---:|---:|---:| +| Gemma 3 1B IT | 62% / 38% | 450 MiB | 1.297 s | +| Qwen 1.8B Chat | 27% / 73% | 824 MiB | 1.932 s | +| Llama 3.2 1B | 52% / 48% | 541 MiB | 1.591 s | + +These are local observations, not portable performance claims. Sanitized +evidence is owned by `gguf-observability`. diff --git a/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md b/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md index 2332853..4154c60 100644 --- a/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md +++ b/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md @@ -93,5 +93,5 @@ The Qwen model was deliberately left loaded in GPU memory after validation. Ongoing read-only contract validation and sanitized evidence capture are kept in the complementary -[`qwen-gguf-observability`](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) +[`gguf-observability`](https://github.com/Edge-Computing-LLM/gguf-observability) repository. This stack remains the deployment and model-configuration owner. diff --git a/docs/README.md b/docs/README.md index 9c9d1cf..a9cfb53 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,24 +6,25 @@ The documentation is organized around private LLM deployment and observability o ## Start Here -1. [QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md](QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md) -2. [LIVE-VALIDATION-GO-NATIVE-2026-07-18.md](LIVE-VALIDATION-GO-NATIVE-2026-07-18.md) -3. [LIVE-VALIDATION-2026-07-17.md](LIVE-VALIDATION-2026-07-17.md) -4. [QUICKSTART.md](QUICKSTART.md) -5. [cli.md](cli.md) -6. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) -7. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) -8. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) -9. [ARCHITECTURE.md](ARCHITECTURE.md) -10. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) -11. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) -12. [LANGUAGE-BOUNDARIES.md](LANGUAGE-BOUNDARIES.md) -13. [GO-NATIVE-MIGRATION-2026-07-18.md](GO-NATIVE-MIGRATION-2026-07-18.md) +1. [DEPENDENCY-AUDIT-2026-07-19.md](DEPENDENCY-AUDIT-2026-07-19.md) +2. [QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md](QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md) +3. [LIVE-VALIDATION-GO-NATIVE-2026-07-18.md](LIVE-VALIDATION-GO-NATIVE-2026-07-18.md) +4. [LIVE-VALIDATION-2026-07-17.md](LIVE-VALIDATION-2026-07-17.md) +5. [QUICKSTART.md](QUICKSTART.md) +6. [cli.md](cli.md) +7. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) +8. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) +9. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) +10. [ARCHITECTURE.md](ARCHITECTURE.md) +11. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) +12. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) +13. [LANGUAGE-BOUNDARIES.md](LANGUAGE-BOUNDARIES.md) +14. [GO-NATIVE-MIGRATION-2026-07-18.md](GO-NATIVE-MIGRATION-2026-07-18.md) External companion: -- [`qwen-gguf-observability`](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) - provides read-only Qwen runtime contract checks and sanitized evidence. It +- [`gguf-observability`](https://github.com/Edge-Computing-LLM/gguf-observability) + provides read-only, model-selectable runtime contract checks and sanitized evidence. It does not replace this repository's Helm, Modelfile, or benchmark assets. Dashboard presentation is now owned by the Helm-provisioned Grafana JSON in @@ -32,6 +33,9 @@ browser-side Kubernetes access path. ## Core Guides +- [MULTI-MODEL-LOW-VRAM.md](MULTI-MODEL-LOW-VRAM.md) + - Sequential Qwen, Gemma, and Llama deployment with a 900 MiB observed VRAM ceiling. + - [QUICKSTART.md](QUICKSTART.md) - Fast local setup for k3s, values files, image build/import, install, and first validation. - [cli.md](cli.md) diff --git a/docs/cli.md b/docs/cli.md index 6ebb567..c1cfd6d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -8,7 +8,7 @@ primary organization-level CLI for end-to-end installs and uninstalls. - `k3s-nvidia-edge` owns the local Linux, k3s, k3s containerd, NVIDIA runtime, GPU Operator, NVIDIA device plugin, DCGM exporter, Node Feature Discovery, `RuntimeClass/nvidia`, and `nvidia.com/gpu` validation layer. - `llm-observability-stack` owns Ollama, Open WebUI, Open WebUI Redis, OpenTelemetry Collector, optional Prometheus/Grafana, the optional native Go Ollama gateway, benchmark tooling, notebooks, and local model configuration. - `edge-cli` owns cross-repository ordering: infra first, observability second. -- `qwen-gguf-observability` optionally consumes the deployed state for read-only +- `gguf-observability` optionally consumes the deployed state for read-only contract checks and evidence; it owns no install or uninstall workflow. - The CLI does not import `k3s-nvidia-edge/internal/...`. diff --git a/go.mod b/go.mod index 4b8c764..2667da9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/Edge-Computing-LLM/llm-observability-stack go 1.25.0 require ( - github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013 + github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260719035336-95f5e6cb9c29 github.com/prometheus/client_golang v1.23.2 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 diff --git a/go.sum b/go.sum index 8dfbe66..611dbf1 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013 h1:yxsjOaobZcrVBnbF/Usz5HQ77HExRcOVkCi2o8dZhgs= -github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013/go.mod h1:+7CsSGQtUt/2vgTNKT6NFOjx5gkIhrNIK4YFw4/IVyA= +github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260719035336-95f5e6cb9c29 h1:apfG7YGso9zbX4y8ztOoAAeYUv/9ANP2cnUHBZxH76w= +github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260719035336-95f5e6cb9c29/go.mod h1:+7CsSGQtUt/2vgTNKT6NFOjx5gkIhrNIK4YFw4/IVyA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= diff --git a/internal/stack/options_test.go b/internal/stack/options_test.go index 2ffe2a7..1f86338 100644 --- a/internal/stack/options_test.go +++ b/internal/stack/options_test.go @@ -57,6 +57,18 @@ func TestHelmInstallDisablesBaseLayerChartsForGPUProfiles(t *testing.T) { } } +func TestBaseReadinessRejectsStaleNetworkAndUnhealthyOperator(t *testing.T) { + all := "" + for _, step := range baseReadySteps(DefaultOptions()) { + all += step.Name + "\n" + step.Command + "\n" + } + for _, want := range []string{"k3s node address", "InternalIP", "GPU Operator health", "unhealthy pods"} { + if !stringsContains(all, want) { + t.Fatalf("base readiness missing %q", want) + } + } +} + func TestInstallWithBaseIsDeprecated(t *testing.T) { opts := DefaultOptions() opts.WithBase = true diff --git a/internal/stack/workflows.go b/internal/stack/workflows.go index 173333e..4f46d83 100644 --- a/internal/stack/workflows.go +++ b/internal/stack/workflows.go @@ -180,13 +180,14 @@ func baseReady(ctx context.Context, r *edgebase.Runner, opts Options) error { func baseReadySteps(opts Options) []edgebase.Step { steps := []edgebase.Step{ {Name: "Kubernetes connectivity", Command: "kubectl cluster-info"}, + {Name: "k3s node address", Command: edgebase.NodeAddressHealthCheck()}, {Name: "k3s nodes", Command: "kubectl get nodes -o wide"}, } if GPUProfile(opts.Profile) { steps = append(steps, edgebase.Step{Name: "NVIDIA RuntimeClass", Command: "kubectl get runtimeclass nvidia"}, edgebase.Step{Name: "NVIDIA GPU allocatable", Command: edgebase.GPUCapacityCheck()}, - edgebase.Step{Name: "GPU Operator namespace", Command: "kubectl get pods -n gpu-operator -o wide"}, + edgebase.Step{Name: "GPU Operator health", Command: edgebase.GPUOperatorHealthCheck()}, ) } return steps diff --git a/templates/ollama-local-modelfile-configmap.yaml b/templates/ollama-local-modelfile-configmap.yaml index 1fd18bf..228f8e9 100644 --- a/templates/ollama-local-modelfile-configmap.yaml +++ b/templates/ollama-local-modelfile-configmap.yaml @@ -4,7 +4,10 @@ {{- if .Values.ollamaModel.modelfile.file -}} {{- $modelfileSource = required (printf "ollamaModel.modelfile.file %q was not packaged in the chart" .Values.ollamaModel.modelfile.file) (.Files.Get .Values.ollamaModel.modelfile.file) -}} {{- end -}} -{{- $modelfile := regexReplaceAll "(?m)^FROM[ \\t]+[^\\r\\n]+$" $modelfileSource (printf "FROM %s" $ggufPath) -}} +{{- $modelfile := $modelfileSource -}} +{{- if .Values.ollamaModel.gguf.enabled -}} +{{- $modelfile = regexReplaceAll "(?m)^FROM[ \\t]+[^\\r\\n]+$" $modelfileSource (printf "FROM %s" $ggufPath) -}} +{{- end -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/tests/helm_smoke_test.go b/tests/helm_smoke_test.go index 58ad433..8330ab7 100644 --- a/tests/helm_smoke_test.go +++ b/tests/helm_smoke_test.go @@ -63,6 +63,37 @@ func TestGeForceProfileContract(t *testing.T) { requireAbsent(t, manifest, "name: ollama-gateway", "name: edge-toolbox", "/bin/ollama rm", "kind: ClusterPolicy") } +func TestGeForceModelOverlays(t *testing.T) { + tests := []struct { + name string + valuesFile string + expected []string + absent []string + }{ + { + name: "gemma local GGUF", + valuesFile: "values.gemma-3-1b-geforce-940m-k3s.yaml", + expected: []string{"gemma3-1b-it-gguf-local", "FROM /models/gguf/gemma-3-1b-it-Q4_K_M.gguf", "PARAMETER num_gpu 23", "PARAMETER num_ctx 256", "PARAMETER num_batch 1"}, + }, + { + name: "llama registry model", + valuesFile: "values.llama3.2-1b-geforce-940m-k3s.yaml", + expected: []string{"llama3-2-1b-local", "llama3.2:1b", "FROM llama3.2:1b", "PARAMETER num_gpu 8", "PARAMETER num_ctx 256", "PARAMETER num_batch 1"}, + absent: []string{"FROM /models/gguf/qwen-1.8b-chat-q4_K_M.gguf"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + manifest, err := helm(t, "template", "llm-observability-stack", ".", "-f", "values.geforce-940m-k3s.yaml", "-f", test.valuesFile) + if err != nil { + t.Fatal(manifest) + } + requireContains(t, manifest, test.expected...) + requireAbsent(t, manifest, test.absent...) + }) + } +} + func TestCPUProfileHasNoNVIDIAScheduling(t *testing.T) { manifest, err := helm(t, "template", "llm-observability-stack", ".", "-f", "values.cpu-k3s.yaml") if err != nil { diff --git a/values.gemma-3-1b-geforce-940m-k3s.yaml b/values.gemma-3-1b-geforce-940m-k3s.yaml new file mode 100644 index 0000000..12a2443 --- /dev/null +++ b/values.gemma-3-1b-geforce-940m-k3s.yaml @@ -0,0 +1,41 @@ +# Apply after values.geforce-940m-k3s.yaml to select the local Gemma GGUF. +ollamaModel: + name: gemma3-1b-it-gguf-local + gguf: + enabled: true + filename: gemma-3-1b-it-Q4_K_M.gguf + modelfile: + file: Modelfile.gemma-3-1b-it-gguf + +ollama: + ollama: + models: + pull: [] + run: + - gemma3-1b-it-gguf-local + runPrompt: "Reply with exactly: model ready" + create: + - name: gemma3-1b-it-gguf-local + configMapRef: ollama-local-modelfile + configMapKeyRef: Modelfile + +open-webui: + extraEnvVars: + - name: OLLAMA_BASE_URL + value: http://ollama:11434 + - name: ENABLE_PERSISTENT_CONFIG + value: "False" + - name: WEBUI_AUTH + value: "False" + - name: ENABLE_VERSION_UPDATE_CHECK + value: "False" + - name: RAG_EMBEDDING_ENGINE + value: ollama + - name: RAG_EMBEDDING_MODEL + value: gemma3-1b-it-gguf-local + - name: RAG_EMBEDDING_MODEL_AUTO_UPDATE + value: "False" + - name: RAG_RERANKING_MODEL_AUTO_UPDATE + value: "False" + - name: WHISPER_MODEL_AUTO_UPDATE + value: "False" diff --git a/values.llama3.2-1b-geforce-940m-k3s.yaml b/values.llama3.2-1b-geforce-940m-k3s.yaml new file mode 100644 index 0000000..de4ec61 --- /dev/null +++ b/values.llama3.2-1b-geforce-940m-k3s.yaml @@ -0,0 +1,41 @@ +# Apply after values.geforce-940m-k3s.yaml to select Ollama's Llama 3.2 1B. +ollamaModel: + name: llama3-2-1b-local + gguf: + enabled: false + modelfile: + file: Modelfile.llama3.2-1b + +ollama: + ollama: + models: + pull: + - llama3.2:1b + run: + - llama3-2-1b-local + runPrompt: "Reply with exactly: model ready" + create: + - name: llama3-2-1b-local + configMapRef: ollama-local-modelfile + configMapKeyRef: Modelfile + +open-webui: + extraEnvVars: + - name: OLLAMA_BASE_URL + value: http://ollama:11434 + - name: ENABLE_PERSISTENT_CONFIG + value: "False" + - name: WEBUI_AUTH + value: "False" + - name: ENABLE_VERSION_UPDATE_CHECK + value: "False" + - name: RAG_EMBEDDING_ENGINE + value: ollama + - name: RAG_EMBEDDING_MODEL + value: llama3-2-1b-local + - name: RAG_EMBEDDING_MODEL_AUTO_UPDATE + value: "False" + - name: RAG_RERANKING_MODEL_AUTO_UPDATE + value: "False" + - name: WHISPER_MODEL_AUTO_UPDATE + value: "False" diff --git a/values.yaml b/values.yaml index b461702..d66dfb6 100644 --- a/values.yaml +++ b/values.yaml @@ -330,11 +330,13 @@ ollamaModel: PARAMETER repeat_penalty 1 PARAMETER top_k 64 PARAMETER top_p 0.95 - PARAMETER num_predict 32768 + PARAMETER num_predict 128 PARAMETER stop PARAMETER stop PARAMETER temperature 1 - PARAMETER num_ctx 1024 + PARAMETER num_gpu 23 + PARAMETER num_ctx 256 + PARAMETER num_batch 1 ollama: enabled: true