diff --git a/.github/workflows/helm-validate.yaml b/.github/workflows/helm-validate.yaml index 7ee2f84..c679103 100644 --- a/.github/workflows/helm-validate.yaml +++ b/.github/workflows/helm-validate.yaml @@ -13,6 +13,12 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + - name: Set up Helm uses: azure/setup-helm@v5 with: @@ -42,13 +48,8 @@ jobs: --set open-webui.webuiSecret.existingSecretName= \ >/tmp/rendered-full-stack-nvidia.yaml - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - - name: Install smoke test dependencies - run: python -m pip install --upgrade pip && pip install -r requirements-test.txt + - name: Run Go tests + run: go test ./... - - name: Run smoke tests - run: pytest -q tests/test_langchain_demo_smoke.py tests/test_helm_smoke.py + - name: Run Go vet + run: go vet ./... diff --git a/.helmignore b/.helmignore index e3f6971..83e98fc 100644 --- a/.helmignore +++ b/.helmignore @@ -34,30 +34,24 @@ tests/ .github/ pictures/ hack/ -benchmarks/ README.md CONTRIBUTING.md SECURITY.md SUPPORT.md CODE_OF_CONDUCT.md llm-observability-files-folders.txt -requirements-test.txt +cmd/ +internal/ +bin/ +go.mod +go.sum manifests.yaml values.local-k3s.example.yaml values.geforce-940m-k3s.yaml -# Keep only chart-consumed source files from local app/tooling trees -langchain-demo/.dockerignore -langchain-demo/Dockerfile -langchain-demo/requirements.txt -langchain-demo/__pycache__/ -python-toolbox/Dockerfile -python-toolbox/README.md -python-toolbox/requirements.txt -python-toolbox/examples/__pycache__/ -python-toolbox/examples/ollama_smoke.py -python-toolbox/examples/redis_ping.py -python-toolbox/examples/service_dns_check.py +# Go application images are built separately and are not embedded in the release. +ollama-gateway/ +edge-toolbox/ # Vendored chart docs/assets are not needed at install time charts/*.tgz diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..c17302e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,13 @@ +# Repository instructions + +This repository owns the Layer 2 Helm release: Ollama/GGUF, Open WebUI, +OpenTelemetry, Prometheus/Grafana, the Go Ollama gateway, Go edge toolbox, +benchmarks, and dashboards. Layer 1 NVIDIA resources belong to +`k3s-nvidia-edge` and must not be duplicated. + +The deployable runtime and automated tests are Go-first. Python is allowed only +inside optional Jupyter learning assets and untouched vendored upstream charts. +Before completing a change run `gofmt`, `go test ./...`, `go vet ./...`, +`helm lint .`, and render the CPU and GeForce profiles. Never commit model +weights, API keys, Kubernetes Secrets, kubeconfigs, or private prompt content. +Keep mutations behind `--yes` and retain model-cleanup rejection safeguards. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab2606d..3bd6971 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,14 +8,14 @@ This repository is focused on local Kubernetes observability workflows for: - Ollama - Open WebUI -- LangChain demo + OpenTelemetry tracing +- Native Go Ollama gateway + OpenTelemetry tracing Please keep changes aligned with local k3s reproducibility and interview-support use cases. ## Development Setup 1. Clone the repository. -2. Install prerequisites: `helm`, `kubectl`, and Docker or `nerdctl`. +2. Install prerequisites: Go 1.25+, `helm`, `kubectl`, and Docker or `nerdctl`. 3. Build chart dependencies: ```bash @@ -38,7 +38,8 @@ Run these commands before opening a PR: helm lint . helm template llm-observability-stack . >/tmp/rendered-default.yaml helm template llm-observability-stack . -f values.local-k3s.example.yaml >/tmp/rendered-local.yaml -pytest -q tests/test_helm_smoke.py tests/test_langchain_demo_smoke.py +go test ./... +go vet ./... ``` If your change touches local runtime behavior, include the command output and manual test notes in the PR description. @@ -66,15 +67,16 @@ Use the smallest validation set that actually exercises your change: - `helm lint .` - `helm template llm-observability-stack .` - `helm template llm-observability-stack . -f values.local-k3s.example.yaml` -- Python or template logic changes: - - `pytest -q tests/test_helm_smoke.py tests/test_langchain_demo_smoke.py` +- Go or template logic changes: + - `go test ./...` + - `go vet ./...` - Notebook source changes: - `find jupyter-notebooks -type f -name '*.ipynb' ! -path '*/.ipynb_checkpoints/*' -print0 | xargs -0 -n1 jq empty` ## Documentation and Notebook Style - Prefer relative paths and environment variables over machine-specific absolute paths. -- Prefer `PYTHON_BIN` examples over hard-coded interpreter paths in user-facing guides. +- Keep Python confined to optional Jupyter learning material; runtime and validation examples use Go binaries. - Keep secrets out of notebook source and output. - When describing runtime defaults, point readers to [docs/CONFIG-PROFILES.md](docs/CONFIG-PROFILES.md) instead of duplicating tables everywhere. diff --git a/Chart.yaml b/Chart.yaml index daef376..c79d554 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -7,8 +7,8 @@ description: | Prometheus metrics, Grafana dashboards, blackbox probes, benchmark artifacts, and NVIDIA/DCGM-ready integrations. type: application -version: 0.2.0 -appVersion: "1.1.0" +version: 0.3.0 +appVersion: "1.2.0" kubeVersion: ">=1.27.0-0" home: https://github.com/Edge-Computing-LLM/llm-observability-stack sources: diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e27f331 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +GO ?= go + +.PHONY: build test vet helm-check check + +build: + @mkdir -p bin + $(GO) build -o bin/llm-observability ./cmd/llm-observability + $(GO) build -o bin/ollama-gateway ./cmd/ollama-gateway + $(GO) build -o bin/edge-toolbox ./cmd/edge-toolbox + +test: + $(GO) test ./... + +vet: + $(GO) vet ./... + +helm-check: + helm lint . + helm template llm-observability-stack . -f values.cpu-k3s.yaml >/dev/null + helm template llm-observability-stack . -f values.geforce-940m-k3s.yaml >/dev/null + +check: test vet helm-check build diff --git a/README.md b/README.md index 7412150..95ca7ad 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Kubernetes-native observability, benchmarking, and operations tooling for privat Preferred organization CLI: [`edge-cli`](https://github.com/Edge-Computing-LLM/edge-cli). Repo-local legacy/helper CLI documentation: [docs/cli.md](docs/cli.md). -This repository packages a Helm-based application and observability stack for k3s and Kubernetes with Ollama/GGUF model serving, Open WebUI, an OpenTelemetry GenAI-instrumented FastAPI proxy, Prometheus, Grafana, OpenTelemetry Collector, blackbox probes, benchmark metrics, and NVIDIA/DCGM-compatible dashboards. +This repository packages a Helm-based application and observability stack for k3s and Kubernetes with Ollama/GGUF model serving, Open WebUI, a native Go OpenTelemetry GenAI gateway, Prometheus, Grafana, OpenTelemetry Collector, blackbox probes, benchmark metrics, and NVIDIA/DCGM-compatible dashboards. The repository also includes a Go CLI named `llm-observability` for repo-local helper workflows. New end-to-end installs should use `edge-cli`, which deploys `k3s-nvidia-edge` first and then this chart. @@ -30,10 +30,10 @@ Read the full dependency guide before installing GPU profiles: - NVIDIA GPU scheduling with `runtimeClassName: nvidia` and `nvidia.com/gpu` when a GPU is available. - Optional CPU validation profiles for development clusters without NVIDIA GPUs. - Open WebUI for browser-based interaction with local models. -- A FastAPI proxy with LLM request metrics for TTFT, latency, tokens per second, prompt tokens, generated tokens, active requests, and errors. +- A native Go Ollama gateway with streaming, OpenTelemetry traces, and LLM request metrics for TTFT, latency, active requests, and errors. - Prometheus, Grafana, Alertmanager, kube-state-metrics, node exporter, ServiceMonitors, probes, and alert rules. - OpenTelemetry Collector endpoints for OTLP traces, metrics, and logs. -- Optional diagnostics workloads including Python toolbox, Redis checks, OpenTelemetry seeding, and benchmark reporting. +- Optional native Go diagnostics including DNS/HTTP/TCP checks, Redis checks, OpenTelemetry seeding, and benchmark reporting. ## Verified Local NVIDIA GPU Deployment @@ -77,10 +77,11 @@ The companion repository performs read-only runtime contract checks and captures sanitized evidence. This chart remains the source of truth for the Modelfile, Helm values, model lifecycle, and workload configuration. -The separate -[`Frontend-Edge-LLM-Observability`](https://github.com/Edge-Computing-LLM/Frontend-Edge-LLM-Observability) -project provides the TypeScript/Vue dashboard. It consumes controlled DCGM and -future telemetry endpoints; frontend source is not duplicated in this chart. +The former standalone TypeScript/Vue dashboard has been migrated into the +chart-owned, Helm-provisioned Grafana dashboard +[`dashboards/edge-llm-observability.json`](dashboards/edge-llm-observability.json). +The Grafana version uses Prometheus, DCGM, and kube-state-metrics directly and +keeps the complete dashboard definition reproducible with the Helm release. ## Who This Is For @@ -100,15 +101,18 @@ future telemetry endpoints; frontend source is not duplicated in this chart. ## Platform Components - Vendored Helm charts for Ollama, Open WebUI, kube-prometheus-stack, OpenTelemetry Collector, and OpenTelemetry Operator. -- FastAPI OpenTelemetry GenAI-instrumented proxy with Prometheus metrics. +- Native Go OpenTelemetry GenAI-instrumented Ollama gateway with Prometheus metrics. - TTFT, latency, token, throughput, active-request, HTTP, and error telemetry. - Optional kube-prometheus-stack, Grafana, Alertmanager, node exporter, and kube-state-metrics from the root umbrella chart. - OpenTelemetry Collector endpoint for OTLP traces, metrics, and logs, with an optional operator-managed collector path. - Blackbox endpoint probes and Prometheus alert rules. - NVIDIA DCGM dashboard and external DCGM ServiceMonitor integration. +- A comprehensive Edge LLM dashboard for live GPU metrics, workload + readiness, service inventory, the validated Qwen profile, and telemetry + readiness. - NVIDIA NIM `/v1/metrics` ServiceMonitor path for environments that use NIM. - Pushgateway-compatible benchmark reporting. -- Optional Python diagnostics toolbox, Redis, OpenTelemetry seeder, and etcd failure simulation. +- Optional Go edge toolbox, Redis, OpenTelemetry seeder, and etcd failure simulation. ## Runtime Architecture @@ -116,7 +120,7 @@ future telemetry endpoints; frontend source is not duplicated in this chart. User or benchmark client | v -Open WebUI / FastAPI proxy +Open WebUI / Go Ollama gateway | \ | +--> OpenTelemetry GenAI traces | +--> Prometheus /metrics @@ -147,14 +151,13 @@ llm-observability-stack/ ├── values.cpu-k3s.yaml ├── values.local-k3s.example.yaml ├── artifacts/ # sanitized public benchmark evidence -├── benchmarks/ # repeatable inference benchmark clients -├── cmd/llm-observability/ # Go CLI entrypoint +├── cmd/ # Go CLI, gateway, and toolbox entrypoints ├── dashboards/ # LLM, benchmark, and NVIDIA GPU dashboards -├── internal/stack/ # CLI stack workflows +├── internal/ # CLI, gateway, toolbox, benchmark packages ├── templates/ # application monitoring and security manifests ├── charts/ # vendored dependency charts -├── langchain-demo/ # instrumented FastAPI proxy -├── python-toolbox/ # in-cluster diagnostics +├── ollama-gateway/ # native Go gateway image definition +├── edge-toolbox/ # native Go in-cluster diagnostics image ├── docs/ # architecture, operations, and local runbooks ├── hack/ # validation, device-plugin, and evidence scripts └── tests/ # Helm and application smoke tests @@ -189,7 +192,7 @@ bin/llm-observability validate - NVIDIA driver and NVIDIA Container Toolkit for GPU profiles. - `RuntimeClass/nvidia` and `nvidia.com/gpu` provided by `k3s-nvidia-edge` for GPU mode. - A legally obtained GGUF model available on node storage. -- Python 3.11 for tests and benchmark tooling. +- Go 1.25 or newer for CLI, gateway, toolbox, benchmark, and tests. Quick checks: @@ -270,7 +273,7 @@ helm upgrade --install llm-observability-stack . \ --set kube-prometheus-stack.crds.enabled=false ``` -Import the local `langchain-demo` and `python-toolbox` images into k3s containerd before enabling those two workloads. +Import the local `ollama-gateway` and `edge-toolbox` images into k3s containerd before enabling those two workloads. For a guided local setup, use: @@ -302,14 +305,20 @@ Do not switch an existing release from `values.enterprise-pilot-k3s.yaml` to a p ```bash kubectl get pods -n llm-observability -o wide kubectl port-forward -n llm-observability svc/ollama 11434:11434 +kubectl port-forward -n llm-observability \ + svc/llm-observability-stack-grafana 3000:80 ``` -Run the public benchmark from another terminal: +For the GeForce 940M profile, open and select +**Edge LLM Observability - Ubuntu + k3s + NVIDIA GPU**. See +[`dashboards/README.md`](dashboards/README.md) for provisioning, credentials, +and dashboard-as-code guidance. + +Run the native Go benchmark (it manages a temporary port-forward): ```bash -./benchmarks/ollama_benchmark.py \ +bin/llm-observability benchmark \ --model qwen-1-8b-chat-q4-k-m-local \ - --warmup-runs 1 \ --runs 10 \ --output artifacts/benchmark-local.json ``` @@ -330,7 +339,8 @@ helm template llm-observability-stack . \ --set open-webui.webuiSecret.existingSecretName= \ >/tmp/rendered-full-stack-nvidia.yaml -pytest -q tests +go test ./... +go vet ./... ./hack/validate-local-stack.sh ./hack/validate-local-stack.sh --strict-gpu ``` diff --git a/benchmarks/ollama_benchmark.py b/benchmarks/ollama_benchmark.py deleted file mode 100755 index bab28ce..0000000 --- a/benchmarks/ollama_benchmark.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3.11 -"""Run repeatable Ollama streaming benchmarks and optionally publish summary metrics.""" - -from __future__ import annotations - -import argparse -import json -import statistics -import time -import urllib.error -import urllib.request -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - - -def percentile(values: list[float], quantile: float) -> float: - ordered = sorted(values) - if not ordered: - return 0.0 - index = min(len(ordered) - 1, max(0, round((len(ordered) - 1) * quantile))) - return ordered[index] - - -def run_once(url: str, model: str, prompt: str, timeout: float) -> dict[str, Any]: - payload = json.dumps( - {"model": model, "prompt": prompt, "stream": True, "options": {"temperature": 0}} - ).encode() - request = urllib.request.Request( - url, data=payload, headers={"Content-Type": "application/json"}, method="POST" - ) - started = time.perf_counter() - first_token_at: float | None = None - final: dict[str, Any] = {} - with urllib.request.urlopen(request, timeout=timeout) as response: - for raw_line in response: - if not raw_line.strip(): - continue - chunk = json.loads(raw_line) - if first_token_at is None and chunk.get("response"): - first_token_at = time.perf_counter() - if chunk.get("done") is True: - final = chunk - finished = time.perf_counter() - generated_tokens = int(final.get("eval_count", 0)) - eval_duration = float(final.get("eval_duration", 0)) / 1_000_000_000 - return { - "success": True, - "duration_seconds": finished - started, - "ttft_seconds": (first_token_at or finished) - started, - "prompt_tokens": int(final.get("prompt_eval_count", 0)), - "generated_tokens": generated_tokens, - "tokens_per_second": generated_tokens / eval_duration if eval_duration > 0 else 0.0, - } - - -def push_metrics(pushgateway: str, job: str, model: str, summary: dict[str, Any]) -> None: - labels = f'model="{model}"' - lines = [ - "# TYPE llm_benchmark_success gauge", - f"llm_benchmark_success{{{labels}}} 1", - "# TYPE llm_benchmark_ttft_seconds gauge", - f"llm_benchmark_ttft_seconds{{{labels},quantile=\"0.50\"}} {summary['ttft_p50_seconds']}", - f"llm_benchmark_ttft_seconds{{{labels},quantile=\"0.95\"}} {summary['ttft_p95_seconds']}", - "# TYPE llm_benchmark_tokens_per_second gauge", - f"llm_benchmark_tokens_per_second{{{labels}}} {summary['tokens_per_second_mean']}", - "# TYPE llm_benchmark_duration_seconds gauge", - f"llm_benchmark_duration_seconds{{{labels},quantile=\"0.95\"}} {summary['duration_p95_seconds']}", - "", - ] - endpoint = f"{pushgateway.rstrip('/')}/metrics/job/{job}" - request = urllib.request.Request( - endpoint, - data="\n".join(lines).encode(), - headers={"Content-Type": "text/plain; version=0.0.4"}, - method="PUT", - ) - with urllib.request.urlopen(request, timeout=15): - pass - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--url", default="http://localhost:11434/api/generate") - parser.add_argument("--model", required=True) - parser.add_argument("--prompt", default="Explain GPU observability in three concise sentences.") - parser.add_argument("--runs", type=int, default=10) - parser.add_argument("--warmup-runs", type=int, default=2) - parser.add_argument("--timeout", type=float, default=300) - parser.add_argument("--pushgateway", default="") - parser.add_argument("--push-job", default="llm_local_benchmark") - parser.add_argument("--output", type=Path, default=Path("artifacts/benchmark.json")) - args = parser.parse_args() - - for _ in range(args.warmup_runs): - run_once(args.url, args.model, args.prompt, args.timeout) - results = [run_once(args.url, args.model, args.prompt, args.timeout) for _ in range(args.runs)] - ttft = [float(item["ttft_seconds"]) for item in results] - durations = [float(item["duration_seconds"]) for item in results] - throughput = [float(item["tokens_per_second"]) for item in results] - summary = { - "schema_version": 1, - "captured_at": datetime.now(timezone.utc).isoformat(), - "model": args.model, - "url": args.url, - "runs": args.runs, - "warmup_runs": args.warmup_runs, - "ttft_p50_seconds": percentile(ttft, 0.50), - "ttft_p95_seconds": percentile(ttft, 0.95), - "duration_p50_seconds": percentile(durations, 0.50), - "duration_p95_seconds": percentile(durations, 0.95), - "tokens_per_second_mean": statistics.fmean(throughput), - "results": results, - } - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") - if args.pushgateway: - push_metrics(args.pushgateway, args.push_job, args.model, summary) - print(json.dumps(summary, indent=2)) - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except urllib.error.URLError as exc: - raise SystemExit(f"benchmark request failed: {exc}") from exc diff --git a/cmd/edge-toolbox/main.go b/cmd/edge-toolbox/main.go new file mode 100644 index 0000000..0c98903 --- /dev/null +++ b/cmd/edge-toolbox/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/Edge-Computing-LLM/llm-observability-stack/internal/toolbox" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := toolbox.Run(ctx, os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} diff --git a/cmd/llm-observability/main.go b/cmd/llm-observability/main.go index 99622eb..d3f36ae 100644 --- a/cmd/llm-observability/main.go +++ b/cmd/llm-observability/main.go @@ -21,7 +21,10 @@ Commands: install Install or upgrade the llm-observability-stack Helm release status Print base and LLM stack status validate Run deeper release, pod, service, and optional Ollama checks - benchmark Run the existing Python Ollama benchmark client + benchmark Run the native Go Ollama benchmark client + network Print a namespace networking inventory through kubectl + service-path Trace one Service to its selected Pods and endpoints + watch-endpoints Watch endpoint changes for one Service uninstall Uninstall the LLM stack, optionally including the base layer print-commands Print Helm/kubectl commands used by install/validate/uninstall @@ -71,9 +74,11 @@ func main() { fs.BoolVar(&opts.KeepNamespace, "keep-namespace", false, "with uninstall: keep namespace") fs.StringVar(&opts.Model, "model", opts.Model, "with benchmark/validate: Ollama model") fs.IntVar(&opts.Runs, "runs", opts.Runs, "with benchmark: benchmark runs") + fs.IntVar(&opts.WarmupRuns, "warmup-runs", opts.WarmupRuns, "with benchmark: warmup runs") fs.StringVar(&opts.Prompt, "prompt", opts.Prompt, "with benchmark: benchmark prompt") fs.StringVar(&opts.Output, "output", opts.Output, "with benchmark: output JSON path") fs.BoolVar(&opts.OllamaSmoke, "ollama-smoke", opts.OllamaSmoke, "with validate: run Ollama smoke test") + fs.StringVar(&opts.Service, "service", opts.Service, "with service-path/watch-endpoints: Kubernetes Service") if err := fs.Parse(os.Args[2:]); err != nil { fmt.Fprintln(os.Stderr, err) @@ -107,6 +112,12 @@ func main() { runErr = stack.Validate(ctx, opts) case "benchmark": runErr = stack.Benchmark(ctx, opts) + case "network": + runErr = stack.NetworkInventory(ctx, opts) + case "service-path": + runErr = stack.ServicePath(ctx, opts) + case "watch-endpoints": + runErr = stack.WatchEndpoints(ctx, opts) case "uninstall": runErr = stack.Uninstall(ctx, opts) case "print-commands": diff --git a/cmd/ollama-gateway/main.go b/cmd/ollama-gateway/main.go new file mode 100644 index 0000000..efc3537 --- /dev/null +++ b/cmd/ollama-gateway/main.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/Edge-Computing-LLM/llm-observability-stack/internal/gateway" +) + +func main() { + config := gateway.ConfigFromEnv() + gateway.LogConfig(config) + server, err := gateway.New(config) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := server.Run(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/dashboards/README.md b/dashboards/README.md new file mode 100644 index 0000000..3a45f52 --- /dev/null +++ b/dashboards/README.md @@ -0,0 +1,54 @@ +# Grafana dashboards + +The JSON files in this directory are the source of truth for the Grafana +dashboards installed by this Helm chart. `templates/grafana-dashboards.yaml` +packages every `dashboards/*.json` file into a labelled ConfigMap, and the +Grafana sidecar from `kube-prometheus-stack` provisions them automatically. + +## Edge LLM dashboard migration + +`edge-llm-observability.json` recreates the information formerly rendered by +the standalone `Frontend-Edge-LLM-Observability` Vue application: + +- live NVIDIA DCGM utilization, memory, temperature, clock, PCIe, model, + driver, host, PCI bus, and UUID information; +- workload readiness for the `gpu-operator` and `llm-observability` + namespaces, backed by kube-state-metrics; +- the cluster-internal service inventory and ports; +- the validated Qwen 1.8B Chat Q4_K_M runtime profile; +- DCGM, Ollama/Qwen, Open WebUI, and OpenTelemetry readiness views; +- clear provenance notes that distinguish live Prometheus data from the + chart-owned reference profile. + +The migration was verified against the final frontend source commit +`13a4fc8e18d91d7602ed04aee883e1b4b8515a20` before the standalone repository +was retired. + +The GeForce 940M profile enables a compact Prometheus, Grafana, +kube-state-metrics, node-exporter, and Prometheus Operator deployment. It +disables Alertmanager, default dashboards, default rules, and k3s-incompatible +control-plane scrapes to fit the single-node development machine. + +## Reproduce locally + +```bash +helm dependency build . +helm upgrade --install llm-observability-stack . \ + --namespace llm-observability \ + --create-namespace \ + -f values.geforce-940m-k3s.yaml \ + --wait --timeout 15m + +kubectl -n llm-observability port-forward \ + svc/llm-observability-stack-grafana 3000:80 +``` + +Open , sign in with the local profile credentials, and +select **Edge LLM Observability - Ubuntu + k3s + NVIDIA GPU**. The bundled +credentials are only appropriate while Grafana remains a local ClusterIP +accessed through `kubectl port-forward`; replace them before any ingress or +shared access is enabled. + +Do not edit a provisioned dashboard only in the Grafana UI. Export reviewed +changes as JSON into this directory so the Helm installation remains +reproducible. diff --git a/dashboards/edge-llm-observability.json b/dashboards/edge-llm-observability.json new file mode 100644 index 0000000..1987c6b --- /dev/null +++ b/dashboards/edge-llm-observability.json @@ -0,0 +1,332 @@ +{ + "annotations": {"list": []}, + "description": "Helm-provisioned replacement for the standalone Edge LLM Observability frontend. Live panels use Prometheus, DCGM exporter, and kube-state-metrics; text panels preserve the validated local Qwen profile and service roles.", + "editable": false, + "graphTooltip": 1, + "panels": [ + { + "type": "text", + "title": "Edge LLM Observability", + "id": 1, + "gridPos": {"h": 5, "w": 24, "x": 0, "y": 0}, + "options": { + "mode": "markdown", + "content": "# Ubuntu 24 + k3s + NVIDIA GPU\n\nLocal observability for the running k3s NVIDIA stack, live DCGM metrics, and the resident **Qwen 1.8B Chat Q4_K_M** GGUF inference profile. Green readiness panels are derived from current Prometheus samples; runtime-profile and service-inventory panels are chart-owned reference information." + } + }, + { + "type": "stat", + "title": "GPU utilization", + "description": "Current average GPU utilization reported by NVIDIA DCGM.", + "id": 2, + "gridPos": {"h": 6, "w": 6, "x": 0, "y": 5}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 75}, {"color": "red", "value": 95}]} + }, + "overrides": [] + }, + "targets": [{"expr": "avg(DCGM_FI_DEV_GPU_UTIL)", "refId": "A"}] + }, + { + "type": "stat", + "title": "Framebuffer used", + "description": "Used framebuffer as a percentage of used plus free memory.", + "id": 3, + "gridPos": {"h": 6, "w": 6, "x": 6, "y": 5}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 85}, {"color": "red", "value": 95}]} + }, + "overrides": [] + }, + "targets": [{"expr": "100 * sum(DCGM_FI_DEV_FB_USED) / clamp_min(sum(DCGM_FI_DEV_FB_USED) + sum(DCGM_FI_DEV_FB_FREE), 1)", "refId": "A"}] + }, + { + "type": "stat", + "title": "GPU temperature", + "description": "Maximum current GPU temperature.", + "id": 4, + "gridPos": {"h": 6, "w": 6, "x": 12, "y": 5}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 0, + "max": 100, + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 75}, {"color": "red", "value": 85}]} + }, + "overrides": [] + }, + "targets": [{"expr": "max(DCGM_FI_DEV_GPU_TEMP)", "refId": "A"}] + }, + { + "type": "stat", + "title": "Ready workload replicas", + "description": "Available deployments, ready StatefulSets, and ready DaemonSets in the GPU and LLM namespaces.", + "id": 5, + "gridPos": {"h": 6, "w": 6, "x": 18, "y": 5}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": {"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas_available{namespace=~\"gpu-operator|llm-observability\"}) + sum(kube_statefulset_status_replicas_ready{namespace=~\"gpu-operator|llm-observability\"}) + sum(kube_daemonset_status_number_ready{namespace=~\"gpu-operator|llm-observability\"})", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "GPU utilization and memory", + "description": "GPU utilization, memory-copy utilization, and framebuffer use from DCGM.", + "id": 6, + "gridPos": {"h": 10, "w": 12, "x": 0, "y": 11}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percent", "min": 0, "max": 100}, "overrides": []}, + "options": { + "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "multi", "sort": "desc"} + }, + "targets": [ + {"expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "GPU util - {{modelName}}", "refId": "A"}, + {"expr": "DCGM_FI_DEV_MEM_COPY_UTIL", "legendFormat": "Memory copy - GPU {{gpu}}", "refId": "B"}, + {"expr": "100 * DCGM_FI_DEV_FB_USED / clamp_min(DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE, 1)", "legendFormat": "Framebuffer used - GPU {{gpu}}", "refId": "C"} + ] + }, + { + "type": "timeseries", + "title": "Temperature and clocks", + "description": "GPU and memory temperature plus SM and memory clocks.", + "id": 7, + "gridPos": {"h": 10, "w": 12, "x": 12, "y": 11}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": {"id": "byRegexp", "options": "/temperature/"}, + "properties": [{"id": "unit", "value": "celsius"}, {"id": "custom.axisPlacement", "value": "left"}] + }, + { + "matcher": {"id": "byRegexp", "options": "/clock/"}, + "properties": [{"id": "unit", "value": "MHz"}, {"id": "custom.axisPlacement", "value": "right"}] + } + ] + }, + "options": { + "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "multi", "sort": "desc"} + }, + "targets": [ + {"expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "GPU temperature", "refId": "A"}, + {"expr": "DCGM_FI_DEV_MEMORY_TEMP", "legendFormat": "Memory temperature", "refId": "B"}, + {"expr": "DCGM_FI_DEV_SM_CLOCK", "legendFormat": "SM clock", "refId": "C"}, + {"expr": "DCGM_FI_DEV_MEM_CLOCK", "legendFormat": "Memory clock", "refId": "D"} + ] + }, + { + "type": "table", + "title": "GPU device metrics", + "description": "Current framebuffer, clock, PCIe, utilization, and temperature values.", + "id": 8, + "gridPos": {"h": 9, "w": 12, "x": 0, "y": 21}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {}, "overrides": []}, + "options": {"cellHeight": "sm", "showHeader": true}, + "targets": [ + {"expr": "sum(DCGM_FI_DEV_FB_USED)", "format": "table", "instant": true, "legendFormat": "Framebuffer used MiB", "refId": "A"}, + {"expr": "sum(DCGM_FI_DEV_FB_FREE)", "format": "table", "instant": true, "legendFormat": "Framebuffer free MiB", "refId": "B"}, + {"expr": "sum(DCGM_FI_DEV_FB_RESERVED)", "format": "table", "instant": true, "legendFormat": "Framebuffer reserved MiB", "refId": "C"}, + {"expr": "max(DCGM_FI_DEV_SM_CLOCK)", "format": "table", "instant": true, "legendFormat": "SM clock MHz", "refId": "D"}, + {"expr": "max(DCGM_FI_DEV_MEM_CLOCK)", "format": "table", "instant": true, "legendFormat": "Memory clock MHz", "refId": "E"}, + {"expr": "sum(DCGM_FI_DEV_PCIE_REPLAY_COUNTER)", "format": "table", "instant": true, "legendFormat": "PCIe replay counter", "refId": "F"} + ] + }, + { + "type": "table", + "title": "GPU device identity", + "description": "Live model, host, driver, PCI bus, UUID, and GPU index labels parsed from DCGM.", + "id": 9, + "gridPos": {"h": 9, "w": 12, "x": 12, "y": 21}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {}, "overrides": []}, + "options": {"cellHeight": "sm", "showHeader": true}, + "targets": [ + { + "expr": "max by (gpu, UUID, modelName, Hostname, DCGM_FI_DRIVER_VERSION, pci_bus_id) (DCGM_FI_DEV_GPU_UTIL)", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {"Time": true, "__name__": true}, + "renameByName": { + "gpu": "GPU", + "modelName": "Model", + "Hostname": "Node", + "DCGM_FI_DRIVER_VERSION": "Driver", + "pci_bus_id": "PCI bus", + "UUID": "UUID", + "Value": "Utilization %" + } + } + } + ] + }, + { + "type": "table", + "title": "Cluster workloads", + "description": "Live controller readiness for the GPU Operator and LLM namespaces.", + "id": 10, + "gridPos": {"h": 11, "w": 24, "x": 0, "y": 30}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": {"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, + "overrides": [] + }, + "options": {"cellHeight": "sm", "showHeader": true}, + "targets": [ + { + "expr": "kube_deployment_status_replicas_available{namespace=~\"gpu-operator|llm-observability\"}", + "format": "table", + "instant": true, + "legendFormat": "Deployment {{namespace}} / {{deployment}}", + "refId": "A" + }, + { + "expr": "kube_statefulset_status_replicas_ready{namespace=~\"gpu-operator|llm-observability\"}", + "format": "table", + "instant": true, + "legendFormat": "StatefulSet {{namespace}} / {{statefulset}}", + "refId": "B" + }, + { + "expr": "kube_daemonset_status_number_ready{namespace=~\"gpu-operator|llm-observability\"}", + "format": "table", + "instant": true, + "legendFormat": "DaemonSet {{namespace}} / {{daemonset}}", + "refId": "C" + } + ] + }, + { + "type": "text", + "title": "Cluster-internal service inventory", + "id": 11, + "gridPos": {"h": 11, "w": 12, "x": 0, "y": 41}, + "options": { + "mode": "markdown", + "content": "| Namespace | Service | Type | Ports | Purpose |\n|---|---|---|---|---|\n| gpu-operator | nvidia-dcgm-exporter | ClusterIP | 9400/TCP | GPU metrics scrape target |\n| llm-observability | ollama | ClusterIP | 11434/TCP | Qwen GGUF inference API |\n| llm-observability | open-webui | ClusterIP | 8080/TCP | Browser chat service |\n| llm-observability | open-webui-redis | ClusterIP | 6379/TCP | WebUI state and websocket support |\n| llm-observability | opentelemetry-collector | ClusterIP | 4317, 4318, 8888/TCP | OTLP and collector metrics |\n\nKubernetes remains the source of truth. This chart-owned table documents stable service contracts without publishing ClusterIPs." + } + }, + { + "type": "text", + "title": "Validated Qwen runtime profile", + "id": 12, + "gridPos": {"h": 11, "w": 12, "x": 12, "y": 41}, + "options": { + "mode": "markdown", + "content": "| Setting | Validated value |\n|---|---|\n| Model | qwen-1-8b-chat-q4-k-m-local |\n| Artifact | Qwen 1.8B Chat Q4_K_M GGUF |\n| CUDA layers | 23 / 25 |\n| Context | 256 tokens |\n| Batch | 1 |\n| Parallel requests | 1 |\n| Keep alive | Forever (OLLAMA_KEEP_ALIVE=-1) |\n| GPU memory ceiling | 850 MiB contract |\n\nThese are declarative values owned by the GeForce Helm profile. Live memory, temperature, utilization, and readiness appear in Prometheus-backed panels." + } + }, + { + "type": "stat", + "title": "DCGM exporter connected", + "description": "Prometheus can scrape the GPU Operator-managed DCGM exporter. One means connected.", + "id": 13, + "gridPos": {"h": 7, "w": 6, "x": 0, "y": 52}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": { + "mappings": [{"options": {"0": {"color": "red", "text": "Disconnected"}, "1": {"color": "green", "text": "Connected"}}, "type": "value"}], + "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]} + }, + "overrides": [] + }, + "targets": [{"expr": "max(up{namespace=\"gpu-operator\",service=\"nvidia-dcgm-exporter\"})", "refId": "A"}] + }, + { + "type": "stat", + "title": "Qwen / Ollama ready", + "description": "The Ollama deployment has an available replica.", + "id": 14, + "gridPos": {"h": 7, "w": 6, "x": 6, "y": 52}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": { + "mappings": [{"options": {"0": {"color": "red", "text": "Not ready"}, "1": {"color": "green", "text": "Resident / ready"}}, "type": "value"}], + "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]} + }, + "overrides": [] + }, + "targets": [{"expr": "clamp_max(max(kube_deployment_status_replicas_available{namespace=\"llm-observability\",deployment=\"ollama\"}), 1)", "refId": "A"}] + }, + { + "type": "stat", + "title": "Open WebUI ready", + "description": "The browser chat StatefulSet has a ready replica.", + "id": 15, + "gridPos": {"h": 7, "w": 6, "x": 12, "y": 52}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": { + "mappings": [{"options": {"0": {"color": "red", "text": "Not ready"}, "1": {"color": "green", "text": "Ready"}}, "type": "value"}], + "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]} + }, + "overrides": [] + }, + "targets": [{"expr": "clamp_max(max(kube_statefulset_status_replicas_ready{namespace=\"llm-observability\",statefulset=\"open-webui\"}), 1)", "refId": "A"}] + }, + { + "type": "stat", + "title": "OpenTelemetry ready", + "description": "The OTLP metrics, traces, and logs collector has an available replica.", + "id": 16, + "gridPos": {"h": 7, "w": 6, "x": 18, "y": 52}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": { + "defaults": { + "mappings": [{"options": {"0": {"color": "red", "text": "Not ready"}, "1": {"color": "green", "text": "Ready"}}, "type": "value"}], + "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]} + }, + "overrides": [] + }, + "targets": [{"expr": "clamp_max(max(kube_deployment_status_replicas_available{namespace=\"llm-observability\",deployment=\"opentelemetry-collector\"}), 1)", "refId": "A"}] + }, + { + "type": "text", + "title": "Runtime contract and data provenance", + "id": 17, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 59}, + "options": { + "mode": "markdown", + "content": "### LLM observability readiness\n\n- **DCGM exporter:** live GPU utilization, framebuffer, temperatures, PCIe counters, and clocks come from the GPU Operator service.\n- **Qwen runtime:** the GeForce profile requests 23 GPU layers, context 256, batch 1, one parallel request, and infinite keep-alive.\n- **Runtime contract:** qwen-gguf-observability remains the authoritative read-only validator for node, GPU, Helm, pod, model-parameter, residency, and 850 MiB ceiling checks. Grafana visualizes metrics but does not replace that validator.\n- **OpenTelemetry:** the collector accepts OTLP metrics, traces, and logs on the chart-owned service.\n\nPrometheus is authoritative for live numeric and Kubernetes readiness panels. Static text records stable Helm-owned configuration and contains no ClusterIPs, pod identifiers, credentials, or public tunnel hostnames." + } + } + ], + "refresh": "15s", + "schemaVersion": 41, + "tags": ["edge-llm", "ubuntu", "k3s", "nvidia", "dcgm", "qwen", "ollama", "opentelemetry", "helm-provisioned"], + "templating": {"list": []}, + "time": {"from": "now-1h", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "Edge LLM Observability - Ubuntu + k3s + NVIDIA GPU", + "uid": "edge-llm-observability", + "version": 1 +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 71faf52..b64777e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -17,7 +17,7 @@ Modelfile, model lifecycle, or telemetry backend. ## 1. Design Goals - Keep the stack understandable on a single local node -- Prefer reproducible local images over runtime `pip install` +- Prefer static Go binaries and reproducible local images - Keep Open WebUI easy to reach from the browser - Keep internal APIs private by default and expose them only when needed - Make observability and networking drills easy to demonstrate @@ -34,7 +34,8 @@ Responsibilities: - install and uninstall this Helm chart - report status for Ollama, Open WebUI, Redis, OpenTelemetry Collector, and services - validate model loading, Ollama API behavior, and CUDA/offload evidence -- wrap the existing Python benchmark client +- run the native Go streaming benchmark client +- inspect namespace networking, Service paths, and endpoint watches - print the Helm and kubectl commands used under the hood ### 2.1 Root umbrella chart @@ -45,7 +46,7 @@ The root chart owns: - values layering - custom templates that glue the subcharts together - observability dependencies including kube-prometheus-stack and OpenTelemetry Operator -- optional resources such as Redis, python-toolbox, and etcd simulations +- optional resources such as Redis, edge-toolbox, and etcd simulations Files: @@ -73,9 +74,9 @@ Responsibilities: - user/session state - Ollama-compatible request flow to the traced proxy path -### 2.4 LangChain demo / proxy +### 2.4 Native Go Ollama gateway -Source lives in `langchain-demo/`. +Source lives in `ollama-gateway/`. Responsibilities: @@ -83,16 +84,17 @@ Responsibilities: - simple `/invoke` demo endpoint - Ollama-compatible proxy path at `/ollama/*` - optional OpenTelemetry-traced proxy runs for Open WebUI traffic +- Prometheus request, latency, active-request, and TTFT metrics -### 2.5 Python toolbox +### 2.5 Native Go edge toolbox -Source lives in `python-toolbox/`. +Source lives in `edge-toolbox/`. Responsibilities: - in-cluster diagnostics - DNS and service connectivity checks -- optional OpenTelemetry helper scripts +- optional OpenTelemetry trace-seeding command - notebook support for cluster-side network probing ### 2.6 Redis @@ -123,10 +125,10 @@ Available integration points: Primary user path for the full proxy profile: 1. Browser -> `open-webui` Service -2. `open-webui` pod -> `langchain-demo` Service -3. `langchain-demo` pod -> `ollama` Service -4. `langchain-demo` -> OpenTelemetry API when tracing is enabled -5. `langchain-demo` -> OpenTelemetry Collector when OpenTelemetry is enabled +2. `open-webui` pod -> `ollama-gateway` Service +3. `ollama-gateway` pod -> `ollama` Service +4. `ollama-gateway` -> OpenTelemetry API when tracing is enabled +5. `ollama-gateway` -> OpenTelemetry Collector when OpenTelemetry is enabled Primary user path for `values.geforce-940m-k3s.yaml`: @@ -137,8 +139,8 @@ Primary user path for `values.geforce-940m-k3s.yaml`: Supporting path: -1. Notebook or operator -> `kubectl exec` or Kubernetes Python client -2. `python-toolbox` pod -> internal Services, DNS, OpenTelemetry API +1. Notebook or operator -> `kubectl exec` or the Go CLI +2. `edge-toolbox` pod -> internal Services, DNS, OpenTelemetry API ## 4. Exposure Strategy @@ -146,8 +148,8 @@ Default local pattern: - `open-webui`: externally reachable for browser use - `ollama`: `ClusterIP` -- `langchain-demo`: `ClusterIP` -- `python-toolbox`: no public Service, pod-only diagnostics +- `ollama-gateway`: `ClusterIP` +- `edge-toolbox`: no public Service, pod-only diagnostics This keeps the local demo usable while reducing unnecessary surface area. @@ -178,8 +180,9 @@ There are three main configuration layers: - `cmd/llm-observability/`: Go CLI entrypoint - `internal/stack/`: CLI workflows for this chart and app layer - `templates/`: root chart resources, optional OpenTelemetryCollector CR, and integration glue -- `langchain-demo/app.py`: FastAPI app and traced proxy logic -- `python-toolbox/examples/`: in-cluster helper scripts +- `cmd/ollama-gateway` + `internal/gateway`: traced Go gateway +- `cmd/edge-toolbox` + `internal/toolbox`: in-cluster Go diagnostics +- `internal/benchmark`: native streaming benchmark and evidence schema - `hack/`: local image build/import flow - `jupyter-notebooks/`: notebook-driven operational guides diff --git a/docs/CONFIG-PROFILES.md b/docs/CONFIG-PROFILES.md index fdb0cda..fc1c5cc 100644 --- a/docs/CONFIG-PROFILES.md +++ b/docs/CONFIG-PROFILES.md @@ -48,7 +48,7 @@ For local NVIDIA k3s deployments, all GPU profiles assume `k3s-nvidia-edge` has | Service exposure | `ClusterIP` | `ClusterIP` | `ClusterIP` | `ClusterIP` | `ClusterIP` | | OpenTelemetry tracing | enabled | enabled | enabled | collector enabled | enabled | | Open WebUI secret key | Empty string | placeholder | placeholder, chart-managed secret | chart-managed secret | placeholder, chart-managed secret | -| `pythonToolbox.enabled` | `true` | `true` | `true` | `false` | `true` | +| `edgeToolbox.enabled` | `true` | `true` | `true` | `false` | `true` | | `otelTraceSeeder.enabled` | `false` | `false` | `false` | `false` | `false` | | `etcd.enabled` | `false` | `false` | `false` | `false` | `false` | diff --git a/docs/GO-KUBERNETES-AUTOMATION.md b/docs/GO-KUBERNETES-AUTOMATION.md new file mode 100644 index 0000000..5df31c3 --- /dev/null +++ b/docs/GO-KUBERNETES-AUTOMATION.md @@ -0,0 +1,23 @@ +# Native Go Kubernetes automation + +The repository-local `llm-observability` binary replaces the former Python +Kubernetes-client scripts and has no kubeconfig library dependency. It delegates +authentication and API compatibility to the installed `kubectl` binary. + +```bash +go build -o bin/llm-observability ./cmd/llm-observability + +bin/llm-observability network --namespace llm-observability +bin/llm-observability service-path --namespace llm-observability --service ollama +bin/llm-observability watch-endpoints --namespace llm-observability \ + --service open-webui --timeout 10m +``` + +All three commands are read-only. `network` prints Pods, Services, Endpoints, +EndpointSlices, and NetworkPolicies. `service-path` resolves a Service selector +to Pods and both endpoint APIs. `watch-endpoints` follows endpoint changes for +the requested bounded timeout. + +For deployment and lifecycle operations, prefer the organization-level +`edge-cli`; it enforces Layer 1 before Layer 2 installation and reverse order +for removal. diff --git a/docs/GO-NATIVE-MIGRATION-2026-07-18.md b/docs/GO-NATIVE-MIGRATION-2026-07-18.md new file mode 100644 index 0000000..2a1d06c --- /dev/null +++ b/docs/GO-NATIVE-MIGRATION-2026-07-18.md @@ -0,0 +1,22 @@ +# Go-native migration — 2026-07-18 + +Chart `0.3.0` removes Python from the deployed and validated platform path. + +| Previous component | Go replacement | +|---|---| +| FastAPI/LangChain proxy | `cmd/ollama-gateway` and `internal/gateway` | +| Python diagnostic image | `cmd/edge-toolbox` and `internal/toolbox` | +| Python streaming benchmark | `internal/benchmark` through `llm-observability benchmark` | +| Python Kubernetes scripts | `network`, `service-path`, and `watch-endpoints` CLI commands | +| pytest Helm/application suite | Go tests in `tests` plus package-level tests | +| Python trace seeder | `edge-toolbox seed` CronJob command | + +Values keys changed from `langchainDemo` to `ollamaGateway` and from +`pythonToolbox` to `edgeToolbox`. The optional Kubernetes workload names are +now `ollama-gateway` and `edge-toolbox`. Existing releases using those optional +components must build/import the `0.2.0` Go-native local images before enabling +them. + +Jupyter notebooks remain optional Python learning assets and are excluded from +the Helm package. Vendored upstream chart maintenance scripts are not part of +the project runtime and remain unchanged. diff --git a/docs/K3S-NVIDIA-EDGE-DEPENDENCY.md b/docs/K3S-NVIDIA-EDGE-DEPENDENCY.md index 0482516..6d28515 100644 --- a/docs/K3S-NVIDIA-EDGE-DEPENDENCY.md +++ b/docs/K3S-NVIDIA-EDGE-DEPENDENCY.md @@ -40,7 +40,7 @@ The two projects intentionally own different layers. - Open WebUI Redis helper when enabled by the Open WebUI subchart - OpenTelemetry Collector and GenAI telemetry paths - optional Prometheus/Grafana dashboards and alerting resources -- optional FastAPI proxy, Python toolbox, benchmarks, notebooks, and diagnostics +- optional native Go Ollama gateway, Go edge toolbox, benchmarks, notebooks, and diagnostics Do not enable GPU Operator, standalone NVIDIA device plugin, or standalone DCGM exporter in `llm-observability-stack` when `k3s-nvidia-edge` is already deployed. That would duplicate the GPU substrate that the base layer already owns. @@ -113,8 +113,8 @@ It keeps disabled: - root-level `nvidia-device-plugin` - root-level `dcgm-exporter` - root-level Redis -- LangChain demo -- Python toolbox +- Ollama gateway +- Go edge toolbox - etcd simulation This keeps the GPU substrate in `k3s-nvidia-edge` and the LLM application layer in `llm-observability-stack`. diff --git a/docs/KUBECTL-COMMAND-REFERENCE.md b/docs/KUBECTL-COMMAND-REFERENCE.md index 2036e3b..b470bd2 100644 --- a/docs/KUBECTL-COMMAND-REFERENCE.md +++ b/docs/KUBECTL-COMMAND-REFERENCE.md @@ -10,7 +10,7 @@ NS=llm-observability Profile note: -- The current local example profile keeps `pythonToolbox.enabled=true` and `otelTraceSeeder.enabled=false`. +- The current local example profile keeps `edgeToolbox.enabled=true` and `otelTraceSeeder.enabled=false`. - See [CONFIG-PROFILES.md](CONFIG-PROFILES.md) before assuming a workload is enabled in a given environment. ## 1. Cluster and Context Safety @@ -38,7 +38,7 @@ Detect workload kind first: ```bash kubectl get deploy,statefulset -n $NS -kubectl get deploy,statefulset -n $NS -o name | grep -E 'ollama|open-webui|langchain-demo|python-toolbox' +kubectl get deploy,statefulset -n $NS -o name | grep -E 'ollama|open-webui|ollama-gateway|edge-toolbox' ``` ```bash @@ -46,9 +46,9 @@ kubectl get pods -n $NS -o wide kubectl get deploy,statefulset -n $NS kubectl rollout status deploy/ollama -n $NS kubectl rollout status statefulset/open-webui -n $NS -kubectl rollout status deploy/langchain-demo -n $NS +kubectl rollout status deploy/ollama-gateway -n $NS # optional -kubectl rollout status deploy/python-toolbox -n $NS +kubectl rollout status deploy/edge-toolbox -n $NS ``` ## 4. Service and Endpoint Checks @@ -59,7 +59,7 @@ kubectl get endpoints -n $NS kubectl get endpointslices -n $NS kubectl describe svc ollama -n $NS kubectl describe svc open-webui -n $NS -kubectl describe svc langchain-demo -n $NS +kubectl describe svc ollama-gateway -n $NS ``` ## 5. Pod Detail and Scheduling Checks @@ -87,9 +87,9 @@ kubectl exec -it -n $NS deploy/ollama -- env | grep -E 'NVIDIA|CUDA|OLLAMA' ```bash kubectl logs -n $NS deploy/ollama --tail=200 kubectl logs -n $NS statefulset/open-webui --tail=200 -kubectl logs -n $NS deploy/langchain-demo --tail=200 +kubectl logs -n $NS deploy/ollama-gateway --tail=200 # optional -kubectl logs -n $NS deploy/python-toolbox --tail=200 +kubectl logs -n $NS deploy/edge-toolbox --tail=200 kubectl logs -n $NS deploy/redis --tail=200 kubectl get events -n $NS --sort-by=.lastTimestamp | tail -n 50 ``` @@ -105,13 +105,13 @@ kubectl logs -n $NS -c --previous --tail=200 ```bash # optional -kubectl exec -it -n $NS deploy/python-toolbox -- bash -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/service_dns_check.py -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/ollama_smoke.py -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/redis_ping.py -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/otel_genai_inference_traces.py -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/otel_genai_inference_traces.py -kubectl exec -it -n $NS deploy/python-toolbox -- env OBS_CALL_COUNT_PER_CYCLE=4 OBS_INTERVAL_SECONDS=300 python /workspace/examples/otel_genai_trace_seed_every_5m.py +kubectl exec -it -n $NS deploy/edge-toolbox -- bash +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox dns ollama ollama-gateway open-webui open-webui-redis +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox ollama-smoke +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox redis-ping +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox seed --count 2 +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox seed --count 2 +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox seed --count 4 ``` CronJob checks (only when `otelTraceSeeder.enabled=true`): @@ -125,11 +125,11 @@ kubectl get pods -n $NS -l job-name Quick in-pod DNS/TCP checks: ```bash -kubectl exec -it -n $NS deploy/python-toolbox -- nslookup ollama -kubectl exec -it -n $NS deploy/python-toolbox -- nslookup open-webui -kubectl exec -it -n $NS deploy/python-toolbox -- nc -vz ollama 11434 -kubectl exec -it -n $NS deploy/python-toolbox -- nc -vz open-webui 8080 -kubectl exec -it -n $NS deploy/python-toolbox -- nc -vz langchain-demo 8000 +kubectl exec -it -n $NS deploy/edge-toolbox -- nslookup ollama +kubectl exec -it -n $NS deploy/edge-toolbox -- nslookup open-webui +kubectl exec -it -n $NS deploy/edge-toolbox -- nc -vz ollama 11434 +kubectl exec -it -n $NS deploy/edge-toolbox -- nc -vz open-webui 8080 +kubectl exec -it -n $NS deploy/edge-toolbox -- nc -vz ollama-gateway 8000 ``` ## 9. Port-forward and Local API Tests @@ -137,7 +137,7 @@ kubectl exec -it -n $NS deploy/python-toolbox -- nc -vz langchain-demo 8000 ```bash kubectl port-forward -n $NS svc/ollama 11434:11434 kubectl port-forward -n $NS svc/open-webui 8080:8080 -kubectl port-forward -n $NS svc/langchain-demo 8000:8000 +kubectl port-forward -n $NS svc/ollama-gateway 8000:8000 ``` Ollama API test: @@ -149,7 +149,7 @@ curl -s http://localhost:11434/api/chat \ -d '{"model":"qwen-1-8b-chat-q4-k-m-local","stream":false,"messages":[{"role":"user","content":"health check"}]}' | jq ``` -LangChain demo test: +Ollama gateway test: ```bash curl -s http://localhost:8000/healthz | jq @@ -163,9 +163,9 @@ curl -s http://localhost:8000/invoke \ ```bash kubectl rollout restart deploy/ollama -n $NS kubectl rollout restart statefulset/open-webui -n $NS -kubectl rollout restart deploy/langchain-demo -n $NS +kubectl rollout restart deploy/ollama-gateway -n $NS # optional -kubectl rollout restart deploy/python-toolbox -n $NS +kubectl rollout restart deploy/edge-toolbox -n $NS kubectl rollout status deploy/ollama -n $NS kubectl rollout status statefulset/open-webui -n $NS ``` @@ -174,7 +174,7 @@ Undo rollout (if deployment strategy/history allows): ```bash kubectl rollout undo deploy/ollama -n $NS -kubectl rollout undo deploy/langchain-demo -n $NS +kubectl rollout undo deploy/ollama-gateway -n $NS ``` ## 11. Config and Secret Inspection @@ -212,7 +212,7 @@ DNS and CoreDNS: ```bash kubectl get pods -n kube-system -l k8s-app=kube-dns kubectl logs -n kube-system -l k8s-app=kube-dns --tail=200 -kubectl exec -it -n $NS deploy/python-toolbox -- cat /etc/resolv.conf +kubectl exec -it -n $NS deploy/edge-toolbox -- cat /etc/resolv.conf ``` Endpoints and EndpointSlices: @@ -307,16 +307,16 @@ kubectl get endpoints -n $NS ollama -o yaml ```bash kubectl logs -n $NS statefulset/open-webui --tail=200 kubectl get svc,endpoints -n $NS open-webui ollama -kubectl exec -it -n $NS deploy/python-toolbox -- nc -vz ollama 11434 -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/ollama_smoke.py +kubectl exec -it -n $NS deploy/edge-toolbox -- nc -vz ollama 11434 +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox ollama-smoke ``` -### 19.3 LangChain demo failures +### 19.3 Ollama gateway failures ```bash -kubectl logs -n $NS deploy/langchain-demo --tail=200 -kubectl describe deploy -n $NS langchain-demo -kubectl port-forward -n $NS svc/langchain-demo 8000:8000 +kubectl logs -n $NS deploy/ollama-gateway --tail=200 +kubectl describe deploy -n $NS ollama-gateway +kubectl port-forward -n $NS svc/ollama-gateway 8000:8000 curl -s http://localhost:8000/healthz | jq curl -s http://localhost:8000/config | jq ``` diff --git a/docs/KUBERNETES-NETWORKING.md b/docs/KUBERNETES-NETWORKING.md index 06a6ef5..4c7f0fa 100644 --- a/docs/KUBERNETES-NETWORKING.md +++ b/docs/KUBERNETES-NETWORKING.md @@ -10,7 +10,7 @@ NS=llm-observability Profile note: -- The current local example profile keeps `pythonToolbox.enabled=true`. +- The current local example profile keeps `edgeToolbox.enabled=true`. - Verify the active profile in [CONFIG-PROFILES.md](CONFIG-PROFILES.md) and with `helm get values ... -a` before assuming toolbox availability. ## 1. Networking Topology @@ -19,14 +19,14 @@ Core in-cluster services: - `ollama` on port `11434` - `open-webui` on port `8080` -- `langchain-demo` on port `8000` +- `ollama-gateway` on port `8000` - `open-webui-redis` (default mode) or `redis` (custom mode) on `6379` ### 1.1 Traffic flows 1. Browser -> `open-webui` service -2. Open WebUI app container -> `langchain-demo` proxy (`OLLAMA_BASE_URLS=http://langchain-demo:8000/ollama`) -3. LangChain demo -> `ollama` service (`OLLAMA_BASE_URL`) +2. Open WebUI app container -> `ollama-gateway` proxy (`OLLAMA_BASE_URLS=http://ollama-gateway:8000/ollama`) +3. Ollama gateway -> `ollama` service (`OLLAMA_BASE_URL`) 4. Open WebUI websocket manager -> Redis URL (`REDIS_URL` / `WEBSOCKET_REDIS_URL`) ## 2. Service Discovery and DNS @@ -40,15 +40,15 @@ Pods resolve services by: Check DNS from toolbox pod: ```bash -kubectl exec -it -n $NS deploy/python-toolbox -- nslookup ollama -kubectl exec -it -n $NS deploy/python-toolbox -- nslookup open-webui -kubectl exec -it -n $NS deploy/python-toolbox -- nslookup langchain-demo +kubectl exec -it -n $NS deploy/edge-toolbox -- nslookup ollama +kubectl exec -it -n $NS deploy/edge-toolbox -- nslookup open-webui +kubectl exec -it -n $NS deploy/edge-toolbox -- nslookup ollama-gateway ``` Inspect resolver config in pod: ```bash -kubectl exec -it -n $NS deploy/python-toolbox -- cat /etc/resolv.conf +kubectl exec -it -n $NS deploy/edge-toolbox -- cat /etc/resolv.conf ``` Inspect CoreDNS health: @@ -98,7 +98,7 @@ kubectl get svc -n $NS ```bash kubectl port-forward -n $NS svc/open-webui 8080:8080 kubectl port-forward -n $NS svc/ollama 11434:11434 -kubectl port-forward -n $NS svc/langchain-demo 8000:8000 +kubectl port-forward -n $NS svc/ollama-gateway 8000:8000 ``` ### 4.3 LoadBalancer on local k3s @@ -133,8 +133,8 @@ kubectl logs -n $NS statefulset/open-webui --tail=200 | grep -Ei 'redis|websocke ### 5.2 Check Redis reachability from toolbox ```bash -kubectl exec -it -n $NS deploy/python-toolbox -- nc -vz redis 6379 -kubectl exec -it -n $NS deploy/python-toolbox -- nc -vz open-webui-redis 6379 +kubectl exec -it -n $NS deploy/edge-toolbox -- nc -vz redis 6379 +kubectl exec -it -n $NS deploy/edge-toolbox -- nc -vz open-webui-redis 6379 ``` ## 6. Pod-to-Service and Pod-to-Pod Debugging @@ -142,16 +142,16 @@ kubectl exec -it -n $NS deploy/python-toolbox -- nc -vz open-webui-redis 6379 ### 6.1 Toolbox DNS + TCP + HTTP checks ```bash -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/service_dns_check.py -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/ollama_smoke.py +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox dns ollama ollama-gateway open-webui open-webui-redis +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox ollama-smoke ``` ### 6.2 Direct health probes ```bash -kubectl exec -it -n $NS deploy/python-toolbox -- curl -sS http://ollama:11434/api/tags -kubectl exec -it -n $NS deploy/python-toolbox -- curl -sS http://langchain-demo:8000/healthz -kubectl exec -it -n $NS deploy/python-toolbox -- curl -sSI http://open-webui:8080/ +kubectl exec -it -n $NS deploy/edge-toolbox -- curl -sS http://ollama:11434/api/tags +kubectl exec -it -n $NS deploy/edge-toolbox -- curl -sS http://ollama-gateway:8000/healthz +kubectl exec -it -n $NS deploy/edge-toolbox -- curl -sSI http://open-webui:8080/ ``` ## 7. Endpoint Troubleshooting Playbook @@ -183,15 +183,15 @@ kubectl logs -n $NS --all-containers --tail=200 4. Check in-cluster connectivity from toolbox ```bash -kubectl exec -it -n $NS deploy/python-toolbox -- nslookup -kubectl exec -it -n $NS deploy/python-toolbox -- nc -vz +kubectl exec -it -n $NS deploy/edge-toolbox -- nslookup +kubectl exec -it -n $NS deploy/edge-toolbox -- nc -vz ``` 5. Check namespace consistency ```bash kubectl get all -n $NS -kubectl get all -A | grep -E 'ollama|open-webui|langchain-demo|python-toolbox|redis' +kubectl get all -A | grep -E 'ollama|open-webui|ollama-gateway|edge-toolbox|redis' ``` ## 8. Network Policies @@ -276,14 +276,14 @@ Run this after each deploy/upgrade: ```bash kubectl get svc,endpoints -n $NS kubectl get pods -n $NS -o wide -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/service_dns_check.py -kubectl exec -it -n $NS deploy/python-toolbox -- python /workspace/examples/ollama_smoke.py -kubectl port-forward -n $NS svc/langchain-demo 8000:8000 +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox dns ollama ollama-gateway open-webui open-webui-redis +kubectl exec -it -n $NS deploy/edge-toolbox -- edge-toolbox ollama-smoke +kubectl port-forward -n $NS svc/ollama-gateway 8000:8000 curl -s http://localhost:8000/healthz | jq ``` ## 14. Related Docs - `docs/KUBECTL-COMMAND-REFERENCE.md` -- `docs/PYTHON-KUBERNETES-AUTOMATION.md` +- `docs/GO-KUBERNETES-AUTOMATION.md` - `docs/PROJECT-DOCUMENTATION.md` diff --git a/docs/LANGUAGE-BOUNDARIES.md b/docs/LANGUAGE-BOUNDARIES.md index 67b85b1..a24cf70 100644 --- a/docs/LANGUAGE-BOUNDARIES.md +++ b/docs/LANGUAGE-BOUNDARIES.md @@ -1,50 +1,31 @@ -# Programming language and script boundaries +# Language boundaries -This repository deliberately uses several implementation languages because it -contains a Helm application, a compatibility CLI, an instrumented API, and -operator tooling. +The deployable and testable project runtime is Go-first. ## Go -Use Go in `cmd/llm-observability` and `internal/stack` for durable repo-level -CLI behavior, typed options, process execution, timeouts, errors, and lifecycle -operations. New cross-repository workflows belong in `edge-cli`. +Use Go for the `llm-observability` CLI, Ollama gateway, edge toolbox, +benchmarks, Kubernetes observation workflows, OpenTelemetry instrumentation, +and automated tests. Go binaries must remain non-interactive, support bounded +timeouts, avoid secret output, and keep mutating workflows behind `--yes`. -Do not create a second Go implementation of Helm templates, Python benchmark -math, or browser behavior. - -## Python 3.11 - -Use Python 3.11 for: - -- the instrumented FastAPI/LangChain service; -- benchmarks and structured result processing; -- Kubernetes API reports and notebook code; -- Helm/application tests; -- in-cluster diagnostics whose container image pins its own interpreter. +## Helm and YAML -Host commands use `python3.11` and install packages directly with -`python3.11 -m pip`. This local workflow does not create virtual environments. -Container commands may use `python` when that name is supplied by the pinned -container image. +Helm templates and values own declarative Kubernetes resources. Do not recreate +Helm rendering or release state in Go. Layer 1 NVIDIA resources remain owned by +`k3s-nvidia-edge`; this chart owns only the application/observability layer. ## Bash -Keep Bash in `hack/` for short, transparent composition of existing tools such -as Helm, kubectl, Docker/nerdctl, and k3s containerd import. Scripts must use -`set -euo pipefail`, quote variables, expose safe defaults, and delegate complex -logic to Go or Python 3.11. - -Do not add growing state machines, JSON parsers, model policy, or duplicated -install/uninstall logic to Bash. - -## Helm and YAML - -Helm templates and values own Kubernetes desired state. Runtime detection may -write a generated values overlay, but it must not modify tracked defaults. +Small Bash helpers may glue established tools such as Helm, kubectl, Docker, +nerdctl, and k3s containerd together. Business logic, parsing, benchmarking, +HTTP services, evidence schemas, and test assertions belong in Go. -## TypeScript +## Python -Browser functionality belongs in the separate -`Frontend-Edge-LLM-Observability` repository. This chart may integrate that -frontend later, but it should not vendor a second frontend source tree. +Python is not required by the deployed chart, host validation, benchmark, +gateway, toolbox, or runtime-contract observer. It remains only in optional +Jupyter notebooks and their exported learning exercises, because those assets +are explicitly notebook-oriented and disabled in production profiles. Vendored +upstream charts may retain their upstream maintenance scripts; this project +does not execute them. diff --git a/docs/LIVE-VALIDATION-GO-NATIVE-2026-07-18.md b/docs/LIVE-VALIDATION-GO-NATIVE-2026-07-18.md new file mode 100644 index 0000000..50dc2fe --- /dev/null +++ b/docs/LIVE-VALIDATION-GO-NATIVE-2026-07-18.md @@ -0,0 +1,52 @@ +# Go-native live validation — 2026-07-18 + +## Host and root-cause repair + +The Xubuntu 24.04.3 host changed from network address `10.165.80.186` to +`10.53.163.158` while k3s was running. k3s retained the former node address, +which caused pod-to-API timeouts, failed health probes, metrics-server failure, +and CrashLoopBackOff in NFD, GPU Operator, kube-state-metrics, and node exporter. +Restarting k3s allowed automatic interface detection and restored the node, +service network, remotedialer, and Metrics API. + +## Clean deployment + +The Layer 2 release was removed first, followed by Layer 1. Both were then +installed from the local repositories without deleting retained user/model +data: + +| Release | Namespace | Revision | Chart | Result | +|---|---|---:|---|---| +| `k3s-nvidia-edge` | `gpu-operator` | 1 | 0.1.0 | deployed | +| `llm-observability-stack` | `llm-observability` | 1 | 0.3.0 / app 1.2.0 | deployed | + +All current project pods became Ready. The application/observability pods had +zero restarts. GPU Operator and NFD master each restarted once during the +expected k3s/containerd restart triggered by NVIDIA toolkit reconciliation. + +## Verified runtime + +- k3s `v1.36.2+k3s1`, node Ready at `10.53.163.158`. +- NVIDIA GeForce 940M, driver `580.95.05`, `nvidia.com/gpu: 1` allocatable. +- `RuntimeClass/nvidia`, device plugin, validator, and DCGM exporter Ready. +- Qwen local alias registered and resident `Forever`. +- Ollama reports 27% CPU / 73% GPU and 23/25 CUDA layers. +- The Go Qwen observer passed 13/13 contract checks. +- Explicit Qwen smoke inference passed in 1.909 seconds. +- Native Go benchmark: TTFT 0.359 seconds, total duration 0.566 seconds, and + 14.54 generated tokens/second for the one-run post-deployment smoke sample. +- `kubectl top node` succeeded after Metrics API recovery. +- The Helm-provisioned `Edge LLM Observability - Ubuntu + k3s + NVIDIA GPU` + Grafana dashboard was present in the release ConfigMap. + +## Validation matrix + +- `go test ./...` and `go vet ./...` for all four first-party repositories. +- Go builds for `edge`, `k3s-nvidia-edge`, `llm-observability`, + `ollama-gateway`, `edge-toolbox`, and `qwen-observe`. +- `helm lint` for both project charts. +- Helm rendering for default, CPU, enterprise, full-NVIDIA, local, and GeForce + profiles. +- Static container-target builds for the Go gateway and toolbox. +- Native Go Helm/package tests, gateway proxy tests, toolbox tests, benchmark + tests, and Qwen contract tests. diff --git a/docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md b/docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md index 4438d1b..b058d23 100644 --- a/docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md +++ b/docs/LOCAL-K3S-NVIDIA-REPORT-2026-07-02.md @@ -117,7 +117,7 @@ All expected pods were Running: - `alertmanager-kube-prometheus-stack-alertmanager-0` `2/2` - `dcgm-exporter` `1/1` - `kube-prometheus-stack-operator` `1/1` -- `langchain-demo` `1/1` +- `ollama-gateway` `1/1` - `llm-observability-stack-grafana` `3/3` - `kube-state-metrics` `1/1` - `prometheus-node-exporter` `1/1` @@ -126,7 +126,7 @@ All expected pods were Running: - `open-webui-redis` `1/1` - `opentelemetry-collector` `1/1` - `prometheus-kube-prometheus-stack-prometheus-0` `2/2` -- `python-toolbox` `1/1` +- `edge-toolbox` `1/1` ## Test Results diff --git a/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md b/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md index 917c928..1edec6f 100644 --- a/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md +++ b/docs/LOCAL-K3S-NVIDIA-RUNBOOK.md @@ -56,27 +56,27 @@ The chart mounts that host directory read-only into Ollama at `/models/gguf`, an ## 5. Build and Import Local Images -The `langchain-demo` and `python-toolbox` images are local project images. Build and import them into k3s containerd: +The `ollama-gateway` and `edge-toolbox` images are local project images. Build and import them into k3s containerd: ```bash -./hack/build-local-image.sh langchain-demo 0.1.1 ./langchain-demo -./hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 +./hack/build-local-image.sh ollama-gateway 0.2.0 . ollama-gateway/Dockerfile +./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 -./hack/build-local-image.sh python-toolbox 0.2.0 ./python-toolbox -./hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 +./hack/build-local-image.sh edge-toolbox 0.2.0 . edge-toolbox/Dockerfile +./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 ``` Verify: ```bash -sudo k3s ctr images ls | grep -E 'langchain-demo|python-toolbox' +sudo k3s ctr images ls | grep -E 'ollama-gateway|edge-toolbox' ``` -The images that must be built locally before enabling `langchainDemo` and `pythonToolbox` are: +The images that must be built locally before enabling `ollamaGateway` and `edgeToolbox` are: ```text -langchain-demo:0.1.1 -python-toolbox:0.2.0 +ollama-gateway:0.2.0 +edge-toolbox:0.2.0 ``` The platform images rendered by the enterprise profile are: @@ -119,8 +119,8 @@ For a lighter first install without local app images: ```bash ./hack/bootstrap-enterprise-pilot-k3s.sh \ - --set langchainDemo.enabled=false \ - --set pythonToolbox.enabled=false + --set ollamaGateway.enabled=false \ + --set edgeToolbox.enabled=false ``` After importing local images, enable the workloads with: @@ -138,7 +138,7 @@ helm upgrade --install llm-observability-stack . \ kubectl get pods,svc,pvc -n llm-observability -o wide kubectl rollout status deploy/ollama -n llm-observability --timeout=300s kubectl rollout status deploy/opentelemetry-collector -n llm-observability --timeout=180s -kubectl rollout status deploy/langchain-demo -n llm-observability --timeout=180s +kubectl rollout status deploy/ollama-gateway -n llm-observability --timeout=180s ``` ## 9. Verify Ollama and the Local Model @@ -191,7 +191,7 @@ kubectl port-forward -n llm-observability svc/opentelemetry-collector 4317:4317 curl -s http://127.0.0.1:8888/metrics | head ``` -`langchain-demo` sends OTLP to `http://opentelemetry-collector:4317` when OpenTelemetry is enabled. +`ollama-gateway` sends OTLP to `http://opentelemetry-collector:4317` when OpenTelemetry is enabled. ## 11. Access Open WebUI and Grafana @@ -217,7 +217,7 @@ password: admin ## 12. Run Tests ```bash -pytest -q tests +go test ./... ./hack/validate-local-stack.sh ./hack/validate-local-stack.sh --strict-gpu ``` diff --git a/docs/NOTEBOOKS-GUIDE.md b/docs/NOTEBOOKS-GUIDE.md index f7d8af3..f949607 100644 --- a/docs/NOTEBOOKS-GUIDE.md +++ b/docs/NOTEBOOKS-GUIDE.md @@ -12,9 +12,9 @@ Notebook classification is maintained in [../jupyter-notebooks/CATALOG.md](../ju 2. `02-ollama-api-basics.ipynb` - Exercises Ollama directly through its HTTP API. - Requires `kubectl port-forward -n llm-observability svc/ollama 11434:11434`. -3. `03-langchain-proxy-deep-dive.ipynb` - - Validates the local FastAPI proxy and compares direct-vs-proxy request behavior. - - Requires `kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000`. +3. `03-ollama-gateway-deep-dive.ipynb` + - Validates the local Go Ollama gateway and compares direct-vs-proxy request behavior. + - Requires `kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000`. 4. `04-opentelemetry-tracing-setup.ipynb` - Confirms OpenTelemetry credentials, pushes traced inference traffic, and queries recorded runs. - Requires the OpenTelemetry environment variables in the notebook kernel plus the same port-forwards used by `02` and `03`. @@ -24,9 +24,9 @@ Notebook classification is maintained in [../jupyter-notebooks/CATALOG.md](../ju 6. `06-custom-modelfile-workflow.ipynb` - Focuses on Modelfile customization, optional model creation, and comparison benchmarking. - Does not create custom models unless you explicitly enable the relevant cells. -7. `07-python-toolbox-diagnostics.ipynb` - - Uses the in-cluster `python-toolbox` deployment for diagnostics, DNS checks, and script execution. - - Assumes `pythonToolbox.enabled: true` in the active release or local values profile. +7. `07-edge-toolbox-diagnostics.ipynb` + - Uses the in-cluster `edge-toolbox` deployment for diagnostics, DNS checks, and script execution. + - Assumes `edgeToolbox.enabled: true` in the active release or local values profile. 8. `08-troubleshooting-etcd-simulations.ipynb` - Covers troubleshooting drills, rendered-manifest inspection, and operational failure simulation. - Best used after you understand the happy-path stack behavior from notebooks `01` through `07`. @@ -45,7 +45,7 @@ The repository also contains `llm-observability-stack-example-*.ipynb` notebooks - `kubectl` must point at the intended local k3s cluster. - The Helm release `llm-observability-stack` should be installed in the `llm-observability` namespace. -- `pythonToolbox.enabled` should remain `true` for the local profile. +- `edgeToolbox.enabled` should remain `true` for the local profile. - Python 3.11 and the `python311` kernelspec should be available. - The local machine should have enough RAM and GPU headroom for Ollama inference. - If Jupyter is launched from outside the repo, set `LLM_OBSERVABILITY_PROJECT_ROOT` to the chart root before running path-sensitive notebooks. @@ -64,7 +64,7 @@ Use separate terminals for these when working from the host: ```bash kubectl port-forward -n llm-observability svc/ollama 11434:11434 -kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 +kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000 kubectl port-forward -n llm-observability svc/open-webui 8080:8080 ``` @@ -76,7 +76,7 @@ Typical dependency map: - `05` needs browser access to Open WebUI plus OpenTelemetry environment variables if tracing is being validated - `06` usually needs Ollama on `localhost:11434` - `07` primarily uses the in-cluster toolbox, but some optional checks use `localhost:8000` -- `09` mostly uses the Kubernetes API directly and only shells into `python-toolbox` for in-cluster probes +- `09` mostly uses the Kubernetes API directly and only shells into `edge-toolbox` for in-cluster probes ## Execution Guidance @@ -90,7 +90,7 @@ Typical dependency map: - `Connection refused` to `localhost:11434` or `localhost:8000` - Missing port-forward or the backing pod is not ready. -- `No running python-toolbox pod found` +- `No running edge-toolbox pod found` - The release is disabled or the deployment needs a restart. - Empty OpenTelemetry result tables - Credentials are missing, tracing is disabled, or the project name does not match the release configuration. diff --git a/docs/OPERATIONS-RUNBOOK.md b/docs/OPERATIONS-RUNBOOK.md index 9f5e4f4..5897e02 100644 --- a/docs/OPERATIONS-RUNBOOK.md +++ b/docs/OPERATIONS-RUNBOOK.md @@ -15,22 +15,22 @@ git status --short ## 2. Build and Refresh Local Images -### Rebuild `langchain-demo` +### Rebuild `ollama-gateway` ```bash -./hack/build-local-image.sh langchain-demo 0.1.1 ./langchain-demo -./hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 -kubectl rollout restart deploy/langchain-demo -n llm-observability -kubectl rollout status deploy/langchain-demo -n llm-observability +./hack/build-local-image.sh ollama-gateway 0.2.0 . ollama-gateway/Dockerfile +./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 +kubectl rollout restart deploy/ollama-gateway -n llm-observability +kubectl rollout status deploy/ollama-gateway -n llm-observability ``` -### Rebuild `python-toolbox` +### Rebuild `edge-toolbox` ```bash -./hack/build-local-image.sh python-toolbox 0.2.0 ./python-toolbox -./hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 -kubectl rollout restart deploy/python-toolbox -n llm-observability -kubectl rollout status deploy/python-toolbox -n llm-observability +./hack/build-local-image.sh edge-toolbox 0.2.0 . edge-toolbox/Dockerfile +./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 +kubectl rollout restart deploy/edge-toolbox -n llm-observability +kubectl rollout status deploy/edge-toolbox -n llm-observability ``` ## 3. Install, Upgrade, and Roll Back @@ -79,23 +79,23 @@ kubectl top pods -n llm-observability ### Service-specific logs ```bash -kubectl logs -n llm-observability deploy/langchain-demo --tail=100 +kubectl logs -n llm-observability deploy/ollama-gateway --tail=100 kubectl logs -n llm-observability deploy/ollama --tail=100 kubectl logs -n llm-observability statefulset/open-webui --tail=100 -kubectl logs -n llm-observability deploy/python-toolbox --tail=100 +kubectl logs -n llm-observability deploy/edge-toolbox --tail=100 ``` ## 5. Access Internal APIs ```bash kubectl port-forward -n llm-observability svc/ollama 11434:11434 -kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 +kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000 ``` Use these for: - direct Ollama API tests -- LangChain proxy notebook cells +- Ollama gateway notebook cells - OpenTelemetry traced proxy requests from the host ## 6. Jupyter Notebook Operations @@ -111,7 +111,7 @@ cd jupyter-notebooks Useful pairings: - `01` before any major change -- `07` when validating `python-toolbox` +- `07` when validating `edge-toolbox` - `09` when validating cluster networking If notebook cells fail: @@ -123,18 +123,19 @@ If notebook cells fail: ## 7. In-Cluster Toolbox Checks -Open a shell: +Run a diagnostic command. The hardened distroless image intentionally has no +interactive shell: ```bash -kubectl exec -it -n llm-observability deploy/python-toolbox -- bash +kubectl exec -n llm-observability deploy/edge-toolbox -- edge-toolbox dns ollama ollama-gateway ``` Run helper scripts: ```bash -kubectl exec -it -n llm-observability deploy/python-toolbox -- python /workspace/examples/service_dns_check.py -kubectl exec -it -n llm-observability deploy/python-toolbox -- python /workspace/examples/ollama_smoke.py -kubectl exec -it -n llm-observability deploy/python-toolbox -- python /workspace/examples/redis_ping.py +kubectl exec -it -n llm-observability deploy/edge-toolbox -- edge-toolbox dns ollama ollama-gateway open-webui open-webui-redis +kubectl exec -it -n llm-observability deploy/edge-toolbox -- edge-toolbox ollama-smoke +kubectl exec -it -n llm-observability deploy/edge-toolbox -- edge-toolbox redis-ping ``` ## 8. GPU Checks @@ -151,9 +152,9 @@ If scheduling is failing, inspect the node plugin runtime behavior and confirm t ### `open-webui` works but notebooks fail -- likely missing `kubectl port-forward` for `ollama` and/or `langchain-demo` +- likely missing `kubectl port-forward` for `ollama` and/or `ollama-gateway` -### `langchain-demo` is unhealthy +### `ollama-gateway` is unhealthy - inspect logs - confirm the local image tag imported into k3s matches the chart values @@ -165,9 +166,9 @@ If scheduling is failing, inspect the node plugin runtime behavior and confirm t - verify Modelfile render values - verify the model exists through `/api/tags` -### `python-toolbox` is present but scripts are missing +### `edge-toolbox` is present but scripts are missing -- rebuild/import the local `python-toolbox` image +- rebuild/import the local `edge-toolbox` image - restart the toolbox deployment ## 10. Cleanup and Hygiene @@ -186,5 +187,5 @@ Before publishing: ```bash git status --short helm lint . -pytest -q tests +go test ./... ``` diff --git a/docs/PROJECT-ANALYSIS.md b/docs/PROJECT-ANALYSIS.md index 39ba835..51835b9 100644 --- a/docs/PROJECT-ANALYSIS.md +++ b/docs/PROJECT-ANALYSIS.md @@ -6,22 +6,22 @@ This repository is a local k3s-focused LLM platform chart: - Ollama for model serving - Open WebUI for chat interface -- LangChain demo API for integration and tracing checks -- Python toolbox for in-cluster diagnostics +- Ollama gateway API for integration and tracing checks +- Go edge toolbox for in-cluster diagnostics Current runtime preference in local overrides: - keep only Open WebUI externally exposed - keep Ollama/LangChain internal and access via port-forward as needed -- disable continuous Python inference jobs +- disable continuous inference seeding jobs ## What Is Stable Today - Umbrella chart and vendored dependencies are pinned and render cleanly. -- Local image workflow for `langchain-demo` and `python-toolbox` is documented and scriptable. +- Local image workflow for `ollama-gateway` and `edge-toolbox` is documented and scriptable. - GGUF Modelfile creation path is implemented and deployable. - CI validates chart linting and template rendering. -- OpenTelemetry tracing is wired through the `langchain-demo` proxy path (`/ollama/api/*`) for on-demand observability from Open WebUI traffic. +- OpenTelemetry tracing is wired through the `ollama-gateway` proxy path (`/ollama/api/*`) for on-demand observability from Open WebUI traffic. ## Main Risk Areas @@ -47,5 +47,5 @@ For full details, use: - [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) - [KUBECTL-COMMAND-REFERENCE.md](KUBECTL-COMMAND-REFERENCE.md) - [KUBERNETES-NETWORKING.md](KUBERNETES-NETWORKING.md) -- [PYTHON-KUBERNETES-AUTOMATION.md](PYTHON-KUBERNETES-AUTOMATION.md) +- [GO-KUBERNETES-AUTOMATION.md](GO-KUBERNETES-AUTOMATION.md) - [scripts/README.md](scripts/README.md) diff --git a/docs/PROJECT-DOCUMENTATION.md b/docs/PROJECT-DOCUMENTATION.md index c564d34..11e4f6d 100644 --- a/docs/PROJECT-DOCUMENTATION.md +++ b/docs/PROJECT-DOCUMENTATION.md @@ -20,8 +20,8 @@ Primary goals: - Run local LLM inference with Ollama using GGUF models. - Provide UI access through Open WebUI. -- Provide an API integration surface through a FastAPI + LangChain demo app. -- Provide observability and connectivity triage vian OpenTelemetry and an in-cluster Python toolbox. +- Provide an API integration surface through a native Go Ollama gateway. +- Provide observability and connectivity triage vian OpenTelemetry and an in-cluster Go edge toolbox. The current implementation is a local-ready, production-oriented reference architecture with a verified local edge profile. It is not yet customer-production-proven and requires workload-specific @@ -39,9 +39,9 @@ security, reliability, storage, and scale validation. - OpenTelemetry Secret (optional) - Open WebUI Secret (optional) - Ollama Modelfile ConfigMap - - LangChain demo app ConfigMap (optional mount-over-image mode) - - LangChain demo Deployment + Service - - Python toolbox Deployment (optional) + - Ollama gateway app ConfigMap (optional mount-over-image mode) + - Ollama gateway Deployment + Service + - Go edge toolbox Deployment (optional) - OpenTelemetry dashboard seeder CronJob (optional) - Optional Redis Deployment/Service/PVC/Secret - Optional etcd StatefulSet + Services @@ -49,10 +49,10 @@ security, reliability, storage, and scale validation. ### 2.2 Runtime traffic paths 1. User -> Open WebUI (`open-webui:8080`) -2. Open WebUI -> LangChain demo proxy (`langchain-demo:8000/ollama`) -3. LangChain demo proxy -> Ollama (`OLLAMA_UPSTREAM_BASE_URL=http://ollama:11434`) -4. LangChain demo -> OpenTelemetry API for proxy traces (when configured) -5. Optional Python toolbox -> OpenTelemetry API (when enabled) +2. Open WebUI -> Ollama gateway (`ollama-gateway:8000/ollama`) +3. Ollama gateway -> Ollama (`OLLAMA_UPSTREAM_BASE_URL=http://ollama:11434`) +4. Ollama gateway -> OpenTelemetry API for proxy traces (when configured) +5. Optional Go edge toolbox -> OpenTelemetry API (when enabled) 5. Open WebUI websocket manager -> Redis - Default: subchart `open-webui-redis` - Optional: custom `redis` resource from root templates @@ -70,8 +70,8 @@ llm-observability-stack/ ├── charts/ │ ├── ollama/ # vendored dependency │ └── open-webui/ # vendored dependency -├── langchain-demo/ # FastAPI app image source -├── python-toolbox/ # debug image source + scripts +├── ollama-gateway/ # native Go gateway image source +├── edge-toolbox/ # debug image source + scripts ├── hack/ # local image build/import helpers ├── files/ # pre/post change cluster snapshots └── docs/ # documentation @@ -108,9 +108,9 @@ Key value paths: - Subchart runtime config: `open-webui.*` - Secret input wrapper: `openWebUI.webuiSecretKey` and `openWebUI.existingSecret` -### 4.3 LangChain demo service +### 4.3 Ollama gateway service -- Containerized FastAPI app in `langchain-demo/app.py`. +- Containerized native Go gateway in `cmd/ollama-gateway` and `internal/gateway`. - Exposes: - `GET /` - `GET /healthz` @@ -119,7 +119,7 @@ Key value paths: - Uses `langchain-ollama` (`ChatOllama`) against Ollama API. - Can emit tracing to OpenTelemetry when API key is configured. -### 4.4 Python toolbox +### 4.4 Go edge toolbox - Utility pod for in-cluster diagnosis. - Ships troubleshooting scripts for: @@ -183,21 +183,21 @@ helm version ### 7.1 Build local images ```bash -./hack/build-local-image.sh langchain-demo 0.1.1 ./langchain-demo -./hack/build-local-image.sh python-toolbox 0.2.0 ./python-toolbox +./hack/build-local-image.sh ollama-gateway 0.2.0 . ollama-gateway/Dockerfile +./hack/build-local-image.sh edge-toolbox 0.2.0 . edge-toolbox/Dockerfile ``` ### 7.2 Import images into k3s containerd ```bash -./hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 -./hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 +./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 +./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 ``` ### 7.3 Verify images ```bash -sudo k3s ctr images ls | grep -E 'langchain-demo|python-toolbox' +sudo k3s ctr images ls | grep -E 'ollama-gateway|edge-toolbox' ``` ## 8. Deploy, Validate, and Upgrade @@ -255,7 +255,7 @@ Typical service endpoints in namespace: - Ollama: `http://ollama:11434` - Open WebUI: `http://open-webui:8080` -- LangChain demo: `http://langchain-demo:8000` +- Ollama gateway: `http://ollama-gateway:8000` Local access options: @@ -267,7 +267,7 @@ Example: ```bash kubectl port-forward -n llm-observability svc/open-webui 8080:8080 kubectl port-forward -n llm-observability svc/ollama 11434:11434 -kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 +kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000 ``` ## 10. Health and Observability Workflow @@ -276,15 +276,15 @@ kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 ```bash kubectl get pods -n llm-observability -kubectl logs -n llm-observability deploy/langchain-demo --tail=100 +kubectl logs -n llm-observability deploy/ollama-gateway --tail=100 kubectl logs -n llm-observability deploy/ollama --tail=100 kubectl logs -n llm-observability statefulset/open-webui --tail=100 ``` -### 10.2 LangChain demo API +### 10.2 Ollama gateway API ```bash -kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 +kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000 curl -s http://localhost:8000/healthz | jq curl -s http://localhost:8000/config | jq curl -s http://localhost:8000/invoke \ @@ -292,18 +292,17 @@ curl -s http://localhost:8000/invoke \ -d '{"prompt":"Say hello in one short sentence."}' | jq ``` -### 10.3 Python toolbox checks +### 10.3 Go edge toolbox checks -Only when `pythonToolbox.enabled=true`. +Only when `edgeToolbox.enabled=true`. ```bash -kubectl exec -it -n llm-observability deploy/python-toolbox -- bash -python /workspace/examples/service_dns_check.py -python /workspace/examples/ollama_smoke.py -python /workspace/examples/redis_ping.py -python /workspace/examples/otel_genai_inference_traces.py -python /workspace/examples/otel_genai_inference_traces.py -python /workspace/examples/otel_genai_trace_seed_every_5m.py +kubectl exec -it -n llm-observability deploy/edge-toolbox -- bash +edge-toolbox dns ollama ollama-gateway open-webui open-webui-redis +edge-toolbox http http://ollama:11434/api/tags +edge-toolbox redis-ping +edge-toolbox ollama-smoke +edge-toolbox seed --count 2 ``` ## 11. Security Guidance @@ -316,7 +315,7 @@ python /workspace/examples/otel_genai_trace_seed_every_5m.py ## 12. Common Failure Modes and Fixes -### 12.1 `ImagePullBackOff` on langchain-demo or python-toolbox +### 12.1 `ImagePullBackOff` on ollama-gateway or edge-toolbox Cause: @@ -325,13 +324,13 @@ Cause: Fix: ```bash -./hack/build-local-image.sh langchain-demo 0.1.1 ./langchain-demo -./hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 -./hack/build-local-image.sh python-toolbox 0.2.0 ./python-toolbox -./hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 -kubectl rollout restart deploy/langchain-demo -n llm-observability +./hack/build-local-image.sh ollama-gateway 0.2.0 . ollama-gateway/Dockerfile +./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 +./hack/build-local-image.sh edge-toolbox 0.2.0 . edge-toolbox/Dockerfile +./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 +kubectl rollout restart deploy/ollama-gateway -n llm-observability # only if enabled -kubectl rollout restart deploy/python-toolbox -n llm-observability +kubectl rollout restart deploy/edge-toolbox -n llm-observability ``` ### 12.2 Ollama model not available @@ -360,7 +359,7 @@ Fix: ```bash kubectl get svc -n llm-observability ollama open-webui kubectl get endpoints -n llm-observability ollama open-webui -kubectl exec -it -n llm-observability deploy/python-toolbox -- python /workspace/examples/service_dns_check.py +kubectl exec -it -n llm-observability deploy/edge-toolbox -- edge-toolbox dns ollama ollama-gateway open-webui open-webui-redis ``` ### 12.4 Websocket or chat instability with Redis @@ -403,5 +402,5 @@ kubectl describe pvc -n llm-observability - `docs/KUBECTL-COMMAND-REFERENCE.md` - `docs/KUBERNETES-NETWORKING.md` -- `docs/PYTHON-KUBERNETES-AUTOMATION.md` +- `docs/GO-KUBERNETES-AUTOMATION.md` - `docs/GITHUB-PUBLISHING.md` diff --git a/docs/PYTHON-KUBERNETES-AUTOMATION.md b/docs/PYTHON-KUBERNETES-AUTOMATION.md deleted file mode 100644 index 81b2ab4..0000000 --- a/docs/PYTHON-KUBERNETES-AUTOMATION.md +++ /dev/null @@ -1,217 +0,0 @@ -# Python Kubernetes Automation Guide (`pip install kubernetes`) - -This guide shows how to automate cluster/network checks for `llm-observability-stack` using the official Kubernetes Python client. - -Profile note: - -- These scripts can be run from host, CI, or a temporary debug pod. -- The current local example profile keeps `pythonToolbox.enabled=true`. -- Use [CONFIG-PROFILES.md](CONFIG-PROFILES.md) as the source of truth for generic defaults vs local-example behavior. - -## 1. Install and Environment - -```bash -/usr/local/bin/python3.11 -m pip install kubernetes -``` - -Or use project script requirements: - -```bash -/usr/local/bin/python3.11 -m pip install -r docs/scripts/requirements.txt -``` - -## 2. Authentication Modes - -### 2.1 Local kubeconfig (most common) - -```python -from kubernetes import config -config.load_kube_config() -``` - -### 2.2 Specific kubeconfig/context - -```python -from kubernetes import config -config.load_kube_config(config_file="/path/to/kubeconfig", context="my-k3s") -``` - -### 2.3 In-cluster auth - -```python -from kubernetes import config -config.load_incluster_config() -``` - -## 3. Ready-to-use Scripts in this Repository - -- `docs/scripts/network_inventory.py` -- `docs/scripts/service_path_inspector.py` -- `docs/scripts/watch_endpoints.py` - -Quick usage: - -```bash -/usr/local/bin/python3.11 docs/scripts/network_inventory.py --namespace llm-observability -/usr/local/bin/python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service ollama -/usr/local/bin/python3.11 docs/scripts/watch_endpoints.py --namespace llm-observability --service open-webui --timeout 600 -``` - -## 4. Practical Python Examples - -### 4.1 List pods and services - -```python -from kubernetes import client, config - -config.load_kube_config() -ns = "llm-observability" -core = client.CoreV1Api() - -pods = core.list_namespaced_pod(ns).items -services = core.list_namespaced_service(ns).items - -print("Pods:") -for p in pods: - print(f"- {p.metadata.name} phase={p.status.phase} ip={p.status.pod_ip}") - -print("Services:") -for s in services: - ports = ", ".join(str(port.port) for port in (s.spec.ports or [])) - print(f"- {s.metadata.name} type={s.spec.type} cluster_ip={s.spec.cluster_ip} ports={ports}") -``` - -### 4.2 Validate a service endpoint path - -```python -from kubernetes import client, config - -config.load_kube_config() -ns = "llm-observability" -service_name = "ollama" - -core = client.CoreV1Api() -svc = core.read_namespaced_service(service_name, ns) -selector = svc.spec.selector or {} -label_query = ",".join(f"{k}={v}" for k, v in selector.items()) - -pods = core.list_namespaced_pod(ns, label_selector=label_query).items if label_query else [] -ep = core.read_namespaced_endpoints(service_name, ns) - -print(f"service={service_name} selector={selector}") -print(f"selected_pods={[p.metadata.name for p in pods]}") - -addresses = [] -for subset in ep.subsets or []: - for addr in subset.addresses or []: - addresses.append(addr.ip) - -print(f"endpoint_addresses={addresses}") -``` - -### 4.3 Watch endpoint changes - -```python -from kubernetes import client, config, watch - -config.load_kube_config() -ns = "llm-observability" -service_name = "ollama" - -core = client.CoreV1Api() -w = watch.Watch() - -for event in w.stream( - core.list_namespaced_endpoints, - namespace=ns, - field_selector=f"metadata.name={service_name}", - timeout_seconds=300, -): - etype = event["type"] - obj = event["object"] - print(etype, obj.metadata.name, obj.subsets) -``` - -### 4.4 List EndpointSlices for detailed networking state - -```python -from kubernetes import client, config - -config.load_kube_config() -ns = "llm-observability" -service_name = "open-webui" - -discovery = client.DiscoveryV1Api() -items = discovery.list_namespaced_endpoint_slice( - namespace=ns, - label_selector=f"kubernetes.io/service-name={service_name}", -).items - -for eps in items: - addrs = [] - for endpoint in eps.endpoints or []: - addrs.extend(endpoint.addresses or []) - print(eps.metadata.name, addrs) -``` - -### 4.5 Check NetworkPolicies - -```python -from kubernetes import client, config - -config.load_kube_config() -ns = "llm-observability" -net = client.NetworkingV1Api() - -policies = net.list_namespaced_network_policy(ns).items -for np in policies: - types = np.spec.policy_types if np.spec else [] - print(np.metadata.name, types) -``` - -## 5. Usage Patterns for This Project - -### 5.1 Verify post-deploy networking quickly - -1. Run `network_inventory.py` to capture current state. -2. Run `service_path_inspector.py` for `ollama`, `open-webui`, `langchain-demo`. -3. Run `watch_endpoints.py` during rollout/restart events. - -### 5.2 Diagnose intermittent connection failures - -- Collect service + endpoints snapshots over time. -- Compare selected Pod readiness vs endpoint membership. -- Correlate endpoint changes with Kubernetes events: - -```bash -kubectl get events -n llm-observability --sort-by=.lastTimestamp -``` - -## 6. Error Handling Recommendations - -When using the Python client in production scripts: - -- Catch `kubernetes.client.ApiException` and print status/reason/body. -- Retry transient HTTP status (`429`, `500`, `503`) with exponential backoff. -- Set command timeout boundaries for watch loops. -- Emit machine-readable output (JSON) for CI pipelines. - -## 7. Security and Safety - -- Avoid printing decoded secret values in script output. -- Use least-privilege service accounts for in-cluster execution. -- Keep kubeconfig permissions tight (`chmod 600`). -- Avoid broad wildcard RBAC if script is namespace-scoped. - -## 8. Next-Level Automation Ideas - -- Scheduled network baseline snapshots to file. -- Drift detection: compare expected services/selectors to live cluster. -- Automatic alert when service endpoints become empty. -- Latency checks between internal services using exec-based probes. - -## 9. Related docs - -- `docs/KUBERNETES-NETWORKING.md` -- `docs/KUBECTL-COMMAND-REFERENCE.md` -- `docs/scripts/README.md` diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 7829373..47bea67 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -52,25 +52,25 @@ Profile reference: ## 3. Build and Import Local Images The GeForce 940M profile does not require local app images because Open WebUI connects directly to -Ollama. Build these images only when you intentionally enable `langchainDemo` or `pythonToolbox` in +Ollama. Build these images only when you intentionally enable `ollamaGateway` or `edgeToolbox` in another profile: ```bash -./hack/build-local-image.sh langchain-demo 0.1.1 ./langchain-demo -./hack/build-local-image.sh python-toolbox 0.2.0 ./python-toolbox +./hack/build-local-image.sh ollama-gateway 0.2.0 . ollama-gateway/Dockerfile +./hack/build-local-image.sh edge-toolbox 0.2.0 . edge-toolbox/Dockerfile ``` Import them into k3s: ```bash -./hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 -./hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 +./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 +./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 ``` Verify: ```bash -sudo k3s ctr images ls | grep -E 'langchain-demo|python-toolbox' +sudo k3s ctr images ls | grep -E 'ollama-gateway|edge-toolbox' ``` ## 4. Deploy the Chart @@ -144,7 +144,7 @@ kubectl logs -n llm-observability deploy/open-webui-redis --tail=100 If notebook API cells fail: - verify the required port-forwards are active -- verify `pythonToolbox.enabled` and toolbox pod health +- verify `edgeToolbox.enabled` and toolbox pod health - verify `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_SERVICE_NAME` for tracing notebooks ## 9. Next Reading diff --git a/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md b/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md index 5659ac6..2332853 100644 --- a/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md +++ b/docs/QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md @@ -64,7 +64,8 @@ resident and prevent a second model from competing for the GPU. - Gemma was stopped and removed from the Ollama registry; the original read-only host GGUF was not deleted. -- Helm revision 6 is `deployed`. +- The clean redeployment is Helm revision 1 with chart `0.3.0` and status + `deployed`. - Ollama was recreated with zero restarts and automatically created/warmed the local Qwen alias. - Ollama, Open WebUI, Redis, and OpenTelemetry Collector were Ready. @@ -85,7 +86,7 @@ Static validation also passed: - Helm dependency build and lint; - GeForce profile render with `num_gpu=23`, `num_batch=1`, and `num_ctx=256`; -- Python 3.11 Helm smoke tests; +- native Go Helm smoke tests; - edge-cli Go tests. The Qwen model was deliberately left loaded in GPU memory after validation. diff --git a/docs/README.md b/docs/README.md index d884ea6..9c9d1cf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,25 +7,28 @@ The documentation is organized around private LLM deployment and observability o ## Start Here 1. [QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md](QWEN-1.8B-LIVE-VALIDATION-2026-07-18.md) -2. [LIVE-VALIDATION-2026-07-17.md](LIVE-VALIDATION-2026-07-17.md) -3. [QUICKSTART.md](QUICKSTART.md) -4. [cli.md](cli.md) -5. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) -6. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) -7. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) -8. [ARCHITECTURE.md](ARCHITECTURE.md) -9. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) -10. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) -11. [LANGUAGE-BOUNDARIES.md](LANGUAGE-BOUNDARIES.md) +2. [LIVE-VALIDATION-GO-NATIVE-2026-07-18.md](LIVE-VALIDATION-GO-NATIVE-2026-07-18.md) +3. [LIVE-VALIDATION-2026-07-17.md](LIVE-VALIDATION-2026-07-17.md) +4. [QUICKSTART.md](QUICKSTART.md) +5. [cli.md](cli.md) +6. [K3S-NVIDIA-EDGE-DEPENDENCY.md](K3S-NVIDIA-EDGE-DEPENDENCY.md) +7. [CONFIG-PROFILES.md](CONFIG-PROFILES.md) +8. [XUBUNTU-K3S-NVIDIA-RUNBOOK.md](XUBUNTU-K3S-NVIDIA-RUNBOOK.md) +9. [ARCHITECTURE.md](ARCHITECTURE.md) +10. [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) +11. [PROJECT-DOCUMENTATION.md](PROJECT-DOCUMENTATION.md) +12. [LANGUAGE-BOUNDARIES.md](LANGUAGE-BOUNDARIES.md) +13. [GO-NATIVE-MIGRATION-2026-07-18.md](GO-NATIVE-MIGRATION-2026-07-18.md) External companion: - [`qwen-gguf-observability`](https://github.com/Edge-Computing-LLM/qwen-gguf-observability) provides read-only Qwen runtime contract checks and sanitized evidence. It does not replace this repository's Helm, Modelfile, or benchmark assets. -- [`Frontend-Edge-LLM-Observability`](https://github.com/Edge-Computing-LLM/Frontend-Edge-LLM-Observability) - owns the TypeScript/Vue presentation layer and does not receive direct - Kubernetes credentials from this chart. + +Dashboard presentation is now owned by the Helm-provisioned Grafana JSON in +[`../dashboards/`](../dashboards/); there is no separate browser application or +browser-side Kubernetes access path. ## Core Guides @@ -44,7 +47,7 @@ External companion: - [ARCHITECTURE.md](ARCHITECTURE.md) - Component ownership, request paths, service exposure, and configuration boundaries. - [LANGUAGE-BOUNDARIES.md](LANGUAGE-BOUNDARIES.md) - - Go, Python 3.11, Bash, Helm/YAML, and frontend ownership rules. + - Go-first runtime, optional notebook Python, Bash, and Helm/YAML ownership rules. - [OPERATIONS-RUNBOOK.md](OPERATIONS-RUNBOOK.md) - Day-0 and day-1 tasks: deploy, verify, port-forward, rebuild images, debug, and clean up. - [NOTEBOOKS-GUIDE.md](NOTEBOOKS-GUIDE.md) @@ -60,8 +63,8 @@ External companion: - Service, EndpointSlice, DNS, ServiceLB, and traffic-path documentation for this stack. - [KUBECTL-COMMAND-REFERENCE.md](KUBECTL-COMMAND-REFERENCE.md) - High-signal `kubectl` command catalog for local operations. -- [PYTHON-KUBERNETES-AUTOMATION.md](PYTHON-KUBERNETES-AUTOMATION.md) - - Kubernetes Python client usage patterns and script-driven inspection. +- [GO-KUBERNETES-AUTOMATION.md](GO-KUBERNETES-AUTOMATION.md) + - Native Go CLI networking inventory, service-path, and endpoint-watch commands. ## Local Validation @@ -93,7 +96,7 @@ External companion: - Contributor/operator: - [../CONTRIBUTING.md](../CONTRIBUTING.md) - [KUBECTL-COMMAND-REFERENCE.md](KUBECTL-COMMAND-REFERENCE.md) - - [PYTHON-KUBERNETES-AUTOMATION.md](PYTHON-KUBERNETES-AUTOMATION.md) + - [GO-KUBERNETES-AUTOMATION.md](GO-KUBERNETES-AUTOMATION.md) ## Supporting Script Docs @@ -102,8 +105,8 @@ External companion: ## Related Component Docs -- [../langchain-demo/README.md](../langchain-demo/README.md) -- [../python-toolbox/README.md](../python-toolbox/README.md) +- [../ollama-gateway/README.md](../ollama-gateway/README.md) +- [../edge-toolbox/README.md](../edge-toolbox/README.md) - [../hack/README.md](../hack/README.md) - [../jupyter-notebooks/README.md](../jupyter-notebooks/README.md) - [../jupyter-notebooks/llm-observability-in-action/README.md](../jupyter-notebooks/llm-observability-in-action/README.md) diff --git a/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md b/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md index fa4de0b..86b7913 100644 --- a/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md +++ b/docs/XUBUNTU-K3S-NVIDIA-RUNBOOK.md @@ -54,8 +54,8 @@ Expected core workloads in the current full local deployment: - `ollama` - `open-webui` - `open-webui-redis` -- `langchain-demo` -- `python-toolbox` +- `ollama-gateway` +- `edge-toolbox` - `opentelemetry-collector` - `dcgm-exporter` - `kube-prometheus-stack-operator` @@ -67,16 +67,16 @@ Expected core workloads in the current full local deployment: ## Build and Import Local Images -Use this when `langchain-demo` or `python-toolbox` source code changes. +Use this when `ollama-gateway` or `edge-toolbox` source code changes. ```bash -docker build -t langchain-demo:0.1.1 ./langchain-demo -docker build -t python-toolbox:0.2.0 ./python-toolbox +docker build -t ollama-gateway:0.2.0 ./ollama-gateway +docker build -t edge-toolbox:0.2.0 ./edge-toolbox -./hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 -./hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 +./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 +./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 -sudo k3s ctr images list | grep -E 'langchain-demo|python-toolbox' +sudo k3s ctr images list | grep -E 'ollama-gateway|edge-toolbox' ``` ## Minimal GPU/Ollama Profile @@ -107,7 +107,7 @@ kubectl exec -n llm-observability deploy/ollama -- ollama list ## Full Local k3s/NVIDIA Profile -This profile runs Ollama, Open WebUI, LangChain proxy, Python toolbox, +This profile runs Ollama, Open WebUI, Ollama gateway, Go edge toolbox, OpenTelemetry Collector, Prometheus, Grafana, Alertmanager, node exporter, and kube-state-metrics. It may observe the DCGM exporter from the base layer through ServiceMonitor resources, but it does not deploy DCGM exporter. @@ -160,7 +160,7 @@ Run each command in its own terminal when you need local access. ```bash kubectl port-forward -n llm-observability svc/ollama 11434:11434 -kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 +kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000 kubectl port-forward -n llm-observability svc/open-webui 8080:8080 kubectl port-forward -n llm-observability svc/llm-observability-stack-grafana 3000:80 kubectl port-forward -n llm-observability svc/opentelemetry-collector 4317:4317 4318:4318 8888:8888 @@ -169,7 +169,7 @@ kubectl port-forward -n llm-observability svc/opentelemetry-collector 4317:4317 Useful local URLs: - Ollama: `http://localhost:11434` -- LangChain proxy: `http://localhost:8000` +- Ollama gateway: `http://localhost:8000` - Open WebUI: `http://localhost:8080` - Grafana: `http://localhost:3000` - OpenTelemetry Collector metrics: `http://localhost:8888/metrics` @@ -190,7 +190,7 @@ curl -s http://127.0.0.1:11434/api/generate \ -d '{"model":"qwen-1-8b-chat-q4-k-m-local:latest","prompt":"Reply with one short sentence.","stream":false}' | jq ``` -LangChain proxy: +Ollama gateway: ```bash curl -s http://127.0.0.1:8000/healthz | jq @@ -218,8 +218,8 @@ helm template llm-observability-stack . \ --set open-webui.webuiSecret.existingSecretName= \ >/tmp/rendered-full-stack-nvidia.yaml -python3.11 -m pytest -q -PYTHON_BIN=python3.11 ./hack/validate-local-stack.sh --strict-gpu +go test ./... +./hack/validate-local-stack.sh --strict-gpu ``` ## Benchmark @@ -227,22 +227,21 @@ PYTHON_BIN=python3.11 ./hack/validate-local-stack.sh --strict-gpu Keep benchmark outputs under `artifacts/` and commit only sanitized evidence. ```bash -python3.11 benchmarks/ollama_benchmark.py \ - --url 'http://127.0.0.1:11434/api/generate' \ +bin/llm-observability benchmark \ --model qwen-1-8b-chat-q4-k-m-local:latest \ --warmup-runs 1 \ --runs 3 \ --output artifacts/local-benchmark.json ``` -## Python Toolbox Checks +## Go Edge Toolbox Checks Run these after the full local profile is running: ```bash -kubectl exec -it -n llm-observability deploy/python-toolbox -- python /workspace/examples/service_dns_check.py -kubectl exec -it -n llm-observability deploy/python-toolbox -- python /workspace/examples/ollama_smoke.py -kubectl exec -it -n llm-observability deploy/python-toolbox -- python /workspace/examples/redis_ping.py +kubectl exec -it -n llm-observability deploy/edge-toolbox -- edge-toolbox dns ollama ollama-gateway open-webui open-webui-redis +kubectl exec -it -n llm-observability deploy/edge-toolbox -- edge-toolbox ollama-smoke +kubectl exec -it -n llm-observability deploy/edge-toolbox -- edge-toolbox redis-ping ``` ## Capture Local Evidence diff --git a/docs/cli.md b/docs/cli.md index d80597b..6ebb567 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -6,7 +6,7 @@ primary organization-level CLI for end-to-end installs and uninstalls. ## Architecture Boundary - `k3s-nvidia-edge` owns the local Linux, k3s, k3s containerd, NVIDIA runtime, GPU Operator, NVIDIA device plugin, DCGM exporter, Node Feature Discovery, `RuntimeClass/nvidia`, and `nvidia.com/gpu` validation layer. -- `llm-observability-stack` owns Ollama, Open WebUI, Open WebUI Redis, OpenTelemetry Collector, optional Prometheus/Grafana, optional LangChain/FastAPI proxy, benchmark tooling, notebooks, and local model configuration. +- `llm-observability-stack` owns Ollama, Open WebUI, Open WebUI Redis, OpenTelemetry Collector, optional Prometheus/Grafana, the optional native Go Ollama gateway, benchmark tooling, notebooks, and local model configuration. - `edge-cli` owns cross-repository ordering: infra first, observability second. - `qwen-gguf-observability` optionally consumes the deployed state for read-only contract checks and evidence; it owns no install or uninstall workflow. @@ -20,13 +20,15 @@ From this repository: go build -o bin/llm-observability ./cmd/llm-observability ``` -During local sibling-repo development, `go.mod` uses: +`go.mod` pins the published `k3s-nvidia-edge` commit that introduced the +reusable `pkg/edgebase` package: ```text -replace github.com/Edge-Computing-LLM/k3s-nvidia-edge => ../k3s-nvidia-edge +github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013 ``` -That keeps the long-term import path clean while allowing local changes in `k3s-nvidia-edge/pkg/edgebase` to be tested immediately. When `k3s-nvidia-edge` publishes a version tag containing `pkg/edgebase`, this can be changed to a normal tagged requirement. +This lets a standalone clone build without relying on a sibling checkout. Move +to a normal semantic version after `k3s-nvidia-edge` publishes its next tag. ## Commands @@ -112,7 +114,7 @@ The base CUDA validation pod from `edgebase` is dry-run unless `--yes` is provid ## Benchmark -The benchmark command wraps the existing Python client: +The benchmark command uses the native Go streaming client: ```bash bin/llm-observability benchmark \ @@ -122,7 +124,8 @@ bin/llm-observability benchmark \ --output artifacts/benchmark-local.json ``` -It starts a temporary `kubectl port-forward` to `svc/ollama` and then runs `benchmarks/ollama_benchmark.py`. +It starts a temporary `kubectl port-forward` to `svc/ollama`, measures TTFT, +duration, tokens, and throughput, and writes versioned JSON evidence. ## Uninstall diff --git a/docs/scripts/README.md b/docs/scripts/README.md index 864def3..d9f6b1c 100644 --- a/docs/scripts/README.md +++ b/docs/scripts/README.md @@ -1,62 +1,11 @@ -# Python Kubernetes Scripts +# Replaced documentation scripts -These scripts use the official Python Kubernetes client (`pip install kubernetes`) and focus on networking diagnostics for `llm-observability-stack`. +The former Python 3.11 Kubernetes scripts were replaced by native Go commands: -They are designed to run from host or CI and do not require `python-toolbox` to be enabled in-cluster. +| Former script | Go command | +|---|---| +| `network_inventory.py` | `bin/llm-observability network` | +| `service_path_inspector.py` | `bin/llm-observability service-path --service NAME` | +| `watch_endpoints.py` | `bin/llm-observability watch-endpoints --service NAME` | -## Install - -```bash -/usr/local/bin/python3.11 -m pip install -r docs/scripts/requirements.txt -``` - -## Scripts - -### 1) `network_inventory.py` - -Purpose: - -- Print namespace networking inventory for Pods, Services, Endpoints, EndpointSlices, and NetworkPolicies. - -Usage: - -```bash -/usr/local/bin/python3.11 docs/scripts/network_inventory.py --namespace llm-observability -``` - -### 2) `service_path_inspector.py` - -Purpose: - -- Trace one Service to its selector, selected Pods, Endpoints, and EndpointSlices. - -Usage: - -```bash -/usr/local/bin/python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service ollama -/usr/local/bin/python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service open-webui -/usr/local/bin/python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service langchain-demo -``` - -### 3) `watch_endpoints.py` - -Purpose: - -- Watch endpoint changes for a specific Service in near real-time. - -Usage: - -```bash -/usr/local/bin/python3.11 docs/scripts/watch_endpoints.py --namespace llm-observability --service ollama --timeout 600 -``` - -## Config options - -All scripts support: - -- `--namespace` -- `--context` -- `--kubeconfig` -- `--in-cluster` - -Use `--in-cluster` only when running script inside Kubernetes with service account auth. +See [Native Go Kubernetes automation](../GO-KUBERNETES-AUTOMATION.md). diff --git a/docs/scripts/network_inventory.py b/docs/scripts/network_inventory.py deleted file mode 100755 index 46341b6..0000000 --- a/docs/scripts/network_inventory.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3.11 -"""Print a networking inventory for a Kubernetes namespace. - -Requires: - python3.11 -m pip install kubernetes - -Examples: - python3.11 docs/scripts/network_inventory.py --namespace llm-observability - python3.11 docs/scripts/network_inventory.py --namespace llm-observability --context my-k3s -""" - -from __future__ import annotations - -import argparse -import datetime as dt -import sys -from typing import Iterable, List - -from kubernetes import client, config -from kubernetes.client import ApiException - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Kubernetes namespace networking inventory") - parser.add_argument("--namespace", default="llm-observability", help="Namespace to inspect") - parser.add_argument("--context", default=None, help="Optional kubeconfig context") - parser.add_argument("--kubeconfig", default=None, help="Optional kubeconfig file path") - parser.add_argument("--in-cluster", action="store_true", help="Use in-cluster auth") - return parser.parse_args() - - -def load_kube_config(args: argparse.Namespace) -> None: - if args.in_cluster: - config.load_incluster_config() - return - config.load_kube_config(config_file=args.kubeconfig, context=args.context) - - -def fmt_ports(ports: Iterable) -> str: - parts: List[str] = [] - for item in ports or []: - port = getattr(item, "port", None) - target_port = getattr(item, "target_port", None) - protocol = getattr(item, "protocol", "TCP") - name = getattr(item, "name", "") - if target_port is None: - parts.append(f"{name}:{port}/{protocol}" if name else f"{port}/{protocol}") - else: - left = f"{name}:{port}" if name else str(port) - parts.append(f"{left}->{target_port}/{protocol}") - return ", ".join(parts) if parts else "-" - - -def fmt_selector(selector: dict | None) -> str: - if not selector: - return "-" - return ",".join(f"{k}={v}" for k, v in sorted(selector.items())) - - -def print_table(title: str, headers: List[str], rows: List[List[str]]) -> None: - print(f"\n=== {title} ===") - if not rows: - print("") - return - - widths = [len(h) for h in headers] - for row in rows: - for i, val in enumerate(row): - widths[i] = max(widths[i], len(str(val))) - - def fmt_row(values: List[str]) -> str: - return " | ".join(str(v).ljust(widths[i]) for i, v in enumerate(values)) - - print(fmt_row(headers)) - print("-+-".join("-" * w for w in widths)) - for row in rows: - print(fmt_row(row)) - - -def collect(namespace: str) -> int: - core = client.CoreV1Api() - net = client.NetworkingV1Api() - discovery = client.DiscoveryV1Api() - - try: - pods = core.list_namespaced_pod(namespace=namespace).items - services = core.list_namespaced_service(namespace=namespace).items - endpoints = core.list_namespaced_endpoints(namespace=namespace).items - endpoint_slices = discovery.list_namespaced_endpoint_slice(namespace=namespace).items - network_policies = net.list_namespaced_network_policy(namespace=namespace).items - except ApiException as exc: - print(f"API error: {exc.status} {exc.reason}", file=sys.stderr) - print(exc.body or "", file=sys.stderr) - return 2 - - pod_rows: List[List[str]] = [] - for pod in pods: - pod_rows.append( - [ - pod.metadata.name, - pod.status.phase or "", - pod.status.pod_ip or "", - pod.spec.node_name or "", - pod.status.host_ip or "", - ] - ) - - svc_rows: List[List[str]] = [] - for svc in services: - svc_rows.append( - [ - svc.metadata.name, - svc.spec.type or "", - svc.spec.cluster_ip or "", - fmt_ports(svc.spec.ports), - fmt_selector(svc.spec.selector), - ] - ) - - ep_rows: List[List[str]] = [] - for ep in endpoints: - subset_addrs: List[str] = [] - subset_ports: List[str] = [] - for subset in ep.subsets or []: - for addr in subset.addresses or []: - subset_addrs.append(addr.ip) - for port in subset.ports or []: - subset_ports.append(f"{port.name or ''}:{port.port}/{port.protocol}") - ep_rows.append( - [ - ep.metadata.name, - ",".join(subset_addrs) if subset_addrs else "-", - ",".join(subset_ports) if subset_ports else "-", - ] - ) - - eps_rows: List[List[str]] = [] - for eps in endpoint_slices: - addresses: List[str] = [] - for endpoint in eps.endpoints or []: - addresses.extend(endpoint.addresses or []) - ports = [f"{p.name or ''}:{p.port}/{p.protocol}" for p in eps.ports or []] - eps_rows.append( - [ - eps.metadata.name, - eps.metadata.labels.get("kubernetes.io/service-name", "-") if eps.metadata.labels else "-", - ",".join(addresses) if addresses else "-", - ",".join(ports) if ports else "-", - ] - ) - - np_rows: List[List[str]] = [] - for np in network_policies: - ptypes = ",".join(np.spec.policy_types or []) if np.spec else "" - pod_sel = fmt_selector(np.spec.pod_selector.match_labels if np.spec and np.spec.pod_selector else {}) - np_rows.append([np.metadata.name, ptypes or "-", pod_sel]) - - print(f"Timestamp: {dt.datetime.now(dt.timezone.utc).isoformat()}") - print(f"Namespace: {namespace}") - - print_table("Pods", ["name", "phase", "pod_ip", "node", "host_ip"], pod_rows) - print_table("Services", ["name", "type", "cluster_ip", "ports", "selector"], svc_rows) - print_table("Endpoints", ["name", "addresses", "ports"], ep_rows) - print_table("EndpointSlices", ["name", "service", "addresses", "ports"], eps_rows) - print_table("NetworkPolicies", ["name", "policy_types", "pod_selector"], np_rows) - - return 0 - - -def main() -> int: - args = parse_args() - try: - load_kube_config(args) - except Exception as exc: # pragma: no cover - print(f"Failed to load kube config: {exc}", file=sys.stderr) - return 1 - return collect(args.namespace) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/docs/scripts/requirements.txt b/docs/scripts/requirements.txt deleted file mode 100644 index b5fe3da..0000000 --- a/docs/scripts/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -kubernetes>=29.0.0,<31.0.0 diff --git a/docs/scripts/service_path_inspector.py b/docs/scripts/service_path_inspector.py deleted file mode 100755 index fc245af..0000000 --- a/docs/scripts/service_path_inspector.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3.11 -"""Trace a Kubernetes Service to selectors, Pods, Endpoints, and EndpointSlices. - -Requires: - python3.11 -m pip install kubernetes - -Example: - python3.11 docs/scripts/service_path_inspector.py --namespace llm-observability --service ollama -""" - -from __future__ import annotations - -import argparse -import sys -from typing import Dict, List - -from kubernetes import client, config -from kubernetes.client import ApiException - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Inspect end-to-end service path") - parser.add_argument("--namespace", default="llm-observability", help="Namespace") - parser.add_argument("--service", required=True, help="Service name") - parser.add_argument("--context", default=None, help="Optional kubeconfig context") - parser.add_argument("--kubeconfig", default=None, help="Optional kubeconfig file path") - parser.add_argument("--in-cluster", action="store_true", help="Use in-cluster auth") - return parser.parse_args() - - -def load_cfg(args: argparse.Namespace) -> None: - if args.in_cluster: - config.load_incluster_config() - else: - config.load_kube_config(config_file=args.kubeconfig, context=args.context) - - -def selector_to_query(selector: Dict[str, str]) -> str: - if not selector: - return "" - return ",".join(f"{k}={v}" for k, v in selector.items()) - - -def print_header(label: str) -> None: - print(f"\n=== {label} ===") - - -def main() -> int: - args = parse_args() - try: - load_cfg(args) - except Exception as exc: - print(f"Failed to load kube config: {exc}", file=sys.stderr) - return 1 - - core = client.CoreV1Api() - discovery = client.DiscoveryV1Api() - - try: - svc = core.read_namespaced_service(name=args.service, namespace=args.namespace) - except ApiException as exc: - if exc.status == 404: - print(f"Service not found: {args.namespace}/{args.service}", file=sys.stderr) - return 2 - print(f"API error: {exc.status} {exc.reason}", file=sys.stderr) - return 2 - - selector = svc.spec.selector or {} - selector_query = selector_to_query(selector) - - print_header("Service") - print(f"name: {svc.metadata.name}") - print(f"namespace: {svc.metadata.namespace}") - print(f"type: {svc.spec.type}") - print(f"cluster_ip: {svc.spec.cluster_ip}") - print(f"ports: {[{'name': p.name, 'port': p.port, 'targetPort': p.target_port, 'protocol': p.protocol} for p in (svc.spec.ports or [])]}") - print(f"selector: {selector if selector else ''}") - - print_header("Selected Pods") - if not selector_query: - print("Service has no selector (possibly externalName or manually managed endpoints).") - pods = [] - else: - pods = core.list_namespaced_pod(namespace=args.namespace, label_selector=selector_query).items - if not pods: - print("No Pods matched service selector.") - for pod in pods: - print( - f"- {pod.metadata.name} phase={pod.status.phase} ready={all((cs.ready for cs in (pod.status.container_statuses or [])))} " - f"pod_ip={pod.status.pod_ip} node={pod.spec.node_name}" - ) - - print_header("Endpoints") - try: - ep = core.read_namespaced_endpoints(name=args.service, namespace=args.namespace) - if not ep.subsets: - print("No endpoint subsets present.") - else: - for subset in ep.subsets: - addrs = [a.ip for a in (subset.addresses or [])] - not_ready = [a.ip for a in (subset.not_ready_addresses or [])] - ports = [f"{p.name}:{p.port}/{p.protocol}" for p in (subset.ports or [])] - print(f"addresses={addrs or []} not_ready={not_ready or []} ports={ports or []}") - except ApiException as exc: - print(f"Failed to read Endpoints: {exc.status} {exc.reason}") - - print_header("EndpointSlices") - try: - slices = discovery.list_namespaced_endpoint_slice( - namespace=args.namespace, - label_selector=f"kubernetes.io/service-name={args.service}", - ).items - if not slices: - print("No EndpointSlices found for service.") - for eps in slices: - addresses: List[str] = [] - for endpoint in eps.endpoints or []: - addresses.extend(endpoint.addresses or []) - ports = [f"{p.name}:{p.port}/{p.protocol}" for p in eps.ports or []] - print(f"- {eps.metadata.name} addresses={addresses or []} ports={ports or []}") - except ApiException as exc: - print(f"Failed to list EndpointSlices: {exc.status} {exc.reason}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/docs/scripts/watch_endpoints.py b/docs/scripts/watch_endpoints.py deleted file mode 100755 index 445d2cd..0000000 --- a/docs/scripts/watch_endpoints.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3.11 -"""Watch endpoint changes for a single service. - -Requires: - python3.11 -m pip install kubernetes - -Example: - python3.11 docs/scripts/watch_endpoints.py --namespace llm-observability --service ollama --timeout 600 -""" - -from __future__ import annotations - -import argparse -import datetime as dt -import sys -from typing import List - -from kubernetes import client, config, watch -from kubernetes.client import ApiException - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Watch endpoint changes for one service") - parser.add_argument("--namespace", default="llm-observability", help="Namespace") - parser.add_argument("--service", required=True, help="Service name") - parser.add_argument("--context", default=None, help="Optional kubeconfig context") - parser.add_argument("--kubeconfig", default=None, help="Optional kubeconfig path") - parser.add_argument("--in-cluster", action="store_true", help="Use in-cluster auth") - parser.add_argument("--timeout", type=int, default=300, help="Watch timeout seconds") - return parser.parse_args() - - -def load_cfg(args: argparse.Namespace) -> None: - if args.in_cluster: - config.load_incluster_config() - else: - config.load_kube_config(config_file=args.kubeconfig, context=args.context) - - -def endpoint_snapshot(ep: client.V1Endpoints) -> str: - blocks: List[str] = [] - for subset in ep.subsets or []: - addrs = [a.ip for a in (subset.addresses or [])] - not_ready = [a.ip for a in (subset.not_ready_addresses or [])] - ports = [f"{p.name or ''}:{p.port}/{p.protocol}" for p in (subset.ports or [])] - blocks.append(f"ready={addrs or []} not_ready={not_ready or []} ports={ports or []}") - return " | ".join(blocks) if blocks else "" - - -def main() -> int: - args = parse_args() - try: - load_cfg(args) - except Exception as exc: - print(f"Failed to load kube config: {exc}", file=sys.stderr) - return 1 - - core = client.CoreV1Api() - - try: - core.read_namespaced_service(name=args.service, namespace=args.namespace) - except ApiException as exc: - if exc.status == 404: - print(f"Service not found: {args.namespace}/{args.service}", file=sys.stderr) - return 2 - print(f"API error while checking service: {exc.status} {exc.reason}", file=sys.stderr) - return 2 - - print(f"Watching endpoints for service {args.namespace}/{args.service} (timeout={args.timeout}s)") - print("Press Ctrl+C to stop.") - - w = watch.Watch() - try: - for event in w.stream( - core.list_namespaced_endpoints, - namespace=args.namespace, - timeout_seconds=args.timeout, - field_selector=f"metadata.name={args.service}", - ): - etype = event.get("type", "UNKNOWN") - obj: client.V1Endpoints = event["object"] - timestamp = dt.datetime.now(dt.timezone.utc).isoformat() - print(f"[{timestamp}] {etype} {obj.metadata.name} -> {endpoint_snapshot(obj)}") - except KeyboardInterrupt: - print("Stopped by user.") - except ApiException as exc: - print(f"Watch API error: {exc.status} {exc.reason}", file=sys.stderr) - return 2 - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/edge-toolbox/Dockerfile b/edge-toolbox/Dockerfile new file mode 100644 index 0000000..195abab --- /dev/null +++ b/edge-toolbox/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.26-bookworm AS build +WORKDIR /src +COPY go.mod go.sum ./ +COPY cmd/edge-toolbox ./cmd/edge-toolbox +COPY internal/toolbox ./internal/toolbox +RUN go mod edit -droprequire github.com/Edge-Computing-LLM/k3s-nvidia-edge \ + -dropreplace github.com/Edge-Computing-LLM/k3s-nvidia-edge \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/edge-toolbox ./cmd/edge-toolbox + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/edge-toolbox /usr/local/bin/edge-toolbox +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/edge-toolbox"] +CMD ["serve"] diff --git a/edge-toolbox/README.md b/edge-toolbox/README.md new file mode 100644 index 0000000..e1c98e7 --- /dev/null +++ b/edge-toolbox/README.md @@ -0,0 +1,23 @@ +# edge-toolbox + +`edge-toolbox` is the native Go in-cluster diagnostic image. Its static binary +supports `dns`, `http`, `tcp`, `redis-ping`, `ollama-smoke`, and OpenTelemetry +`seed` commands. The default `serve` command exposes `/healthz` on port 8080. + +Examples: + +```bash +kubectl exec -n llm-observability deploy/edge-toolbox -- edge-toolbox dns ollama open-webui +kubectl exec -n llm-observability deploy/edge-toolbox -- edge-toolbox http http://ollama:11434/api/tags +kubectl exec -n llm-observability deploy/edge-toolbox -- edge-toolbox redis-ping +kubectl exec -n llm-observability deploy/edge-toolbox -- edge-toolbox ollama-smoke +kubectl exec -n llm-observability deploy/edge-toolbox -- edge-toolbox seed --count 2 +``` + +Build from the repository root with `edge-toolbox/Dockerfile`. The resulting +image contains no shell, package manager, or Python runtime. + +```bash +./hack/build-local-image.sh edge-toolbox 0.2.0 . edge-toolbox/Dockerfile +./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 +``` diff --git a/go.mod b/go.mod index bbf1459..4b8c764 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,38 @@ module github.com/Edge-Computing-LLM/llm-observability-stack -go 1.22 +go 1.25.0 -require github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0 +require ( + github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013 + github.com/prometheus/client_golang v1.23.2 + go.opentelemetry.io/otel v1.44.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 + go.opentelemetry.io/otel/sdk v1.44.0 +) -replace github.com/Edge-Computing-LLM/k3s-nvidia-edge => ../k3s-nvidia-edge +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.81.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8dfbe66 --- /dev/null +++ b/go.sum @@ -0,0 +1,90 @@ +github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013 h1:yxsjOaobZcrVBnbF/Usz5HQ77HExRcOVkCi2o8dZhgs= +github.com/Edge-Computing-LLM/k3s-nvidia-edge v0.0.0-20260717201314-0b18be607013/go.mod h1:+7CsSGQtUt/2vgTNKT6NFOjx5gkIhrNIK4YFw4/IVyA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/hack/README.md b/hack/README.md index 8607809..8014c7c 100644 --- a/hack/README.md +++ b/hack/README.md @@ -30,26 +30,26 @@ This directory contains the local image workflow helpers for `llm-observability- ## Typical Usage -Build/import `langchain-demo`: +Build/import `ollama-gateway`: ```bash -./hack/build-local-image.sh langchain-demo 0.1.1 ./langchain-demo -./hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 +./hack/build-local-image.sh ollama-gateway 0.2.0 . ollama-gateway/Dockerfile +./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 ``` -Build/import `python-toolbox`: +Build/import `edge-toolbox`: ```bash -./hack/build-local-image.sh python-toolbox 0.2.0 ./python-toolbox -./hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 +./hack/build-local-image.sh edge-toolbox 0.2.0 . edge-toolbox/Dockerfile +./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 ``` Bootstrap the local full-stack profile: ```bash ./hack/bootstrap-enterprise-pilot-k3s.sh \ - --set langchainDemo.enabled=false \ - --set pythonToolbox.enabled=false + --set ollamaGateway.enabled=false \ + --set edgeToolbox.enabled=false ``` Force a CPU-only render or install path for validation: @@ -61,12 +61,12 @@ helm template llm-observability-stack . \ -f .generated/values.runtime-detected.yaml ``` -Enable `langchainDemo` and `pythonToolbox` only after their local images have been imported into k3s containerd. +Enable `ollamaGateway` and `edgeToolbox` only after their local images have been imported into k3s containerd. ## When To Use These Scripts -- after changing `langchain-demo/app.py` -- after changing scripts or dependencies in `python-toolbox/` +- after changing `cmd/ollama-gateway` or `internal/gateway` +- after changing scripts or dependencies in `edge-toolbox/` - when refreshing local image tags used by the chart ## After Import @@ -74,14 +74,14 @@ Enable `langchainDemo` and `pythonToolbox` only after their local images have be Restart the matching Kubernetes workload: ```bash -kubectl rollout restart deploy/langchain-demo -n llm-observability -kubectl rollout restart deploy/python-toolbox -n llm-observability +kubectl rollout restart deploy/ollama-gateway -n llm-observability +kubectl rollout restart deploy/edge-toolbox -n llm-observability ``` Then verify: ```bash -kubectl rollout status deploy/langchain-demo -n llm-observability -kubectl rollout status deploy/python-toolbox -n llm-observability -sudo k3s ctr images ls | grep -E 'langchain-demo|python-toolbox' +kubectl rollout status deploy/ollama-gateway -n llm-observability +kubectl rollout status deploy/edge-toolbox -n llm-observability +sudo k3s ctr images ls | grep -E 'ollama-gateway|edge-toolbox' ``` diff --git a/hack/bootstrap-enterprise-pilot-k3s.sh b/hack/bootstrap-enterprise-pilot-k3s.sh index 5a8b3e2..ee582e5 100755 --- a/hack/bootstrap-enterprise-pilot-k3s.sh +++ b/hack/bootstrap-enterprise-pilot-k3s.sh @@ -12,12 +12,12 @@ RUNTIME_VALUES_FILE="${RUNTIME_VALUES_FILE:-.generated/values.runtime-detected.y cd "${ROOT_DIR}" args_text=" $* " -langchain_disabled=false +gateway_disabled=false toolbox_disabled=false -if [[ "${args_text}" == *"langchainDemo.enabled=false"* ]]; then - langchain_disabled=true +if [[ "${args_text}" == *"ollamaGateway.enabled=false"* ]]; then + gateway_disabled=true fi -if [[ "${args_text}" == *"pythonToolbox.enabled=false"* ]]; then +if [[ "${args_text}" == *"edgeToolbox.enabled=false"* ]]; then toolbox_disabled=true fi @@ -30,8 +30,7 @@ check_runtime_image() { -n "${NAMESPACE}" \ --image="${image}" \ --image-pull-policy=Never \ - --restart=Never \ - --command -- python --version >/dev/null + --restart=Never >/dev/null sleep 5 local phase reason @@ -39,7 +38,7 @@ check_runtime_image() { reason="$(kubectl get pod -n "${NAMESPACE}" "${pod_name}" -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true)" kubectl delete pod -n "${NAMESPACE}" "${pod_name}" --ignore-not-found=true >/dev/null 2>&1 || true - [[ "${phase}" == "Succeeded" && -z "${reason}" ]] + [[ ( "${phase}" == "Running" || "${phase}" == "Succeeded" ) && -z "${reason}" ]] } echo "Checking Kubernetes access" @@ -54,23 +53,23 @@ if [[ "${AUTO_DETECT_RUNTIME}" == "true" ]]; then fi missing_images=() -if [[ "${langchain_disabled}" == false ]] && ! check_runtime_image image-check-langchain langchain-demo:0.1.1; then - missing_images+=("langchain-demo:0.1.1") +if [[ "${gateway_disabled}" == false ]] && ! check_runtime_image image-check-gateway ollama-gateway:0.2.0; then + missing_images+=("ollama-gateway:0.2.0") fi -if [[ "${toolbox_disabled}" == false ]] && ! check_runtime_image image-check-toolbox python-toolbox:0.2.0; then - missing_images+=("python-toolbox:0.2.0") +if [[ "${toolbox_disabled}" == false ]] && ! check_runtime_image image-check-toolbox edge-toolbox:0.2.0; then + missing_images+=("edge-toolbox:0.2.0") fi if (( ${#missing_images[@]} > 0 )); then printf 'Missing local image(s) from k3s containerd: %s\n' "${missing_images[*]}" >&2 cat >&2 <<'EOF' Build and import the local images first: - ./hack/build-local-image.sh langchain-demo 0.1.1 ./langchain-demo - ./hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 - ./hack/build-local-image.sh python-toolbox 0.2.0 ./python-toolbox - ./hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 + ./hack/build-local-image.sh ollama-gateway 0.2.0 . ollama-gateway/Dockerfile + ./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 + ./hack/build-local-image.sh edge-toolbox 0.2.0 . edge-toolbox/Dockerfile + ./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 Or run a platform-only install: - ./hack/bootstrap-enterprise-pilot-k3s.sh --set langchainDemo.enabled=false --set pythonToolbox.enabled=false + ./hack/bootstrap-enterprise-pilot-k3s.sh --set ollamaGateway.enabled=false --set edgeToolbox.enabled=false EOF exit 1 fi diff --git a/hack/build-local-image.sh b/hack/build-local-image.sh index 9de767d..dc9d01a 100755 --- a/hack/build-local-image.sh +++ b/hack/build-local-image.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash set -euo pipefail -IMAGE_REPO="${1:-langchain-demo}" +IMAGE_REPO="${1:-ollama-gateway}" IMAGE_TAG="${2:-0.1.0}" BUILD_CONTEXT="${3:-}" +DOCKERFILE="${4:-}" FULL_IMAGE="${IMAGE_REPO}:${IMAGE_TAG}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -13,12 +14,17 @@ if [[ -z "${BUILD_CONTEXT}" ]]; then BUILD_CONTEXT="${REPO_ROOT}/${IMAGE_REPO}" fi +build_args=() +if [[ -n "${DOCKERFILE}" ]]; then + build_args+=(--file "${DOCKERFILE}") +fi + if command -v nerdctl >/dev/null 2>&1; then echo "Building ${FULL_IMAGE} with nerdctl into containerd (k8s.io namespace)" - sudo nerdctl --namespace k8s.io build -t "${FULL_IMAGE}" "${BUILD_CONTEXT}" + sudo nerdctl --namespace k8s.io build "${build_args[@]}" -t "${FULL_IMAGE}" "${BUILD_CONTEXT}" else echo "nerdctl not found; falling back to docker build for ${FULL_IMAGE}" - docker build -t "${FULL_IMAGE}" "${BUILD_CONTEXT}" + docker build "${build_args[@]}" -t "${FULL_IMAGE}" "${BUILD_CONTEXT}" fi echo "Done: ${FULL_IMAGE}" diff --git a/hack/import-local-image-to-k3s.sh b/hack/import-local-image-to-k3s.sh index 7b83d6b..41925b7 100755 --- a/hack/import-local-image-to-k3s.sh +++ b/hack/import-local-image-to-k3s.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -IMAGE_REPO="${1:-langchain-demo}" +IMAGE_REPO="${1:-ollama-gateway}" IMAGE_TAG="${2:-0.1.0}" FULL_IMAGE="${IMAGE_REPO}:${IMAGE_TAG}" TMP_TAR="$(mktemp /tmp/local-image.XXXXXX.tar)" diff --git a/hack/validate-local-stack.sh b/hack/validate-local-stack.sh index 2e8e09d..4cfe121 100755 --- a/hack/validate-local-stack.sh +++ b/hack/validate-local-stack.sh @@ -7,9 +7,7 @@ if [[ "${1:-}" == "--strict-gpu" ]]; then STRICT_GPU=true fi -PYTHON_BIN="${PYTHON_BIN:-python3.11}" - -for command in helm kubectl "$PYTHON_BIN"; do +for command in go helm kubectl; do command -v "$command" >/dev/null || { echo "missing required command: $command" >&2; exit 1; } done @@ -21,7 +19,8 @@ helm template llm-observability-stack . \ --set opentelemetry.tracing.enabled=false \ --set openWebUI.existingSecret="" \ --set open-webui.webuiSecret.existingSecretName="" >/dev/null -"$PYTHON_BIN" -m pytest -q tests +go test ./... +go vet ./... kubectl get nodes kubectl get storageclass diff --git a/internal/benchmark/benchmark.go b/internal/benchmark/benchmark.go new file mode 100644 index 0000000..888ab7b --- /dev/null +++ b/internal/benchmark/benchmark.go @@ -0,0 +1,188 @@ +package benchmark + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "time" +) + +type Config struct { + URL, Model, Prompt, Output string + Runs, WarmupRuns int + Timeout time.Duration +} +type Result struct { + Success bool `json:"success"` + DurationSeconds float64 `json:"duration_seconds"` + TTFTSeconds float64 `json:"ttft_seconds"` + PromptTokens int `json:"prompt_tokens"` + GeneratedTokens int `json:"generated_tokens"` + TokensPerSecond float64 `json:"tokens_per_second"` +} +type Summary struct { + SchemaVersion int `json:"schema_version"` + CapturedAt string `json:"captured_at"` + Model string `json:"model"` + URL string `json:"url"` + Runs int `json:"runs"` + WarmupRuns int `json:"warmup_runs"` + TTFTP50Seconds float64 `json:"ttft_p50_seconds"` + TTFTP95Seconds float64 `json:"ttft_p95_seconds"` + DurationP50Seconds float64 `json:"duration_p50_seconds"` + DurationP95Seconds float64 `json:"duration_p95_seconds"` + TokensPerSecondMean float64 `json:"tokens_per_second_mean"` + Results []Result `json:"results"` +} + +func Run(ctx context.Context, config Config) (*Summary, error) { + if config.Runs < 1 { + return nil, fmt.Errorf("runs must be positive") + } + if config.Timeout <= 0 { + config.Timeout = 300 * time.Second + } + client := &http.Client{Timeout: config.Timeout} + for i := 0; i < config.WarmupRuns; i++ { + if _, err := runOnce(ctx, client, config); err != nil { + return nil, fmt.Errorf("warmup %d: %w", i+1, err) + } + } + results := make([]Result, 0, config.Runs) + for i := 0; i < config.Runs; i++ { + result, err := runOnce(ctx, client, config) + if err != nil { + return nil, fmt.Errorf("run %d: %w", i+1, err) + } + results = append(results, result) + } + ttft, duration := make([]float64, 0, len(results)), make([]float64, 0, len(results)) + throughput := 0.0 + for _, result := range results { + ttft = append(ttft, result.TTFTSeconds) + duration = append(duration, result.DurationSeconds) + throughput += result.TokensPerSecond + } + return &Summary{1, time.Now().UTC().Format(time.RFC3339Nano), config.Model, config.URL, config.Runs, config.WarmupRuns, Percentile(ttft, .50), Percentile(ttft, .95), Percentile(duration, .50), Percentile(duration, .95), throughput / float64(len(results)), results}, nil +} + +func runOnce(ctx context.Context, client *http.Client, config Config) (Result, error) { + payload, _ := json.Marshal(map[string]any{"model": config.Model, "prompt": config.Prompt, "stream": true, "options": map[string]any{"temperature": 0}}) + request, err := http.NewRequestWithContext(ctx, http.MethodPost, config.URL, bytes.NewReader(payload)) + if err != nil { + return Result{}, err + } + request.Header.Set("Content-Type", "application/json") + started := time.Now() + response, err := client.Do(request) + if err != nil { + return Result{}, err + } + defer response.Body.Close() + if response.StatusCode >= 400 { + detail, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) + return Result{}, fmt.Errorf("Ollama status %d: %s", response.StatusCode, detail) + } + firstTokenAt := time.Time{} + final := struct { + Done bool `json:"done"` + Response string `json:"response"` + EvalCount int `json:"eval_count"` + PromptEvalCount int `json:"prompt_eval_count"` + EvalDuration int64 `json:"eval_duration"` + }{} + scanner := bufio.NewScanner(response.Body) + scanner.Buffer(make([]byte, 64*1024), 2<<20) + for scanner.Scan() { + var chunk struct { + Done bool `json:"done"` + Response string `json:"response"` + EvalCount int `json:"eval_count"` + PromptEvalCount int `json:"prompt_eval_count"` + EvalDuration int64 `json:"eval_duration"` + } + if err := json.Unmarshal(scanner.Bytes(), &chunk); err != nil { + return Result{}, err + } + if firstTokenAt.IsZero() && chunk.Response != "" { + firstTokenAt = time.Now() + } + if chunk.Done { + final = chunk + } + } + if err := scanner.Err(); err != nil { + return Result{}, err + } + finished := time.Now() + if firstTokenAt.IsZero() { + firstTokenAt = finished + } + evalSeconds := float64(final.EvalDuration) / 1e9 + throughput := 0.0 + if evalSeconds > 0 { + throughput = float64(final.EvalCount) / evalSeconds + } + return Result{true, finished.Sub(started).Seconds(), firstTokenAt.Sub(started).Seconds(), final.PromptEvalCount, final.EvalCount, throughput}, nil +} + +func Percentile(values []float64, quantile float64) float64 { + if len(values) == 0 { + return 0 + } + ordered := append([]float64(nil), values...) + sort.Float64s(ordered) + index := int(float64(len(ordered)-1)*quantile + .5) + if index < 0 { + index = 0 + } + if index >= len(ordered) { + index = len(ordered) - 1 + } + return ordered[index] +} +func Write(path string, summary *Summary) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + data, err := json.MarshalIndent(summary, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0o644) +} + +func WithPortForward(ctx context.Context, namespace string, readyTimeout time.Duration, run func() error) error { + command := exec.CommandContext(ctx, "kubectl", "port-forward", "-n", namespace, "svc/ollama", "11434:11434") + var logs bytes.Buffer + command.Stdout, command.Stderr = &logs, &logs + if err := command.Start(); err != nil { + return err + } + defer func() { _ = command.Process.Kill(); _ = command.Wait() }() + deadline := time.Now().Add(readyTimeout) + for time.Now().Before(deadline) { + request, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://127.0.0.1:11434/api/tags", nil) + response, err := http.DefaultClient.Do(request) + if err == nil { + _ = response.Body.Close() + if response.StatusCode < 500 { + return run() + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(200 * time.Millisecond): + } + } + return fmt.Errorf("port-forward did not become ready: %s", logs.String()) +} diff --git a/internal/benchmark/benchmark_test.go b/internal/benchmark/benchmark_test.go new file mode 100644 index 0000000..6744860 --- /dev/null +++ b/internal/benchmark/benchmark_test.go @@ -0,0 +1,13 @@ +package benchmark + +import "testing" + +func TestPercentile(t *testing.T) { + values := []float64{4, 1, 2, 3} + if got := Percentile(values, .5); got != 3 { + t.Fatalf("p50=%v", got) + } + if got := Percentile(values, .95); got != 4 { + t.Fatalf("p95=%v", got) + } +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go new file mode 100644 index 0000000..3dd7e6e --- /dev/null +++ b/internal/gateway/gateway.go @@ -0,0 +1,294 @@ +package gateway + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httputil" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + otlptracegrpc "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +const defaultModel = "qwen-1-8b-chat-q4-k-m-local" + +type Config struct { + Address string + UpstreamURL string + Model string + ServiceName string + OTLPEndpoint string + TraceEnabled bool + Timeout time.Duration +} + +func ConfigFromEnv() Config { + port := env("PORT", "8000") + timeout, err := time.ParseDuration(env("OLLAMA_PROXY_TIMEOUT", "180s")) + if err != nil { + timeout = 180 * time.Second + } + return Config{Address: ":" + port, UpstreamURL: env("OLLAMA_UPSTREAM_BASE_URL", env("OLLAMA_BASE_URL", "http://ollama:11434")), Model: env("OLLAMA_MODEL", defaultModel), ServiceName: env("OTEL_SERVICE_NAME", "ollama-gateway"), OTLPEndpoint: os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"), TraceEnabled: strings.EqualFold(env("OTEL_TRACES_ENABLED", "true"), "true"), Timeout: timeout} +} + +func env(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + +type Metrics struct { + Requests *prometheus.CounterVec + Duration *prometheus.HistogramVec + Active *prometheus.GaugeVec + TTFT *prometheus.HistogramVec +} + +func newMetrics(registry *prometheus.Registry) *Metrics { + m := &Metrics{ + Requests: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "llm_observability_http_requests_total", Help: "Gateway HTTP requests."}, []string{"method", "route", "status"}), + Duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: "llm_observability_llm_request_duration_seconds", Help: "LLM request duration.", Buckets: prometheus.DefBuckets}, []string{"model", "route"}), + Active: prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "llm_observability_llm_active_requests", Help: "Active LLM requests."}, []string{"model", "route"}), + TTFT: prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: "llm_observability_time_to_first_token_seconds", Help: "Time to first upstream byte.", Buckets: prometheus.DefBuckets}, []string{"model", "route"}), + } + registry.MustRegister(m.Requests, m.Duration, m.Active, m.TTFT) + return m +} + +type Server struct { + config Config + client *http.Client + mux *http.ServeMux + metrics *Metrics + registry *prometheus.Registry + proxy *httputil.ReverseProxy + shutdownTrace func(context.Context) error +} + +func New(config Config) (*Server, error) { + upstream, err := url.Parse(config.UpstreamURL) + if err != nil { + return nil, fmt.Errorf("invalid Ollama upstream: %w", err) + } + registry := prometheus.NewRegistry() + s := &Server{config: config, client: &http.Client{Timeout: config.Timeout}, mux: http.NewServeMux(), registry: registry, metrics: newMetrics(registry), shutdownTrace: func(context.Context) error { return nil }} + s.proxy = httputil.NewSingleHostReverseProxy(upstream) + originalDirector := s.proxy.Director + s.proxy.Director = func(request *http.Request) { + originalDirector(request) + request.URL.Path = "/" + strings.TrimPrefix(request.URL.Path, "/ollama/") + request.Host = upstream.Host + } + s.proxy.ErrorHandler = func(writer http.ResponseWriter, request *http.Request, proxyErr error) { + http.Error(writer, "Ollama proxy request failed: "+proxyErr.Error(), http.StatusBadGateway) + } + if config.TraceEnabled && config.OTLPEndpoint != "" { + shutdown, err := configureTracing(context.Background(), config) + if err != nil { + return nil, err + } + s.shutdownTrace = shutdown + } + s.routes() + return s, nil +} + +func configureTracing(ctx context.Context, config Config) (func(context.Context) error, error) { + endpoint := strings.TrimPrefix(strings.TrimPrefix(config.OTLPEndpoint, "http://"), "https://") + exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithEndpoint(endpoint), otlptracegrpc.WithInsecure()) + if err != nil { + return nil, err + } + res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceName(config.ServiceName))) + if err != nil { + return nil, err + } + provider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(exporter), sdktrace.WithResource(res)) + otel.SetTracerProvider(provider) + return provider.Shutdown, nil +} + +func (s *Server) routes() { + s.mux.HandleFunc("/", s.root) + s.mux.HandleFunc("/healthz", s.health) + s.mux.HandleFunc("/config", s.configHandler) + s.mux.Handle("/metrics", promhttp.HandlerFor(s.registry, promhttp.HandlerOpts{})) + s.mux.HandleFunc("/invoke", s.invoke) + s.mux.HandleFunc("/ollama/", s.proxyHandler) +} + +func (s *Server) Handler() http.Handler { return s.observe(s.mux) } +func (s *Server) Shutdown(ctx context.Context) error { return s.shutdownTrace(ctx) } + +func (s *Server) Run(ctx context.Context) error { + server := &http.Server{Addr: s.config.Address, Handler: s.Handler(), ReadHeaderTimeout: 10 * time.Second} + errCh := make(chan error, 1) + go func() { errCh <- server.ListenAndServe() }() + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = s.Shutdown(shutdownCtx) + return server.Shutdown(shutdownCtx) + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + } +} + +type responseStatus struct { + http.ResponseWriter + code int +} + +func (w *responseStatus) WriteHeader(code int) { w.code = code; w.ResponseWriter.WriteHeader(code) } +func (w *responseStatus) Flush() { + if flusher, ok := w.ResponseWriter.(http.Flusher); ok { + flusher.Flush() + } +} + +func (s *Server) observe(next http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/metrics" { + next.ServeHTTP(writer, request) + return + } + started := time.Now() + route := request.URL.Path + if strings.HasPrefix(route, "/ollama/") { + route = "ollama_proxy" + } + ctx, span := otel.Tracer(s.config.ServiceName).Start(request.Context(), request.Method+" "+route) + defer span.End() + status := &responseStatus{ResponseWriter: writer, code: http.StatusOK} + next.ServeHTTP(status, request.WithContext(ctx)) + span.SetAttributes(attribute.Int("http.response.status_code", status.code), attribute.String("http.request.method", request.Method), attribute.String("url.path", request.URL.Path)) + s.metrics.Requests.WithLabelValues(request.Method, route, strconv.Itoa(status.code)).Inc() + s.metrics.Duration.WithLabelValues(s.config.Model, route).Observe(time.Since(started).Seconds()) + }) +} + +func writeJSON(writer http.ResponseWriter, status int, value any) { + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(value) +} +func (s *Server) root(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/" { + http.NotFound(writer, request) + return + } + writeJSON(writer, 200, map[string]string{"name": "Edge Ollama Go Gateway", "health": "/healthz", "invoke": "/invoke", "config": "/config", "ollama_proxy": "/ollama/api/*"}) +} +func (s *Server) health(writer http.ResponseWriter, _ *http.Request) { + writeJSON(writer, 200, map[string]any{"status": "ok", "model": s.config.Model, "ollama_base_url": s.config.UpstreamURL, "ollama_upstream_base_url": s.config.UpstreamURL, "otel_traces_enabled": s.config.TraceEnabled, "otel_exporter_otlp_endpoint": s.config.OTLPEndpoint}) +} +func (s *Server) configHandler(writer http.ResponseWriter, _ *http.Request) { + writeJSON(writer, 200, map[string]any{"model": s.config.Model, "ollama_base_url": s.config.UpstreamURL, "ollama_upstream_base_url": s.config.UpstreamURL, "otel_service_name": s.config.ServiceName, "otel_exporter_otlp_endpoint": s.config.OTLPEndpoint}) +} + +type promptInput struct { + Prompt string `json:"prompt"` + System string `json:"system"` +} + +func (s *Server) invoke(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost { + writer.Header().Set("Allow", http.MethodPost) + http.Error(writer, "method not allowed", http.StatusMethodNotAllowed) + return + } + var input promptInput + if err := json.NewDecoder(io.LimitReader(request.Body, 1<<20)).Decode(&input); err != nil || strings.TrimSpace(input.Prompt) == "" { + http.Error(writer, "prompt is required", http.StatusBadRequest) + return + } + prompt := input.Prompt + if input.System != "" { + prompt = "System: " + input.System + "\n\nUser: " + input.Prompt + } + payload, _ := json.Marshal(map[string]any{"model": s.config.Model, "prompt": prompt, "stream": false}) + upstreamRequest, err := http.NewRequestWithContext(request.Context(), http.MethodPost, strings.TrimRight(s.config.UpstreamURL, "/")+"/api/generate", strings.NewReader(string(payload))) + if err != nil { + http.Error(writer, err.Error(), 500) + return + } + upstreamRequest.Header.Set("Content-Type", "application/json") + s.metrics.Active.WithLabelValues(s.config.Model, "invoke").Inc() + defer s.metrics.Active.WithLabelValues(s.config.Model, "invoke").Dec() + response, err := s.client.Do(upstreamRequest) + if err != nil { + http.Error(writer, "Ollama request failed: "+err.Error(), http.StatusBadGateway) + return + } + defer response.Body.Close() + body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20)) + if err != nil { + http.Error(writer, err.Error(), 502) + return + } + if response.StatusCode >= 400 { + writer.Header().Set("Content-Type", response.Header.Get("Content-Type")) + writer.WriteHeader(response.StatusCode) + _, _ = writer.Write(body) + return + } + var result struct { + Response string `json:"response"` + } + if err := json.Unmarshal(body, &result); err != nil { + http.Error(writer, "invalid Ollama response", 502) + return + } + writeJSON(writer, 200, map[string]string{"response": result.Response, "model": s.config.Model, "ollama_base_url": s.config.UpstreamURL}) +} + +type firstWrite struct { + http.ResponseWriter + once sync.Once + observe func() +} + +func (w *firstWrite) Write(data []byte) (int, error) { + w.once.Do(w.observe) + return w.ResponseWriter.Write(data) +} +func (w *firstWrite) WriteHeader(code int) { w.once.Do(w.observe); w.ResponseWriter.WriteHeader(code) } +func (w *firstWrite) Flush() { + if flusher, ok := w.ResponseWriter.(http.Flusher); ok { + flusher.Flush() + } +} +func (s *Server) proxyHandler(writer http.ResponseWriter, request *http.Request) { + started := time.Now() + s.metrics.Active.WithLabelValues(s.config.Model, "ollama_proxy").Inc() + defer s.metrics.Active.WithLabelValues(s.config.Model, "ollama_proxy").Dec() + wrapped := &firstWrite{ResponseWriter: writer, observe: func() { + s.metrics.TTFT.WithLabelValues(s.config.Model, "ollama_proxy").Observe(time.Since(started).Seconds()) + }} + s.proxy.ServeHTTP(wrapped, request) +} + +func LogConfig(config Config) { + slog.Info("starting Go Ollama gateway", "address", config.Address, "upstream", config.UpstreamURL, "model", config.Model, "tracing", config.TraceEnabled) +} diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go new file mode 100644 index 0000000..1250a30 --- /dev/null +++ b/internal/gateway/gateway_test.go @@ -0,0 +1,70 @@ +package gateway + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func testServer(t *testing.T) (*Server, *httptest.Server) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/generate" { + _ = json.NewEncoder(w).Encode(map[string]any{"response": "synthetic-response", "done": true}) + return + } + w.Header().Set("Content-Type", "application/x-ndjson") + _, _ = w.Write([]byte("{\"token\":\"hello\"}\n{\"token\":\"world\"}\n")) + })) + t.Cleanup(upstream.Close) + server, err := New(Config{Address: ":0", UpstreamURL: upstream.URL, Model: defaultModel, ServiceName: "test", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + return server, httptest.NewServer(server.Handler()) +} +func TestEndpointsAndInvoke(t *testing.T) { + _, httpServer := testServer(t) + defer httpServer.Close() + for _, path := range []string{"/", "/healthz", "/config", "/metrics"} { + response, err := http.Get(httpServer.URL + path) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != 200 { + t.Fatalf("%s returned %d", path, response.StatusCode) + } + _ = response.Body.Close() + } + response, err := http.Post(httpServer.URL+"/invoke", "application/json", strings.NewReader(`{"prompt":"hello","system":"brief"}`)) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var result map[string]any + if err := json.NewDecoder(response.Body).Decode(&result); err != nil { + t.Fatal(err) + } + if result["response"] != "synthetic-response" { + t.Fatalf("response: %#v", result) + } +} +func TestStreamingProxy(t *testing.T) { + _, httpServer := testServer(t) + defer httpServer.Close() + response, err := http.Post(httpServer.URL+"/ollama/api/chat", "application/json", strings.NewReader(`{"stream":true}`)) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + content, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if string(content) != "{\"token\":\"hello\"}\n{\"token\":\"world\"}\n" { + t.Fatalf("content: %q", content) + } +} diff --git a/internal/stack/options.go b/internal/stack/options.go index 7d4bfe0..8d71f08 100644 --- a/internal/stack/options.go +++ b/internal/stack/options.go @@ -20,9 +20,11 @@ type Options struct { KeepNamespace bool Model string Runs int + WarmupRuns int Prompt string Output string OllamaSmoke bool + Service string } func DefaultOptions() Options { @@ -33,9 +35,11 @@ func DefaultOptions() Options { Timeout: "5m", Model: "qwen-1-8b-chat-q4-k-m-local", Runs: 3, + WarmupRuns: 1, Prompt: "Explain GPU observability in one concise sentence.", Output: "artifacts/benchmark-local.json", OllamaSmoke: true, + Service: "ollama", } } diff --git a/internal/stack/options_test.go b/internal/stack/options_test.go index 506a14c..2ffe2a7 100644 --- a/internal/stack/options_test.go +++ b/internal/stack/options_test.go @@ -33,13 +33,13 @@ func TestGPUProfile(t *testing.T) { } } -func TestGeForceDefaultsUseQwenAndPython311(t *testing.T) { +func TestGeForceDefaultsUseQwenAndNativeGoBenchmark(t *testing.T) { opts := DefaultOptions() if opts.Model != "qwen-1-8b-chat-q4-k-m-local" { t.Fatalf("default model = %q, want Qwen local alias", opts.Model) } - if !stringsContains(benchmarkCommand(opts), "python3.11 benchmarks/ollama_benchmark.py") { - t.Fatalf("benchmark command must use Python 3.11") + if !stringsContains(benchmarkCommand(opts), "llm-observability benchmark") { + t.Fatalf("benchmark command must use the native Go CLI") } } diff --git a/internal/stack/workflows.go b/internal/stack/workflows.go index 9474b7a..173333e 100644 --- a/internal/stack/workflows.go +++ b/internal/stack/workflows.go @@ -4,8 +4,10 @@ import ( "context" "fmt" "strings" + "time" "github.com/Edge-Computing-LLM/k3s-nvidia-edge/pkg/edgebase" + "github.com/Edge-Computing-LLM/llm-observability-stack/internal/benchmark" ) func Doctor(ctx context.Context, opts Options) error { @@ -69,13 +71,47 @@ func Validate(ctx context.Context, opts Options) error { } func Benchmark(ctx context.Context, opts Options) error { - r := runner(opts) - return r.Run(ctx, edgebase.Step{ - Name: "Ollama benchmark", - Command: benchmarkCommand(opts), + if err := runner(opts).Run(ctx, edgebase.Step{Name: "Wait for Ollama", Command: fmt.Sprintf("kubectl rollout status deploy/ollama -n %s --timeout=%s", shellQuote(opts.Namespace), shellQuote(opts.Timeout))}); err != nil { + return err + } + return benchmark.WithPortForward(ctx, opts.Namespace, 15*time.Second, func() error { + summary, err := benchmark.Run(ctx, benchmark.Config{URL: "http://127.0.0.1:11434/api/generate", Model: opts.Model, Prompt: opts.Prompt, Output: opts.Output, Runs: opts.Runs, WarmupRuns: opts.WarmupRuns, Timeout: 300 * time.Second}) + if err != nil { + return err + } + if err := benchmark.Write(opts.Output, summary); err != nil { + return err + } + fmt.Printf("benchmark written to %s\n", opts.Output) + return nil }) } +func NetworkInventory(ctx context.Context, opts Options) error { + return runner(opts).Run(ctx, edgebase.Step{Name: "Kubernetes network inventory", Command: fmt.Sprintf("kubectl get pods,services,endpoints,endpointslices.discovery.k8s.io,networkpolicies.networking.k8s.io -n %s -o wide", shellQuote(opts.Namespace))}) +} + +func ServicePath(ctx context.Context, opts Options) error { + if strings.TrimSpace(opts.Service) == "" { + return fmt.Errorf("--service is required") + } + ns, service := shellQuote(opts.Namespace), shellQuote(opts.Service) + command := fmt.Sprintf(`set -e +kubectl get service -n %s %s -o wide +selector="$(kubectl get service -n %s %s -o go-template='{{range $key,$value := .spec.selector}}{{printf "%%s=%%s," $key $value}}{{end}}' | sed 's/,$//')" +if [ -n "$selector" ]; then kubectl get pods -n %s -l "$selector" -o wide; else echo "Service has no selector"; fi +kubectl get endpoints -n %s %s -o wide +kubectl get endpointslices.discovery.k8s.io -n %s -l kubernetes.io/service-name=%s -o wide`, ns, service, ns, service, ns, ns, service, ns, service) + return runner(opts).Run(ctx, edgebase.Step{Name: "Kubernetes service path", Command: command}) +} + +func WatchEndpoints(ctx context.Context, opts Options) error { + if strings.TrimSpace(opts.Service) == "" { + return fmt.Errorf("--service is required") + } + return runner(opts).Run(ctx, edgebase.Step{Name: "Watch Kubernetes endpoints", Command: fmt.Sprintf("kubectl get endpoints -n %s %s --watch --request-timeout=%s", shellQuote(opts.Namespace), shellQuote(opts.Service), shellQuote(opts.Timeout))}) +} + func Uninstall(ctx context.Context, opts Options) error { r := runner(opts) for _, step := range uninstallSteps(opts) { @@ -159,7 +195,7 @@ func baseReadySteps(opts Options) []edgebase.Step { func stackDoctorSteps(opts Options) []edgebase.Step { ns := shellQuote(opts.Namespace) steps := []edgebase.Step{ - {Name: "Required commands", Command: "missing=0; for c in kubectl helm python3.11; do command -v $c >/dev/null && echo \"$c: $(command -v $c)\" || { echo \"$c: missing\"; missing=1; }; done; exit $missing"}, + {Name: "Required commands", Command: "missing=0; for c in kubectl helm; do command -v $c >/dev/null && echo \"$c: $(command -v $c)\" || { echo \"$c: missing\"; missing=1; }; done; exit $missing"}, {Name: "Helm release", Command: fmt.Sprintf("helm status %s -n %s || true", shellQuote(opts.Release), ns)}, {Name: "LLM namespace", Command: fmt.Sprintf("kubectl get namespace %s || true", ns)}, {Name: "LLM workloads", Command: fmt.Sprintf("kubectl get pods,deploy,statefulset,svc,pvc -n %s -o wide || true", ns)}, @@ -191,7 +227,7 @@ fi`)}, func statusSteps(opts Options) []edgebase.Step { ns := shellQuote(opts.Namespace) return []edgebase.Step{ - {Name: "Base GPU layer", Command: "kubectl get pods -n gpu-operator -o wide || true\nkubectl get runtimeclass nvidia || true\nkubectl get nodes -o custom-columns=NAME:.metadata.name,GPU:.status.allocatable.nvidia\\.com/gpu || true"}, + {Name: "Base GPU layer", Command: "kubectl get pods -n gpu-operator -o wide || true\nkubectl get runtimeclass nvidia || true\nkubectl get nodes -o custom-columns=NAME:.metadata.name,GPU:.status.allocatable.nvidia\\\\.com/gpu || true"}, {Name: "Helm release", Command: fmt.Sprintf("helm status %s -n %s || true", shellQuote(opts.Release), ns)}, {Name: "LLM workloads", Command: fmt.Sprintf("kubectl get pods,deploy,statefulset,svc,pvc -n %s -o wide", ns)}, {Name: "Services and ports", Command: fmt.Sprintf("kubectl get svc -n %s -o wide", ns)}, @@ -265,23 +301,7 @@ func helmInstallCommand(opts Options) string { } func benchmarkCommand(opts Options) string { - return withRoot(fmt.Sprintf(`set -euo pipefail -command -v python3.11 >/dev/null -kubectl rollout status deploy/ollama -n %s --timeout=%s -pf_log="$(mktemp)" -kubectl port-forward -n %s svc/ollama 11434:11434 >"$pf_log" 2>&1 & -pf_pid="$!" -cleanup() { kill "$pf_pid" >/dev/null 2>&1 || true; rm -f "$pf_log"; } -trap cleanup EXIT -sleep 3 -python3.11 benchmarks/ollama_benchmark.py --url http://127.0.0.1:11434/api/generate --model %s --runs %d --warmup-runs 1 --prompt %s --output %s`, - shellQuote(opts.Namespace), - shellQuote(opts.Timeout), - shellQuote(opts.Namespace), - shellQuote(opts.Model), - opts.Runs, - shellQuote(opts.Prompt), - shellQuote(opts.Output))) + return fmt.Sprintf("llm-observability benchmark --namespace %s --model %s --runs %d --prompt %s --output %s", shellQuote(opts.Namespace), shellQuote(opts.Model), opts.Runs, shellQuote(opts.Prompt), shellQuote(opts.Output)) } func withRoot(command string) string { diff --git a/internal/toolbox/toolbox.go b/internal/toolbox/toolbox.go new file mode 100644 index 0000000..d1e2c1f --- /dev/null +++ b/internal/toolbox/toolbox.go @@ -0,0 +1,261 @@ +package toolbox + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net" + "net/http" + "os" + "strconv" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + otlptracegrpc "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +func Run(ctx context.Context, args []string, output io.Writer) error { + if len(args) == 0 { + return errors.New("command required: serve, dns, http, tcp, redis-ping, ollama-smoke, or seed") + } + switch args[0] { + case "serve": + return serve(ctx, args[1:]) + case "dns": + return dns(args[1:], output) + case "http": + return httpCheck(ctx, args[1:], output) + case "tcp": + return tcpCheck(ctx, args[1:], output) + case "redis-ping": + return redisPing(ctx, args[1:], output) + case "ollama-smoke": + return ollamaSmoke(ctx, args[1:], output) + case "seed": + return seed(ctx, args[1:], output) + default: + return fmt.Errorf("unknown command %q", args[0]) + } +} + +func serve(ctx context.Context, args []string) error { + flags := flag.NewFlagSet("serve", flag.ContinueOnError) + address := flags.String("address", ":8080", "health server address") + if err := flags.Parse(args); err != nil { + return err + } + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, "{\"status\":\"ok\",\"runtime\":\"go\"}\n") + }) + server := &http.Server{Addr: *address, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + errCh := make(chan error, 1) + go func() { errCh <- server.ListenAndServe() }() + select { + case <-ctx.Done(): + stop, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return server.Shutdown(stop) + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + } +} + +func dns(args []string, output io.Writer) error { + if len(args) == 0 { + return errors.New("dns requires at least one host") + } + for _, host := range args { + addresses, err := net.LookupHost(host) + if err != nil { + return fmt.Errorf("resolve %s: %w", host, err) + } + fmt.Fprintf(output, "%s\t%s\n", host, strings.Join(addresses, ",")) + } + return nil +} + +func httpCheck(ctx context.Context, args []string, output io.Writer) error { + flags := flag.NewFlagSet("http", flag.ContinueOnError) + timeout := flags.Duration("timeout", 10*time.Second, "request timeout") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() != 1 { + return errors.New("http requires one URL") + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, flags.Arg(0), nil) + if err != nil { + return err + } + client := &http.Client{Timeout: *timeout} + started := time.Now() + response, err := client.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + body, _ := io.ReadAll(io.LimitReader(response.Body, 500)) + fmt.Fprintf(output, "status=%d duration=%s body=%q\n", response.StatusCode, time.Since(started).Round(time.Millisecond), strings.TrimSpace(string(body))) + if response.StatusCode >= 400 { + return fmt.Errorf("HTTP status %d", response.StatusCode) + } + return nil +} + +func tcpCheck(ctx context.Context, args []string, output io.Writer) error { + flags := flag.NewFlagSet("tcp", flag.ContinueOnError) + timeout := flags.Duration("timeout", 5*time.Second, "connect timeout") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() != 1 { + return errors.New("tcp requires host:port") + } + dialer := net.Dialer{Timeout: *timeout} + started := time.Now() + connection, err := dialer.DialContext(ctx, "tcp", flags.Arg(0)) + if err != nil { + return err + } + _ = connection.Close() + fmt.Fprintf(output, "connected=%s duration=%s\n", flags.Arg(0), time.Since(started).Round(time.Millisecond)) + return nil +} + +func redisPing(ctx context.Context, args []string, output io.Writer) error { + flags := flag.NewFlagSet("redis-ping", flag.ContinueOnError) + address := flags.String("address", env("REDIS_HOST", "open-webui-redis")+":"+env("REDIS_PORT", "6379"), "Redis host:port") + if err := flags.Parse(args); err != nil { + return err + } + dialer := net.Dialer{Timeout: 5 * time.Second} + connection, err := dialer.DialContext(ctx, "tcp", *address) + if err != nil { + return err + } + defer connection.Close() + _ = connection.SetDeadline(time.Now().Add(5 * time.Second)) + if _, err := io.WriteString(connection, "*1\r\n$4\r\nPING\r\n"); err != nil { + return err + } + response, err := bufio.NewReader(connection).ReadString('\n') + if err != nil { + return err + } + fmt.Fprint(output, response) + if !strings.HasPrefix(response, "+PONG") { + return fmt.Errorf("unexpected Redis response %q", response) + } + return nil +} + +func ollamaSmoke(ctx context.Context, args []string, output io.Writer) error { + flags := flag.NewFlagSet("ollama-smoke", flag.ContinueOnError) + base := flags.String("url", env("OLLAMA_BASE_URL", "http://ollama:11434"), "Ollama base URL") + model := flags.String("model", env("OLLAMA_MODEL", "qwen-1-8b-chat-q4-k-m-local"), "model") + prompt := flags.String("prompt", "Reply with one short sentence saying the stack is healthy.", "prompt") + if err := flags.Parse(args); err != nil { + return err + } + payload, _ := json.Marshal(map[string]any{"model": *model, "messages": []map[string]string{{"role": "user", "content": *prompt}}, "stream": false}) + request, _ := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(*base, "/")+"/api/chat", strings.NewReader(string(payload))) + request.Header.Set("Content-Type", "application/json") + response, err := (&http.Client{Timeout: 180 * time.Second}).Do(request) + if err != nil { + return err + } + defer response.Body.Close() + body, _ := io.ReadAll(io.LimitReader(response.Body, 2<<20)) + fmt.Fprintln(output, string(body)) + if response.StatusCode >= 400 { + return fmt.Errorf("Ollama status %d", response.StatusCode) + } + return nil +} + +func seed(ctx context.Context, args []string, output io.Writer) error { + flags := flag.NewFlagSet("seed", flag.ContinueOnError) + endpoint := flags.String("otlp-endpoint", env("OTEL_EXPORTER_OTLP_ENDPOINT", "opentelemetry-collector:4317"), "OTLP gRPC endpoint") + apiURL := flags.String("url", env("OBS_INFERENCE_API_URL", "http://ollama:11434/api/chat"), "Ollama chat URL") + model := flags.String("model", env("OLLAMA_MODEL", "qwen-1-8b-chat-q4-k-m-local"), "model") + count := flags.Int("count", envInt("OBS_CALL_COUNT", 2), "call count") + if err := flags.Parse(args); err != nil { + return err + } + shutdown, err := tracing(ctx, *endpoint) + if err != nil { + return err + } + defer func() { _ = shutdown(context.Background()) }() + tracer := otel.Tracer("llm-observability-stack.edge-toolbox") + prompts := []string{"Explain Kubernetes readiness probes briefly.", "Give one use of GPU acceleration for LLM inference.", "Name useful OpenTelemetry GenAI span attributes."} + client := &http.Client{Timeout: 180 * time.Second} + failures := 0 + for i := 0; i < *count; i++ { + prompt := prompts[i%len(prompts)] + callCtx, span := tracer.Start(ctx, "ollama seed chat") + span.SetAttributes(attribute.String("gen_ai.system", "ollama"), attribute.String("gen_ai.operation.name", "chat"), attribute.String("gen_ai.request.model", *model), attribute.Int("llm.seed.index", i+1)) + payload, _ := json.Marshal(map[string]any{"model": *model, "stream": false, "messages": []map[string]string{{"role": "user", "content": prompt}}}) + request, _ := http.NewRequestWithContext(callCtx, http.MethodPost, *apiURL, strings.NewReader(string(payload))) + request.Header.Set("Content-Type", "application/json") + response, requestErr := client.Do(request) + if requestErr != nil { + failures++ + span.RecordError(requestErr) + } else { + _, _ = io.Copy(io.Discard, response.Body) + _ = response.Body.Close() + if response.StatusCode >= 400 { + failures++ + } + } + span.End() + } + fmt.Fprintf(output, "seeded=%d failed=%d\n", *count, failures) + if failures > 0 { + return fmt.Errorf("%d seed requests failed", failures) + } + return nil +} + +func tracing(ctx context.Context, endpoint string) (func(context.Context) error, error) { + endpoint = strings.TrimPrefix(strings.TrimPrefix(endpoint, "http://"), "https://") + exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithEndpoint(endpoint), otlptracegrpc.WithInsecure()) + if err != nil { + return nil, err + } + res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceName(env("OTEL_SERVICE_NAME", "edge-toolbox")))) + if err != nil { + return nil, err + } + provider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(exporter), sdktrace.WithResource(res)) + otel.SetTracerProvider(provider) + return provider.Shutdown, nil +} +func env(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} +func envInt(key string, fallback int) int { + value, err := strconv.Atoi(os.Getenv(key)) + if err == nil { + return value + } + return fallback +} diff --git a/internal/toolbox/toolbox_test.go b/internal/toolbox/toolbox_test.go new file mode 100644 index 0000000..679a831 --- /dev/null +++ b/internal/toolbox/toolbox_test.go @@ -0,0 +1,31 @@ +package toolbox + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestDNS(t *testing.T) { + var output bytes.Buffer + if err := dns([]string{"localhost"}, &output); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "localhost") { + t.Fatalf("output: %s", output.String()) + } +} +func TestHTTPCheck(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("ok")) })) + defer server.Close() + var output bytes.Buffer + if err := httpCheck(context.Background(), []string{server.URL}, &output); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "status=200") { + t.Fatalf("output: %s", output.String()) + } +} diff --git a/jupyter-notebooks/01-environment-smoke-test.ipynb b/jupyter-notebooks/01-environment-smoke-test.ipynb index a33ada4..c7ac48b 100644 --- a/jupyter-notebooks/01-environment-smoke-test.ipynb +++ b/jupyter-notebooks/01-environment-smoke-test.ipynb @@ -234,9 +234,9 @@ " \"gpu_count\": values.get(\"nvidia\", {}).get(\"gpuCount\"),\n", " \"opentelemetry_project\": values.get(\"opentelemetry\", {}).get(\"project\"),\n", " \"ollama_service_type\": values.get(\"ollama\", {}).get(\"service\", {}).get(\"type\"),\n", - " \"langchain_service_type\": values.get(\"langchainDemo\", {}).get(\"service\", {}).get(\"type\"),\n", + " \"langchain_service_type\": values.get(\"ollamaGateway\", {}).get(\"service\", {}).get(\"type\"),\n", " \"open_webui_service_type\": values.get(\"openWebUI\", {}).get(\"service\", {}).get(\"type\"),\n", - " \"python_toolbox_enabled\": values.get(\"pythonToolbox\", {}).get(\"enabled\"),\n", + " \"python_toolbox_enabled\": values.get(\"edgeToolbox\", {}).get(\"enabled\"),\n", " \"seeder_enabled\": values.get(\"opentelemetryDashboardSeeder\", {}).get(\"enabled\"),\n", " \"redis_enabled\": values.get(\"redis\", {}).get(\"enabled\"),\n", "}\n", diff --git a/jupyter-notebooks/02-ollama-api-basics.ipynb b/jupyter-notebooks/02-ollama-api-basics.ipynb index ba7b8c7..64f7525 100644 --- a/jupyter-notebooks/02-ollama-api-basics.ipynb +++ b/jupyter-notebooks/02-ollama-api-basics.ipynb @@ -321,7 +321,7 @@ "metadata": {}, "source": [ "## Done\n", - "Next: `03-langchain-proxy-deep-dive.ipynb` to compare direct Ollama calls with your traced proxy path.\n" + "Next: `03-ollama-gateway-deep-dive.ipynb` to compare direct Ollama calls with your traced proxy path.\n" ] } ], diff --git a/jupyter-notebooks/03-langchain-proxy-deep-dive.ipynb b/jupyter-notebooks/03-ollama-gateway-deep-dive.ipynb similarity index 94% rename from jupyter-notebooks/03-langchain-proxy-deep-dive.ipynb rename to jupyter-notebooks/03-ollama-gateway-deep-dive.ipynb index 90d1e4f..3e095f5 100644 --- a/jupyter-notebooks/03-langchain-proxy-deep-dive.ipynb +++ b/jupyter-notebooks/03-ollama-gateway-deep-dive.ipynb @@ -5,13 +5,13 @@ "id": "a3005ee8", "metadata": {}, "source": [ - "# 03 - LangChain Proxy Deep Dive\n", + "# 03 - Ollama Go Gateway Deep Dive\n", "\n", - "Your Open WebUI traffic is configured to flow through `langchain-demo` at `/ollama/api/*`.\n", + "Your Open WebUI traffic is configured to flow through `ollama-gateway` at `/ollama/api/*`.\n", "\n", "Recommended pre-step (separate terminal):\n", "```bash\n", - "kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000\n", + "kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000\n", "kubectl port-forward -n llm-observability svc/ollama 11434:11434\n", "```" ] @@ -78,7 +78,7 @@ "\n", "PROXY_BASE = \"http://localhost:8000\"\n", "OLLAMA_DIRECT = \"http://localhost:11434\"\n", - "PROXY_PORT_FORWARD_CMD = \"kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000\"\n", + "PROXY_PORT_FORWARD_CMD = \"kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000\"\n", "OLLAMA_PORT_FORWARD_CMD = \"kubectl port-forward -n llm-observability svc/ollama 11434:11434\"\n", "MODEL_NAME = \"gemma3-1b-it-gguf-local\"\n", "\n", @@ -119,7 +119,7 @@ "outputs": [], "source": [ "if not PROXY_AVAILABLE:\n", - " print(\"Skipping proxy configuration check: langchain-demo is not reachable on localhost:8000\")\n", + " print(\"Skipping proxy configuration check: ollama-gateway is not reachable on localhost:8000\")\n", "else:\n", " for endpoint in [\"/\", \"/healthz\", \"/config\"]:\n", " try:\n", @@ -154,7 +154,7 @@ "}\n", "\n", "if not PROXY_AVAILABLE:\n", - " print(\"Skipping /invoke test: langchain-demo is not reachable on localhost:8000\")\n", + " print(\"Skipping /invoke test: ollama-gateway is not reachable on localhost:8000\")\n", "else:\n", " try:\n", " r = requests.post(f\"{PROXY_BASE}/invoke\", json=invoke_payload, timeout=180)\n", @@ -180,7 +180,7 @@ "outputs": [], "source": [ "if not PROXY_AVAILABLE:\n", - " print(\"Skipping proxy Ollama endpoint tests: langchain-demo is not reachable on localhost:8000\")\n", + " print(\"Skipping proxy Ollama endpoint tests: ollama-gateway is not reachable on localhost:8000\")\n", "else:\n", " try:\n", " tags = requests.get(f\"{PROXY_BASE}/ollama/api/tags\", timeout=20)\n", @@ -217,7 +217,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Keep this benchmark intentionally light so langchain-demo health probes stay responsive.\n", + "# Keep this benchmark intentionally light so ollama-gateway health probes stay responsive.\n", "MAX_ITERS = 4\n", "REQUEST_TIMEOUT_SECONDS = 120\n", "SLEEP_BETWEEN_CALLS_SECONDS = 2\n", @@ -328,7 +328,7 @@ "source": [ "import subprocess\n", "\n", - "cmd = \"kubectl logs -n llm-observability deploy/langchain-demo --tail=80\"\n", + "cmd = \"kubectl logs -n llm-observability deploy/ollama-gateway --tail=80\"\n", "proc = subprocess.run(cmd, shell=True, text=True, capture_output=True)\n", "print(proc.stdout if proc.stdout else proc.stderr)\n" ] diff --git a/jupyter-notebooks/04-opentelemetry-genai-tracing-setup.ipynb b/jupyter-notebooks/04-opentelemetry-genai-tracing-setup.ipynb index 79f479b..5c42dfa 100644 --- a/jupyter-notebooks/04-opentelemetry-genai-tracing-setup.ipynb +++ b/jupyter-notebooks/04-opentelemetry-genai-tracing-setup.ipynb @@ -12,7 +12,7 @@ "It expects:\n", "- `OTEL_EXPORTER_OTLP_ENDPOINT` (or `OTEL_EXPORTER_OTLP_ENDPOINT`) in your notebook environment\n", "- proxy endpoint reachable at `http://localhost:8000/ollama/api/chat`\n", - "- if `localhost:8000` is unavailable, run `kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000` in another terminal\n" + "- if `localhost:8000` is unavailable, run `kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000` in another terminal\n" ] }, { @@ -83,7 +83,7 @@ "OTEL_EXPORTER_OTLP_ENDPOINT = os.getenv(\"OTEL_EXPORTER_OTLP_ENDPOINT\") or os.getenv(\"OTEL_EXPORTER_OTLP_ENDPOINT\")\n", "PROXY_HEALTH_URL = \"http://localhost:8000/healthz\"\n", "PROXY_CHAT_URL = \"http://localhost:8000/ollama/api/chat\"\n", - "PROXY_PORT_FORWARD_CMD = \"kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000\"\n", + "PROXY_PORT_FORWARD_CMD = \"kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000\"\n", "MODEL_NAME = os.getenv(\"OLLAMA_MODEL\", \"gemma3-1b-it-gguf-local\")\n", "\n", "\n", diff --git a/jupyter-notebooks/05-open-webui-end-to-end.ipynb b/jupyter-notebooks/05-open-webui-end-to-end.ipynb index 9a64437..78b69bf 100644 --- a/jupyter-notebooks/05-open-webui-end-to-end.ipynb +++ b/jupyter-notebooks/05-open-webui-end-to-end.ipynb @@ -8,7 +8,7 @@ "# 05 - Open WebUI End-to-End Flow\n", "\n", "This notebook validates the full path:\n", - "**Browser -> Open WebUI -> langchain-demo proxy -> Ollama -> OpenTelemetry**\n", + "**Browser -> Open WebUI -> ollama-gateway proxy -> Ollama -> OpenTelemetry**\n", "\n", "You will trigger prompts from the browser, then verify traces and metrics here.\n" ] @@ -320,10 +320,10 @@ "import subprocess\n", "\n", "for cmd in [\n", - " \"kubectl get pods -n llm-observability -l app.kubernetes.io/name=langchain-demo\",\n", + " \"kubectl get pods -n llm-observability -l app.kubernetes.io/name=ollama-gateway\",\n", " \"kubectl get pods -n llm-observability -l app.kubernetes.io/name=ollama\",\n", " \"kubectl get pods -n llm-observability -l app.kubernetes.io/name=open-webui\",\n", - " \"kubectl logs -n llm-observability deploy/langchain-demo --tail=60\",\n", + " \"kubectl logs -n llm-observability deploy/ollama-gateway --tail=60\",\n", "]:\n", " print(\"=\" * 120)\n", " print(\"$\", cmd)\n", diff --git a/jupyter-notebooks/07-python-toolbox-diagnostics.ipynb b/jupyter-notebooks/07-edge-toolbox-diagnostics.ipynb similarity index 82% rename from jupyter-notebooks/07-python-toolbox-diagnostics.ipynb rename to jupyter-notebooks/07-edge-toolbox-diagnostics.ipynb index 26de15b..a04e8e8 100644 --- a/jupyter-notebooks/07-python-toolbox-diagnostics.ipynb +++ b/jupyter-notebooks/07-edge-toolbox-diagnostics.ipynb @@ -5,11 +5,11 @@ "id": "3ceaae3c", "metadata": {}, "source": [ - "# 07 - Python Toolbox Diagnostics\n", + "# 07 - Go Edge Toolbox Diagnostics\n", "\n", - "This notebook covers toolbox scripts, manual python-toolbox enable/disable controls, in-cluster diagnostics, memory usage, and GPU correlation.\n", + "This notebook covers toolbox scripts, manual edge-toolbox enable/disable controls, in-cluster diagnostics, memory usage, and GPU correlation.\n", "\n", - "Your current local profile keeps `pythonToolbox.enabled: true` by default so in-cluster diagnostics are available immediately. Use the manual toggle below only if you want to switch it off temporarily.\n" + "Your current local profile keeps `edgeToolbox.enabled: true` by default so in-cluster diagnostics are available immediately. Use the manual toggle below only if you want to switch it off temporarily.\n" ] }, { @@ -149,7 +149,7 @@ " return {}\n", "\n", "\n", - "def can_run_toolbox(action: str = \"python-toolbox diagnostics\") -> bool:\n", + "def can_run_toolbox(action: str = \"edge-toolbox diagnostics\") -> bool:\n", " if not PROJECT_SCOPE_OK:\n", " print(f\"Skipping {action}: current working directory is outside project root.\")\n", " print(f\" cwd : {CWD}\")\n", @@ -196,7 +196,7 @@ "\n", "def list_toolbox_pods() -> list[str]:\n", " pod_data = run_kubectl_json(\n", - " f\"get pods -n {NAMESPACE} -l app.kubernetes.io/name=python-toolbox -o json\",\n", + " f\"get pods -n {NAMESPACE} -l app.kubernetes.io/name=edge-toolbox -o json\",\n", " check=False,\n", " )\n", " return [\n", @@ -208,7 +208,7 @@ "\n", "def list_running_toolbox_pods() -> list[str]:\n", " pod_data = run_kubectl_json(\n", - " f\"get pods -n {NAMESPACE} -l app.kubernetes.io/name=python-toolbox -o json\",\n", + " f\"get pods -n {NAMESPACE} -l app.kubernetes.io/name=edge-toolbox -o json\",\n", " check=False,\n", " )\n", " return [\n", @@ -237,7 +237,7 @@ " f\"helm upgrade --install llm-observability-stack . \"\n", " f\"-n {NAMESPACE} --create-namespace \"\n", " f\"-f {shlex.quote(str(values_file))} \"\n", - " f\"--set pythonToolbox.enabled={enabled_literal}\"\n", + " f\"--set edgeToolbox.enabled={enabled_literal}\"\n", " )\n", "\n", "\n", @@ -270,8 +270,8 @@ " values = {}\n", " print(f\"values.local-k3s.yaml not found at: {values_path}\")\n", "\n", - "file_enabled = values.get(\"pythonToolbox\", {}).get(\"enabled\")\n", - "print(\"values.local-k3s.yaml pythonToolbox.enabled:\", file_enabled)\n", + "file_enabled = values.get(\"edgeToolbox\", {}).get(\"enabled\")\n", + "print(\"values.local-k3s.yaml edgeToolbox.enabled:\", file_enabled)\n", "\n", "release_enabled = None\n", "if can_run_helm(\"helm release value check\"):\n", @@ -284,33 +284,33 @@ " if release_proc.returncode == 0 and release_proc.stdout.strip():\n", " try:\n", " release_values = yaml.safe_load(release_proc.stdout) or {}\n", - " release_enabled = release_values.get(\"pythonToolbox\", {}).get(\"enabled\")\n", + " release_enabled = release_values.get(\"edgeToolbox\", {}).get(\"enabled\")\n", " except Exception as exc:\n", " print(f\"Unable to parse Helm release values: {exc}\")\n", " else:\n", " print(\"Helm release values unavailable. Is llm-observability-stack installed?\")\n", "\n", - "print(\"live Helm release pythonToolbox.enabled :\", release_enabled)\n", + "print(\"live Helm release edgeToolbox.enabled :\", release_enabled)\n", "\n", - "if can_run_toolbox(\"python-toolbox status checks\"):\n", - " deploy_obj = run_kubectl_json(f\"get deploy -n {NAMESPACE} python-toolbox -o json\", check=False)\n", + "if can_run_toolbox(\"edge-toolbox status checks\"):\n", + " deploy_obj = run_kubectl_json(f\"get deploy -n {NAMESPACE} edge-toolbox -o json\", check=False)\n", " if deploy_obj:\n", " replicas = deploy_obj.get(\"spec\", {}).get(\"replicas\", 0)\n", " ready = deploy_obj.get(\"status\", {}).get(\"readyReplicas\", 0)\n", - " print(f\"python-toolbox deployment: present (spec.replicas={replicas}, readyReplicas={ready})\")\n", + " print(f\"edge-toolbox deployment: present (spec.replicas={replicas}, readyReplicas={ready})\")\n", " else:\n", - " print(\"python-toolbox deployment: not found\")\n", + " print(\"edge-toolbox deployment: not found\")\n", "\n", " pod_obj = run_kubectl_json(\n", - " f\"get pods -n {NAMESPACE} -l app.kubernetes.io/name=python-toolbox -o json\",\n", + " f\"get pods -n {NAMESPACE} -l app.kubernetes.io/name=edge-toolbox -o json\",\n", " check=False,\n", " )\n", " pod_items = pod_obj.get(\"items\", [])\n", " if not pod_items:\n", - " print(\"python-toolbox pods: none\")\n", + " print(\"edge-toolbox pods: none\")\n", " else:\n", " pod_names = [item.get(\"metadata\", {}).get(\"name\", \"\") for item in pod_items]\n", - " print(\"python-toolbox pods:\", \", \".join([name for name in pod_names if name]))\n" + " print(\"edge-toolbox pods:\", \", \".join([name for name in pod_names if name]))\n" ] }, { @@ -318,11 +318,11 @@ "id": "2cdb23b0", "metadata": {}, "source": [ - "## Manually Enable or Disable Python Toolbox\n", + "## Manually Enable or Disable the Go Edge Toolbox\n", "\n", - "Use `PYTHON_TOOLBOX_TOGGLE = \"on\"` or `\"off\"` below. The cell also accepts `yes`/`no`, `true`/`false`, and `1`/`0`.\n", + "Use `EDGE_TOOLBOX_TOGGLE = \"on\"` or `\"off\"` below. The cell also accepts `yes`/`no`, `true`/`false`, and `1`/`0`.\n", "\n", - "Keep `APPLY_PYTHON_TOOLBOX_TOGGLE = False` for preview mode. Set it to `True` only when you want the notebook to run Helm for you.\n" + "Keep `APPLY_EDGE_TOOLBOX_TOGGLE = False` for preview mode. Set it to `True` only when you want the notebook to run Helm for you.\n" ] }, { @@ -335,41 +335,41 @@ "# Manual toggle: accepts on/off, yes/no, true/false, or 1/0.\n", "import yaml\n", "\n", - "PYTHON_TOOLBOX_TOGGLE = \"on\"\n", - "APPLY_PYTHON_TOOLBOX_TOGGLE = False\n", + "EDGE_TOOLBOX_TOGGLE = \"on\"\n", + "APPLY_EDGE_TOOLBOX_TOGGLE = False\n", "WAIT_FOR_TOOLBOX_ROLLOUT = True\n", "TOOLBOX_ROLLOUT_TIMEOUT_SECONDS = 180\n", "\n", "values_file = PROJECT_ROOT / \"values.local-k3s.yaml\"\n", "toggle_values = yaml.safe_load(values_file.read_text()) or {} if values_file.exists() else {}\n", - "desired_toolbox_enabled = parse_manual_toggle(PYTHON_TOOLBOX_TOGGLE)\n", - "current_values_enabled = toggle_values.get(\"pythonToolbox\", {}).get(\"enabled\")\n", + "desired_toolbox_enabled = parse_manual_toggle(EDGE_TOOLBOX_TOGGLE)\n", + "current_values_enabled = toggle_values.get(\"edgeToolbox\", {}).get(\"enabled\")\n", "\n", - "print(f\"Current values.local-k3s.yaml pythonToolbox.enabled: {current_values_enabled}\")\n", + "print(f\"Current values.local-k3s.yaml edgeToolbox.enabled: {current_values_enabled}\")\n", "print(f\"Desired notebook toggle : {desired_toolbox_enabled}\")\n", "\n", "if not values_file.exists():\n", " print(f\"values.local-k3s.yaml not found at: {values_file}\")\n", - "elif can_run_helm(\"python-toolbox helm toggle\"):\n", + "elif can_run_helm(\"edge-toolbox helm toggle\"):\n", " helm_cmd = build_python_toolbox_helm_command(desired_toolbox_enabled, values_file)\n", " print(\"Helm command:\")\n", " print(helm_cmd)\n", "\n", - " if APPLY_PYTHON_TOOLBOX_TOGGLE:\n", + " if APPLY_EDGE_TOOLBOX_TOGGLE:\n", " run(helm_cmd, cwd=PROJECT_ROOT)\n", " if WAIT_FOR_TOOLBOX_ROLLOUT and desired_toolbox_enabled:\n", " run_kubectl(\n", - " f\"rollout status -n {NAMESPACE} deploy/python-toolbox --timeout={TOOLBOX_ROLLOUT_TIMEOUT_SECONDS}s\",\n", + " f\"rollout status -n {NAMESPACE} deploy/edge-toolbox --timeout={TOOLBOX_ROLLOUT_TIMEOUT_SECONDS}s\",\n", " check=False,\n", " )\n", " elif WAIT_FOR_TOOLBOX_ROLLOUT and not desired_toolbox_enabled:\n", - " deploy_check = run_kubectl(f\"get deploy -n {NAMESPACE} python-toolbox\", check=False, quiet=True)\n", + " deploy_check = run_kubectl(f\"get deploy -n {NAMESPACE} edge-toolbox\", check=False, quiet=True)\n", " if deploy_check.returncode != 0:\n", - " print(\"python-toolbox deployment is absent after disable request.\")\n", + " print(\"edge-toolbox deployment is absent after disable request.\")\n", " else:\n", - " print(\"python-toolbox deployment still exists. Re-run the status cell above to inspect current state.\")\n", + " print(\"edge-toolbox deployment still exists. Re-run the status cell above to inspect current state.\")\n", " else:\n", - " print(\"Preview mode only. Set APPLY_PYTHON_TOOLBOX_TOGGLE = True to execute Helm from this notebook.\")\n" + " print(\"Preview mode only. Set APPLY_EDGE_TOOLBOX_TOGGLE = True to execute Helm from this notebook.\")\n" ] }, { @@ -379,32 +379,26 @@ "metadata": {}, "outputs": [], "source": [ - "TOOLBOX_EXAMPLE_DIR = \"/workspace/examples\"\n", - "EXAMPLE_SCRIPTS = [\"opentelemetry_healthcheck.py\", \"ollama_smoke.py\"]\n", + "TOOLBOX_COMMANDS = [\n", + " \"edge-toolbox dns ollama ollama-gateway open-webui open-webui-redis\",\n", + " \"edge-toolbox http http://ollama:11434/api/tags\",\n", + " \"edge-toolbox redis-ping\",\n", + " \"edge-toolbox ollama-smoke\",\n", + "]\n", "running_pods = list_running_toolbox_pods()\n", "pod_name = running_pods[0] if running_pods else \"\"\n", "\n", "if pod_name:\n", " print(\"Toolbox pod:\", pod_name)\n", - " ls_proc = run_kubectl(\n", - " f\"exec -n {NAMESPACE} {pod_name} -- sh -lc {shlex.quote(f'ls -1 {TOOLBOX_EXAMPLE_DIR} 2>/dev/null')}\",\n", - " check=False,\n", - " )\n", - " available_scripts = {line.strip() for line in ls_proc.stdout.splitlines() if line.strip()}\n", - " if available_scripts:\n", - " print(\"Available toolbox scripts:\", \", \".join(sorted(available_scripts)))\n", - "\n", - " for script in EXAMPLE_SCRIPTS:\n", - " if available_scripts and script not in available_scripts:\n", - " print(f\"Skipping {script}: not found in deployed image. Rebuild/import python-toolbox if needed.\")\n", - " continue\n", + " for command in TOOLBOX_COMMANDS:\n", + " print(\"Running:\", command)\n", " run_kubectl(\n", - " f\"exec -n {NAMESPACE} {pod_name} -- python {TOOLBOX_EXAMPLE_DIR}/{script}\",\n", + " f\"exec -n {NAMESPACE} {pod_name} -- {command}\",\n", " check=False,\n", " )\n", "else:\n", " print(\"No running toolbox pod found.\")\n", - " print(\"Set PYTHON_TOOLBOX_TOGGLE = 'on' and APPLY_PYTHON_TOOLBOX_TOGGLE = True in the previous cell, then re-run this cell.\")\n" + " print(\"Set EDGE_TOOLBOX_TOGGLE = 'on' and APPLY_EDGE_TOOLBOX_TOGGLE = True in the previous cell, then re-run this cell.\")\n" ] }, { @@ -481,7 +475,7 @@ "import requests\n", "import time\n", "\n", - "LANGCHAIN_PROXY_PORT_FORWARD_CMD = \"kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000\"\n", + "LANGCHAIN_PROXY_PORT_FORWARD_CMD = \"kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000\"\n", "PROXY_HEALTH_URL = \"http://localhost:8000/healthz\"\n", "PROXY_CHAT_URL = \"http://localhost:8000/ollama/api/chat\"\n", "\n", @@ -502,7 +496,7 @@ " proxy_ready = False\n", "\n", "if not proxy_ready:\n", - " print(\"Skipping GPU correlation inference: langchain-demo proxy is not reachable on localhost:8000\")\n", + " print(\"Skipping GPU correlation inference: ollama-gateway proxy is not reachable on localhost:8000\")\n", " print(\"In another terminal run:\")\n", " print(f\" {LANGCHAIN_PROXY_PORT_FORWARD_CMD}\")\n", "else:\n", diff --git a/jupyter-notebooks/08-troubleshooting-etcd-simulations.ipynb b/jupyter-notebooks/08-troubleshooting-etcd-simulations.ipynb index 51c625f..cd9a956 100644 --- a/jupyter-notebooks/08-troubleshooting-etcd-simulations.ipynb +++ b/jupyter-notebooks/08-troubleshooting-etcd-simulations.ipynb @@ -237,8 +237,8 @@ "values = yaml.safe_load((PROJECT_ROOT / \"values.local-k3s.yaml\").read_text())\n", "suggestions = []\n", "\n", - "if values.get(\"pythonToolbox\", {}).get(\"enabled\"):\n", - " suggestions.append(\"Disable pythonToolbox when not debugging: --set pythonToolbox.enabled=false\")\n", + "if values.get(\"edgeToolbox\", {}).get(\"enabled\"):\n", + " suggestions.append(\"Disable edgeToolbox when not debugging: --set edgeToolbox.enabled=false\")\n", "if values.get(\"opentelemetryDashboardSeeder\", {}).get(\"enabled\"):\n", " suggestions.append(\"Disable continuous seeding if not needed: --set opentelemetryDashboardSeeder.enabled=false\")\n", "if values.get(\"etcd\", {}).get(\"enabled\"):\n", @@ -246,8 +246,8 @@ "\n", "if values.get(\"ollama\", {}).get(\"service\", {}).get(\"type\") == \"LoadBalancer\":\n", " suggestions.append(\"Set ollama service to ClusterIP in local profile to reduce svclb overhead\")\n", - "if values.get(\"langchainDemo\", {}).get(\"service\", {}).get(\"type\") == \"LoadBalancer\":\n", - " suggestions.append(\"Set langchain-demo service to ClusterIP to remove extra svclb pod\")\n", + "if values.get(\"ollamaGateway\", {}).get(\"service\", {}).get(\"type\") == \"LoadBalancer\":\n", + " suggestions.append(\"Set ollama-gateway service to ClusterIP to remove extra svclb pod\")\n", "\n", "if not suggestions:\n", " suggestions.append(\"Current values already follow a low-footprint local profile.\")\n", @@ -322,7 +322,7 @@ "commands = [\n", " \"kubectl get pods -A\",\n", " \"kubectl get svc -n llm-observability\",\n", - " \"kubectl logs -n llm-observability deploy/langchain-demo --tail=50\",\n", + " \"kubectl logs -n llm-observability deploy/ollama-gateway --tail=50\",\n", " \"nvidia-smi\",\n", "]\n", "for cmd in commands:\n", diff --git a/jupyter-notebooks/09-k3s-networking-deep-dive.ipynb b/jupyter-notebooks/09-k3s-networking-deep-dive.ipynb index dbafd67..259a096 100644 --- a/jupyter-notebooks/09-k3s-networking-deep-dive.ipynb +++ b/jupyter-notebooks/09-k3s-networking-deep-dive.ipynb @@ -14,7 +14,7 @@ "- node, pod, Service, Endpoints, and EndpointSlice inventory\n", "- service selector -> pod -> endpoint path mapping for the LLM stack\n", "- k3s `LoadBalancer` internals via `svclb-*` pods in `kube-system`\n", - "- DNS names and in-cluster TCP/HTTP reachability from `python-toolbox`\n", + "- DNS names and in-cluster TCP/HTTP reachability from `edge-toolbox`\n", "- an optional watch loop for live EndpointSlice changes\n" ] }, @@ -72,10 +72,10 @@ "from kubernetes.stream import stream\n", "\n", "NAMESPACE = os.getenv(\"LLM_NAMESPACE\", \"llm-observability\")\n", - "TARGET_SERVICES = [\"langchain-demo\", \"ollama\", \"open-webui\", \"redis\"]\n", + "TARGET_SERVICES = [\"ollama-gateway\", \"ollama\", \"open-webui\", \"redis\"]\n", "CURRENT_WORKDIR = Path.cwd().resolve()\n", "PROBE_TARGETS = [\n", - " {\"name\": \"langchain-demo\", \"host\": f\"langchain-demo.{NAMESPACE}.svc.cluster.local\", \"port\": 8000, \"mode\": \"http\", \"path\": \"/healthz\"},\n", + " {\"name\": \"ollama-gateway\", \"host\": f\"ollama-gateway.{NAMESPACE}.svc.cluster.local\", \"port\": 8000, \"mode\": \"http\", \"path\": \"/healthz\"},\n", " {\"name\": \"ollama\", \"host\": f\"ollama.{NAMESPACE}.svc.cluster.local\", \"port\": 11434, \"mode\": \"http\", \"path\": \"/api/tags\"},\n", " {\"name\": \"open-webui\", \"host\": f\"open-webui.{NAMESPACE}.svc.cluster.local\", \"port\": 8080, \"mode\": \"http\", \"path\": \"/health\"},\n", " {\"name\": \"redis\", \"host\": f\"redis.{NAMESPACE}.svc.cluster.local\", \"port\": 6379, \"mode\": \"tcp\", \"path\": \"\"},\n", @@ -385,7 +385,7 @@ "source": [ "## Application Flow Interpretation\n", "\n", - "The Kubernetes API shows the network plumbing. This cell adds the higher-level application path used by this stack: `open-webui` receives north-south traffic, then talks east-west to `langchain-demo`, which proxies to `ollama`. `redis` provides local state inside the namespace.\n" + "The Kubernetes API shows the network plumbing. This cell adds the higher-level application path used by this stack: `open-webui` receives north-south traffic, then talks east-west to `ollama-gateway`, which proxies to `ollama`. `redis` provides local state inside the namespace.\n" ] }, { @@ -398,9 +398,9 @@ "flow_rows = [\n", " {\"from\": \"browser or host\", \"to\": \"open-webui service\", \"transport\": \"HTTP\", \"details\": \"LoadBalancer / ServiceLB entrypoint\"},\n", " {\"from\": \"open-webui service\", \"to\": \"open-webui pod\", \"transport\": \"ClusterIP -> pod IP\", \"details\": \"Service selector resolves to StatefulSet pod\"},\n", - " {\"from\": \"open-webui pod\", \"to\": \"langchain-demo service\", \"transport\": \"HTTP\", \"details\": \"Ollama-compatible proxy path inside cluster\"},\n", - " {\"from\": \"langchain-demo service\", \"to\": \"langchain-demo pod\", \"transport\": \"ClusterIP -> pod IP\", \"details\": \"Deployment-backed proxy\"},\n", - " {\"from\": \"langchain-demo pod\", \"to\": \"ollama service\", \"transport\": \"HTTP\", \"details\": \"Inference request forwarded to local model runtime\"},\n", + " {\"from\": \"open-webui pod\", \"to\": \"ollama-gateway service\", \"transport\": \"HTTP\", \"details\": \"Ollama-compatible proxy path inside cluster\"},\n", + " {\"from\": \"ollama-gateway service\", \"to\": \"ollama-gateway pod\", \"transport\": \"ClusterIP -> pod IP\", \"details\": \"Deployment-backed proxy\"},\n", + " {\"from\": \"ollama-gateway pod\", \"to\": \"ollama service\", \"transport\": \"HTTP\", \"details\": \"Inference request forwarded to local model runtime\"},\n", " {\"from\": \"ollama service\", \"to\": \"ollama pod\", \"transport\": \"ClusterIP -> pod IP\", \"details\": \"Deployment-backed model server\"},\n", " {\"from\": \"open-webui pod\", \"to\": \"redis service\", \"transport\": \"TCP\", \"details\": \"Internal cache / state path\"},\n", "]\n", @@ -513,8 +513,8 @@ " graph.add_edge(f\"node:{row['svclb_node']}\", svclb_node, relation=\"hosts\")\n", "\n", "logical_edges = [\n", - " (\"svc:open-webui\", \"svc:langchain-demo\", \"app hop\"),\n", - " (\"svc:langchain-demo\", \"svc:ollama\", \"app hop\"),\n", + " (\"svc:open-webui\", \"svc:ollama-gateway\", \"app hop\"),\n", + " (\"svc:ollama-gateway\", \"svc:ollama\", \"app hop\"),\n", " (\"svc:open-webui\", \"svc:redis\", \"app hop\"),\n", "]\n", "for source, target, relation in logical_edges:\n", @@ -544,7 +544,7 @@ "source": [ "## In-Cluster DNS and Connectivity Probe\n", "\n", - "This cell execs into the running `python-toolbox` pod by using the Kubernetes streaming API. It resolves Service DNS names and performs TCP or HTTP checks from inside the cluster network namespace, which is the cleanest way to prove east-west connectivity without shelling out to `kubectl`.\n" + "This cell execs into the running `edge-toolbox` pod by using the Kubernetes streaming API. It resolves Service DNS names and performs TCP or HTTP checks from inside the cluster network namespace, which is the cleanest way to prove east-west connectivity without shelling out to `kubectl`.\n" ] }, { @@ -554,11 +554,11 @@ "metadata": {}, "outputs": [], "source": [ - "toolbox_pods = core.list_namespaced_pod(NAMESPACE, label_selector=\"app.kubernetes.io/name=python-toolbox\").items\n", + "toolbox_pods = core.list_namespaced_pod(NAMESPACE, label_selector=\"app.kubernetes.io/name=edge-toolbox\").items\n", "toolbox_pod = next((pod for pod in toolbox_pods if pod.status.phase == \"Running\"), None)\n", "\n", "if toolbox_pod is None:\n", - " raise RuntimeError(\"No running python-toolbox pod found. Keep pythonToolbox.enabled=true and re-run this cell.\")\n", + " raise RuntimeError(\"No running edge-toolbox pod found. Keep edgeToolbox.enabled=true and re-run this cell.\")\n", "\n", "toolbox_container = toolbox_pod.spec.containers[0].name\n", "probe_targets_json = json.dumps(PROBE_TARGETS, indent=2)\n", @@ -620,7 +620,7 @@ "\n", " cleaned = (text or \"\").strip()\n", " if not cleaned:\n", - " raise ValueError(\"python-toolbox probe returned empty output\")\n", + " raise ValueError(\"edge-toolbox probe returned empty output\")\n", "\n", " candidates = [cleaned]\n", " list_start = cleaned.find(\"[\")\n", @@ -648,7 +648,7 @@ " preview = cleaned[:500]\n", " joined_errors = \" | \".join(errors[:4])\n", " raise ValueError(\n", - " \"Unable to parse python-toolbox probe output. \"\n", + " \"Unable to parse edge-toolbox probe output. \"\n", " f\"Raw preview: {preview!r}. Errors: {joined_errors}\"\n", " )\n", "\n", diff --git a/jupyter-notebooks/10-k3s-architecture-diagrams.ipynb b/jupyter-notebooks/10-k3s-architecture-diagrams.ipynb index 63cf8b8..063daad 100644 --- a/jupyter-notebooks/10-k3s-architecture-diagrams.ipynb +++ b/jupyter-notebooks/10-k3s-architecture-diagrams.ipynb @@ -547,11 +547,11 @@ "# %%\n", "def build_app_flow_graph() -> nx.DiGraph:\n", " G = nx.DiGraph()\n", - " services = [\"open-webui\", \"langchain-demo\", \"ollama\", \"redis\"]\n", + " services = [\"open-webui\", \"ollama-gateway\", \"ollama\", \"redis\"]\n", " for svc in services:\n", " G.add_node(svc, kind=\"service\", label=svc)\n", - " G.add_edge(\"open-webui\", \"langchain-demo\", relation=\"HTTP\")\n", - " G.add_edge(\"langchain-demo\", \"ollama\", relation=\"HTTP\")\n", + " G.add_edge(\"open-webui\", \"ollama-gateway\", relation=\"HTTP\")\n", + " G.add_edge(\"ollama-gateway\", \"ollama\", relation=\"HTTP\")\n", " G.add_edge(\"open-webui\", \"redis\", relation=\"TCP\")\n", " G.add_node(\"Browser / Host\", kind=\"external\", label=\"Browser / Host\")\n", " G.add_edge(\"Browser / Host\", \"open-webui\", relation=\"LoadBalancer (8080)\")\n", diff --git a/jupyter-notebooks/CATALOG.md b/jupyter-notebooks/CATALOG.md index 849e160..e0dd7da 100644 --- a/jupyter-notebooks/CATALOG.md +++ b/jupyter-notebooks/CATALOG.md @@ -8,11 +8,11 @@ These notebooks form the main guided path for the current stack: 1. `01-environment-smoke-test.ipynb` 2. `02-ollama-api-basics.ipynb` -3. `03-langchain-proxy-deep-dive.ipynb` +3. `03-ollama-gateway-deep-dive.ipynb` 4. `04-opentelemetry-tracing-setup.ipynb` 5. `05-open-webui-end-to-end.ipynb` 6. `06-custom-modelfile-workflow.ipynb` -7. `07-python-toolbox-diagnostics.ipynb` +7. `07-edge-toolbox-diagnostics.ipynb` 8. `08-troubleshooting-etcd-simulations.ipynb` 9. `09-k3s-networking-deep-dive.ipynb` 10. `10-k3s-architecture-diagrams.ipynb` @@ -45,7 +45,7 @@ Notebook checkpoint files under `.ipynb_checkpoints/` are also local-only and sh ## Prerequisite Notes - Notebook kernels should use Python 3.11. -- Several notebooks require `kubectl port-forward` to `ollama`, `langchain-demo`, or `open-webui`. -- `07` and `09` expect `pythonToolbox.enabled=true` in the running release or active local values. +- Several notebooks require `kubectl port-forward` to `ollama`, `ollama-gateway`, or `open-webui`. +- `07` and `09` expect `edgeToolbox.enabled=true` in the running release or active local values. Use [README.md](README.md) for launch instructions and [../docs/NOTEBOOKS-GUIDE.md](../docs/NOTEBOOKS-GUIDE.md) for deeper execution guidance. diff --git a/jupyter-notebooks/README.md b/jupyter-notebooks/README.md index 1e729f4..4ca3165 100644 --- a/jupyter-notebooks/README.md +++ b/jupyter-notebooks/README.md @@ -12,19 +12,19 @@ For classification and status, see [CATALOG.md](CATALOG.md). 2. **02-ollama-api-basics.ipynb** Covers direct Ollama API usage (`/api/tags`, `/api/chat`, streaming mode), multi-prompt benchmarking, and latency charting. -3. **03-langchain-proxy-deep-dive.ipynb** - Explores `langchain-demo` proxy endpoints (`/healthz`, `/config`, `/invoke`, `/ollama/api/*`) and compares direct vs proxy latency. +3. **03-ollama-gateway-deep-dive.ipynb** + Explores `ollama-gateway` proxy endpoints (`/healthz`, `/config`, `/invoke`, `/ollama/api/*`) and compares direct vs proxy latency. 4. **04-opentelemetry-tracing-setup.ipynb** Validates OpenTelemetry credentials, generates traced inference calls, queries runs from your project, and plots observability metrics. 5. **05-open-webui-end-to-end.ipynb** - Validates Browser -> Open WebUI -> LangChain proxy -> Ollama -> OpenTelemetry flow with manual browser prompts plus post-run trace analysis. + Validates Browser -> Open WebUI -> Ollama Go gateway -> Ollama -> OpenTelemetry flow with manual browser prompts plus post-run trace analysis. 6. **06-custom-modelfile-workflow.ipynb** Walks through GGUF-backed Modelfile inspection, tuning, optional model creation via Ollama API, and benchmark comparison. -7. **07-python-toolbox-diagnostics.ipynb** +7. **07-edge-toolbox-diagnostics.ipynb** Covers toolbox deployment checks, in-cluster diagnostic script execution, pod memory profiling, and GPU correlation during inference. 8. **08-troubleshooting-etcd-simulations.ipynb** @@ -76,7 +76,7 @@ For notebooks that call internal APIs from the host, keep these ready in separat ```bash kubectl port-forward -n llm-observability svc/ollama 11434:11434 -kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 +kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000 ``` ## Port-Forward Expectations @@ -85,15 +85,15 @@ Several notebooks assume local access to internal `ClusterIP` services. Keep the ```bash kubectl port-forward -n llm-observability svc/ollama 11434:11434 -kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 +kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000 ``` Notebook-specific expectations: - `02`, `03`, `04`, and `06` need `localhost:11434` and/or `localhost:8000` - `05` assumes browser access to Open WebUI on `localhost:8080` -- `07` uses `python-toolbox` in-cluster plus optional `localhost:8000` GPU-correlation checks -- `09` is mostly API-first and uses the Kubernetes Python client plus `python-toolbox` +- `07` uses `edge-toolbox` in-cluster plus optional `localhost:8000` GPU-correlation checks +- `09` is mostly API-first and uses the Kubernetes Python client plus `edge-toolbox` ## Portability Notes diff --git a/jupyter-notebooks/llm-observability-in-action/BOOK_COVERAGE.md b/jupyter-notebooks/llm-observability-in-action/BOOK_COVERAGE.md index 7c730b4..bab5ddc 100644 --- a/jupyter-notebooks/llm-observability-in-action/BOOK_COVERAGE.md +++ b/jupyter-notebooks/llm-observability-in-action/BOOK_COVERAGE.md @@ -22,11 +22,10 @@ This toolkit maps the source domains from a local `kubernetes-in-action-2nd-edit - Services, DNS, endpoints, ingress, gateway API, traffic rules (Ch. 11, 12, 13): - `kubectl/06_networking_core.sh` - `kubectl/07_networking_advanced.sh` - - `python/02_service_path_inspector.py` - - `python/04_networking_report.py` + - `runbooks/run_go_suite.sh` (`network` and `service-path` commands) - Controllers and rollout management (Ch. 14, 15, 16, 17): - `kubectl/02_namespaces_and_workloads.sh` - - `python/03_workload_health.py` + - `runbooks/run_go_suite.sh` (`status` and `validate` commands) - Batch workloads (Ch. 18): - `kubectl/09_jobs_and_batch.sh` - LLM stack-specific health integration: diff --git a/jupyter-notebooks/llm-observability-in-action/KUBECTL_COMMAND_CATALOG.md b/jupyter-notebooks/llm-observability-in-action/KUBECTL_COMMAND_CATALOG.md index a5b9b2f..ab6d7e4 100644 --- a/jupyter-notebooks/llm-observability-in-action/KUBECTL_COMMAND_CATALOG.md +++ b/jupyter-notebooks/llm-observability-in-action/KUBECTL_COMMAND_CATALOG.md @@ -28,7 +28,7 @@ kubectl -n llm-observability get deploy,statefulset -o name kubectl get ns --show-labels kubectl -n llm-observability get all kubectl -n llm-observability get deploy,rs,sts,ds,pod,job,cronjob -o wide -kubectl -n llm-observability describe deploy/langchain-demo +kubectl -n llm-observability describe deploy/ollama-gateway kubectl -n llm-observability rollout status deployment/ollama kubectl -n llm-observability rollout status statefulset/open-webui ``` @@ -77,7 +77,7 @@ kubectl -n llm-observability get ingress kubectl -n llm-observability get networkpolicy kubectl -n llm-observability port-forward svc/open-webui 8080:8080 kubectl -n llm-observability port-forward svc/ollama 11434:11434 -kubectl -n llm-observability port-forward svc/langchain-demo 8000:8000 +kubectl -n llm-observability port-forward svc/ollama-gateway 8000:8000 ``` ## Networking advanced / gateway API @@ -125,6 +125,6 @@ kubectl taint nodes key=value:NoSchedule kubectl -n llm-observability apply -f file.yaml kubectl -n llm-observability delete -f file.yaml kubectl -n llm-observability patch deploy/open-webui --type merge -p '{"spec":{"replicas":1}}' -kubectl -n llm-observability set image deploy/langchain-demo langchain-demo=langchain-demo:0.1.1 -kubectl -n llm-observability scale deploy/langchain-demo --replicas=1 +kubectl -n llm-observability set image deploy/ollama-gateway ollama-gateway=ollama-gateway:0.2.0 +kubectl -n llm-observability scale deploy/ollama-gateway --replicas=1 ``` diff --git a/jupyter-notebooks/llm-observability-in-action/README.md b/jupyter-notebooks/llm-observability-in-action/README.md index 14c4bb7..906e446 100644 --- a/jupyter-notebooks/llm-observability-in-action/README.md +++ b/jupyter-notebooks/llm-observability-in-action/README.md @@ -7,23 +7,20 @@ This folder provides a practical Kubernetes operations toolkit for your local `l - `config.env.example` - base runtime variables for your local k3s namespace and services. - `lib/common.sh` - shared shell helpers used by all kubectl scripts. - `kubectl/` - executable shell scripts grouped by Kubernetes domain. -- `python/` - Kubernetes Python client scripts for inventory, health, and networking diagnostics. - `manifests/networking/` - optional networking manifests (NetworkPolicy and test pod). -- `runbooks/` - wrappers to run full read-only kubectl and Python suites. +- `runbooks/` - wrappers to run the read-only kubectl and native Go suites. ## Quick start ```bash PROJECT_ROOT="${PROJECT_ROOT:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}" -PYTHON_BIN="${PYTHON_BIN:-python3.11}" cd "${PROJECT_ROOT}/jupyter-notebooks/llm-observability-in-action" cp config.env.example config.env # Optional: set K8S_CONTEXT in config.env if multiple contexts exist. ./runbooks/run_kubectl_suite_readonly.sh -"${PYTHON_BIN}" -m pip install -r python/requirements.txt -./runbooks/run_python_suite.sh +./runbooks/run_go_suite.sh ``` ## Mutation safety model @@ -50,21 +47,19 @@ APPLY_CHANGES=1 ./kubectl/04_configmaps_and_secrets.sh 7. `07_networking_advanced.sh` - Gateway API, DNS/CNI controls, node-network context. 8. `08_security_policy.sh` - ServiceAccounts/RBAC/auth checks/pod security labels. 9. `09_jobs_and_batch.sh` - Jobs/CronJobs execution and diagnostics. -10. `10_llm_observability_stack_checks.sh` - stack-specific checks for `open-webui`, `ollama`, `langchain-demo`. +10. `10_llm_observability_stack_checks.sh` - stack-specific checks for `open-webui`, `ollama`, `ollama-gateway`. -## Script index (python) +## Native Go operations -1. `01_namespace_inventory.py` - broad namespace inventory (workloads + network + config + storage). -2. `02_service_path_inspector.py` - service -> pods -> endpoints -> routes mapping. -3. `03_workload_health.py` - rollout/availability health checks with non-zero exit on issues. -4. `04_networking_report.py` - service selector mapping + endpoint graph report. -5. `05_watch_events.py` - real-time namespace event stream watcher. +`runbooks/run_go_suite.sh` uses the repository CLI for release validation, +namespace inventory, and Service-to-Pod/Endpoint tracing. No Python dependency +is required for these operations. ## Networking manifests - `manifests/networking/netpol.default-deny.yaml` - `manifests/networking/netpol.allow-dns.yaml` -- `manifests/networking/netpol.allow-openwebui-to-langchain.yaml` +- `manifests/networking/netpol.allow-openwebui-to-ollama-gateway.yaml` - `manifests/networking/test-client-pod.yaml` Apply only when intentionally testing networking behavior: diff --git a/jupyter-notebooks/llm-observability-in-action/config.env.example b/jupyter-notebooks/llm-observability-in-action/config.env.example index 5115514..9b9bf48 100644 --- a/jupyter-notebooks/llm-observability-in-action/config.env.example +++ b/jupyter-notebooks/llm-observability-in-action/config.env.example @@ -12,8 +12,8 @@ K8S_RELEASE=llm-observability-stack # Expected service/deployment names used by checks. OPENWEBUI_SERVICE=open-webui OLLAMA_SERVICE=ollama -LANGCHAIN_SERVICE=langchain-demo -PYTHON_TOOLBOX_DEPLOYMENT=python-toolbox +OLLAMA_GATEWAY_SERVICE=ollama-gateway +EDGE_TOOLBOX_DEPLOYMENT=edge-toolbox # 0 = read-only mode (default), 1 = allow mutating kubectl verbs. APPLY_CHANGES=0 diff --git a/jupyter-notebooks/llm-observability-in-action/kubectl/02_namespaces_and_workloads.sh b/jupyter-notebooks/llm-observability-in-action/kubectl/02_namespaces_and_workloads.sh index 882faf8..51c570a 100755 --- a/jupyter-notebooks/llm-observability-in-action/kubectl/02_namespaces_and_workloads.sh +++ b/jupyter-notebooks/llm-observability-in-action/kubectl/02_namespaces_and_workloads.sh @@ -32,9 +32,9 @@ while IFS= read -r sts_name; do done < <(kctl_ns_capture get sts -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true) log_section "llm-observability key workloads" -if resource_exists_ns "deploy/${LANGCHAIN_SERVICE}"; then - kctl_ns_try get "deploy/${LANGCHAIN_SERVICE}" -o wide - kctl_ns_try describe "deploy/${LANGCHAIN_SERVICE}" +if resource_exists_ns "deploy/${OLLAMA_GATEWAY_SERVICE}"; then + kctl_ns_try get "deploy/${OLLAMA_GATEWAY_SERVICE}" -o wide + kctl_ns_try describe "deploy/${OLLAMA_GATEWAY_SERVICE}" fi if resource_exists_ns "deploy/${OPENWEBUI_SERVICE}"; then diff --git a/jupyter-notebooks/llm-observability-in-action/kubectl/03_pods_lifecycle_debug.sh b/jupyter-notebooks/llm-observability-in-action/kubectl/03_pods_lifecycle_debug.sh index 8413c28..297bdc9 100755 --- a/jupyter-notebooks/llm-observability-in-action/kubectl/03_pods_lifecycle_debug.sh +++ b/jupyter-notebooks/llm-observability-in-action/kubectl/03_pods_lifecycle_debug.sh @@ -24,7 +24,7 @@ else fi log_section "Recent logs (key workloads)" -kctl_ns_try logs "deploy/${LANGCHAIN_SERVICE}" --all-containers=true --tail=120 +kctl_ns_try logs "deploy/${OLLAMA_GATEWAY_SERVICE}" --all-containers=true --tail=120 if resource_exists_ns "deploy/${OPENWEBUI_SERVICE}"; then kctl_ns_try logs "deploy/${OPENWEBUI_SERVICE}" --all-containers=true --tail=120 elif resource_exists_ns "sts/${OPENWEBUI_SERVICE}"; then diff --git a/jupyter-notebooks/llm-observability-in-action/kubectl/04_configmaps_and_secrets.sh b/jupyter-notebooks/llm-observability-in-action/kubectl/04_configmaps_and_secrets.sh index 571e2fa..c221176 100755 --- a/jupyter-notebooks/llm-observability-in-action/kubectl/04_configmaps_and_secrets.sh +++ b/jupyter-notebooks/llm-observability-in-action/kubectl/04_configmaps_and_secrets.sh @@ -11,7 +11,7 @@ print_runtime_context log_section "ConfigMaps" kctl_ns_try get configmap -o wide kctl_ns_try describe configmap ollama-local-modelfile -kctl_ns_try describe configmap langchain-demo-app +kctl_ns_try describe configmap ollama-gateway-app log_section "Secrets" kctl_ns_try get secret -o custom-columns=NAME:.metadata.name,TYPE:.type,AGE:.metadata.creationTimestamp diff --git a/jupyter-notebooks/llm-observability-in-action/kubectl/06_networking_core.sh b/jupyter-notebooks/llm-observability-in-action/kubectl/06_networking_core.sh index ba88c6b..f4b301d 100755 --- a/jupyter-notebooks/llm-observability-in-action/kubectl/06_networking_core.sh +++ b/jupyter-notebooks/llm-observability-in-action/kubectl/06_networking_core.sh @@ -20,7 +20,7 @@ kctl_ns_try get ingress -o wide kctl_ns_try get networkpolicy -o wide log_section "llm-observability service deep dive" -for svc in "${OPENWEBUI_SERVICE}" "${OLLAMA_SERVICE}" "${LANGCHAIN_SERVICE}"; do +for svc in "${OPENWEBUI_SERVICE}" "${OLLAMA_SERVICE}" "${OLLAMA_GATEWAY_SERVICE}"; do kctl_ns_try get "svc/${svc}" -o wide kctl_ns_try describe "svc/${svc}" kctl_ns_try get "endpoints/${svc}" -o wide @@ -28,7 +28,7 @@ for svc in "${OPENWEBUI_SERVICE}" "${OLLAMA_SERVICE}" "${LANGCHAIN_SERVICE}"; do done log_section "DNS names" -for svc in "${OPENWEBUI_SERVICE}" "${OLLAMA_SERVICE}" "${LANGCHAIN_SERVICE}"; do +for svc in "${OPENWEBUI_SERVICE}" "${OLLAMA_SERVICE}" "${OLLAMA_GATEWAY_SERVICE}"; do printf '%s\n' "${svc}.${K8S_NAMESPACE}.svc.cluster.local" done @@ -36,5 +36,5 @@ log_section "Port-forward helpers" cat < argparse.Namespace: - parser = argparse.ArgumentParser(description="Namespace inventory") - parser.add_argument("--namespace", default="llm-observability", help="Namespace to inspect") - parser.add_argument("--context", default=None, help="Kubeconfig context") - parser.add_argument("--kubeconfig", default=None, help="Kubeconfig path") - parser.add_argument("--in-cluster", action="store_true", help="Use in-cluster config") - return parser.parse_args() - - -def load_cfg(args: argparse.Namespace) -> None: - if args.in_cluster: - config.load_incluster_config() - else: - config.load_kube_config(config_file=args.kubeconfig, context=args.context) - - -def table(title: str, headers: List[str], rows: List[List[str]]) -> None: - print(f"\n=== {title} ===") - if not rows: - print("") - return - widths = [len(h) for h in headers] - for row in rows: - for i, v in enumerate(row): - widths[i] = max(widths[i], len(str(v))) - - def fmt(r: Iterable[str]) -> str: - return " | ".join(str(v).ljust(widths[i]) for i, v in enumerate(r)) - - print(fmt(headers)) - print("-+-".join("-" * w for w in widths)) - for row in rows: - print(fmt(row)) - - -def fmt_selector(selector: dict | None) -> str: - if not selector: - return "-" - return ",".join(f"{k}={v}" for k, v in sorted(selector.items())) - - -def fmt_ports(ports: Iterable[object] | None) -> str: - if not ports: - return "-" - out: List[str] = [] - for p in ports: - name = getattr(p, "name", "") or "" - port = getattr(p, "port", "") - target = getattr(p, "target_port", None) - proto = getattr(p, "protocol", "TCP") - left = f"{name}:{port}" if name else str(port) - if target is None: - out.append(f"{left}/{proto}") - else: - out.append(f"{left}->{target}/{proto}") - return ", ".join(out) - - -def summarize(namespace: str) -> int: - core = client.CoreV1Api() - apps = client.AppsV1Api() - batch = client.BatchV1Api() - net = client.NetworkingV1Api() - discovery = client.DiscoveryV1Api() - - try: - pods = core.list_namespaced_pod(namespace).items - services = core.list_namespaced_service(namespace).items - endpoints = core.list_namespaced_endpoints(namespace).items - endpoint_slices = discovery.list_namespaced_endpoint_slice(namespace).items - configmaps = core.list_namespaced_config_map(namespace).items - secrets = core.list_namespaced_secret(namespace).items - pvcs = core.list_namespaced_persistent_volume_claim(namespace).items - deployments = apps.list_namespaced_deployment(namespace).items - statefulsets = apps.list_namespaced_stateful_set(namespace).items - daemonsets = apps.list_namespaced_daemon_set(namespace).items - jobs = batch.list_namespaced_job(namespace).items - cronjobs = batch.list_namespaced_cron_job(namespace).items - ingresses = net.list_namespaced_ingress(namespace).items - netpols = net.list_namespaced_network_policy(namespace).items - except ApiException as exc: - print(f"API error: {exc.status} {exc.reason}", file=sys.stderr) - print(exc.body or "", file=sys.stderr) - return 2 - - pod_rows: List[List[str]] = [] - for p in pods: - ready = sum(1 for cs in (p.status.container_statuses or []) if cs.ready) - total = len(p.status.container_statuses or []) - restarts = sum((cs.restart_count or 0) for cs in (p.status.container_statuses or [])) - pod_rows.append( - [ - p.metadata.name, - p.status.phase or "", - f"{ready}/{total}", - str(restarts), - p.status.pod_ip or "", - p.spec.node_name or "", - ] - ) - - svc_rows: List[List[str]] = [] - for s in services: - svc_rows.append( - [ - s.metadata.name, - s.spec.type or "", - s.spec.cluster_ip or "", - fmt_ports(s.spec.ports), - fmt_selector(s.spec.selector), - ] - ) - - ep_rows: List[List[str]] = [] - for ep in endpoints: - addresses: List[str] = [] - ports: List[str] = [] - for subset in ep.subsets or []: - addresses.extend([a.ip for a in (subset.addresses or [])]) - ports.extend([f"{p.name or ''}:{p.port}/{p.protocol}" for p in (subset.ports or [])]) - ep_rows.append([ep.metadata.name, ",".join(addresses) if addresses else "-", ",".join(ports) if ports else "-"]) - - eps_rows: List[List[str]] = [] - for eps in endpoint_slices: - svc_name = (eps.metadata.labels or {}).get("kubernetes.io/service-name", "-") - addresses: List[str] = [] - for e in eps.endpoints or []: - addresses.extend(e.addresses or []) - ports = [f"{p.name or ''}:{p.port}/{p.protocol}" for p in (eps.ports or [])] - eps_rows.append([eps.metadata.name, svc_name, ",".join(addresses) if addresses else "-", ",".join(ports) if ports else "-"]) - - wl_rows: List[List[str]] = [] - for d in deployments: - wl_rows.append(["Deployment", d.metadata.name, str(d.spec.replicas or 0), str(d.status.available_replicas or 0)]) - for s in statefulsets: - wl_rows.append(["StatefulSet", s.metadata.name, str(s.spec.replicas or 0), str(s.status.ready_replicas or 0)]) - for ds in daemonsets: - wl_rows.append(["DaemonSet", ds.metadata.name, str(ds.status.desired_number_scheduled or 0), str(ds.status.number_ready or 0)]) - for j in jobs: - wl_rows.append(["Job", j.metadata.name, str(j.spec.completions or 1), str(j.status.succeeded or 0)]) - for cj in cronjobs: - wl_rows.append(["CronJob", cj.metadata.name, cj.spec.schedule or "", cj.spec.concurrency_policy or ""]) - - cfg_rows: List[List[str]] = [] - cfg_rows.append(["ConfigMaps", str(len(configmaps))]) - cfg_rows.append(["Secrets", str(len(secrets))]) - cfg_rows.append(["PVCs", str(len(pvcs))]) - cfg_rows.append(["Ingresses", str(len(ingresses))]) - cfg_rows.append(["NetworkPolicies", str(len(netpols))]) - - print(f"Timestamp: {dt.datetime.now(dt.timezone.utc).isoformat()}") - print(f"Namespace: {namespace}") - - table("Pods", ["name", "phase", "ready", "restarts", "pod_ip", "node"], pod_rows) - table("Services", ["name", "type", "cluster_ip", "ports", "selector"], svc_rows) - table("Endpoints", ["name", "addresses", "ports"], ep_rows) - table("EndpointSlices", ["name", "service", "addresses", "ports"], eps_rows) - table("Workloads", ["kind", "name", "desired", "current/ready"], wl_rows) - table("Config/Storage summary", ["resource", "count"], cfg_rows) - - return 0 - - -def main() -> int: - args = parse_args() - try: - load_cfg(args) - except Exception as exc: # pragma: no cover - print(f"Failed to load kube config: {exc}", file=sys.stderr) - return 1 - return summarize(args.namespace) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/jupyter-notebooks/llm-observability-in-action/python/02_service_path_inspector.py b/jupyter-notebooks/llm-observability-in-action/python/02_service_path_inspector.py deleted file mode 100755 index 3515af5..0000000 --- a/jupyter-notebooks/llm-observability-in-action/python/02_service_path_inspector.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3.11 -"""Trace service routing path: selector -> pods -> endpoints -> routes.""" - -from __future__ import annotations - -import argparse -import sys -from typing import Dict, List - -from kubernetes import client, config -from kubernetes.client import ApiException - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Inspect end-to-end service path") - parser.add_argument("--namespace", default="llm-observability", help="Namespace") - parser.add_argument("--service", required=True, help="Service name") - parser.add_argument("--context", default=None, help="Optional kubeconfig context") - parser.add_argument("--kubeconfig", default=None, help="Optional kubeconfig file") - parser.add_argument("--in-cluster", action="store_true", help="Use in-cluster auth") - return parser.parse_args() - - -def load_cfg(args: argparse.Namespace) -> None: - if args.in_cluster: - config.load_incluster_config() - else: - config.load_kube_config(config_file=args.kubeconfig, context=args.context) - - -def selector_to_query(selector: Dict[str, str]) -> str: - if not selector: - return "" - return ",".join(f"{k}={v}" for k, v in selector.items()) - - -def print_header(label: str) -> None: - print(f"\n=== {label} ===") - - -def ingress_refs_service(ing: client.V1Ingress, service_name: str) -> bool: - if not ing.spec: - return False - for rule in ing.spec.rules or []: - if not rule.http: - continue - for path in rule.http.paths or []: - backend_svc = path.backend.service if path.backend else None - if backend_svc and backend_svc.name == service_name: - return True - return False - - -def http_route_refs_service(route: dict, namespace: str, service_name: str) -> bool: - spec = route.get("spec", {}) - for rule in spec.get("rules", []): - for backend_ref in rule.get("backendRefs", []): - if backend_ref.get("name") != service_name: - continue - backend_ns = backend_ref.get("namespace", namespace) - if backend_ns == namespace: - return True - return False - - -def main() -> int: - args = parse_args() - try: - load_cfg(args) - except Exception as exc: - print(f"Failed to load kube config: {exc}", file=sys.stderr) - return 1 - - core = client.CoreV1Api() - net = client.NetworkingV1Api() - discovery = client.DiscoveryV1Api() - custom = client.CustomObjectsApi() - - try: - svc = core.read_namespaced_service(name=args.service, namespace=args.namespace) - except ApiException as exc: - if exc.status == 404: - print(f"Service not found: {args.namespace}/{args.service}", file=sys.stderr) - return 2 - print(f"API error: {exc.status} {exc.reason}", file=sys.stderr) - return 2 - - selector = svc.spec.selector or {} - selector_query = selector_to_query(selector) - - print_header("Service") - print(f"name: {svc.metadata.name}") - print(f"namespace: {svc.metadata.namespace}") - print(f"type: {svc.spec.type}") - print(f"cluster_ip: {svc.spec.cluster_ip}") - print(f"selector: {selector if selector else ''}") - print( - "ports: " - f"{[{'name': p.name, 'port': p.port, 'targetPort': p.target_port, 'protocol': p.protocol} for p in (svc.spec.ports or [])]}" - ) - - print_header("Selected Pods") - pods = [] - if selector_query: - pods = core.list_namespaced_pod(namespace=args.namespace, label_selector=selector_query).items - if not pods: - print("No pods matched service selector or selector is empty.") - for pod in pods: - ready = all(cs.ready for cs in (pod.status.container_statuses or [])) if pod.status.container_statuses else False - print( - f"- {pod.metadata.name} phase={pod.status.phase} ready={ready} " - f"pod_ip={pod.status.pod_ip} node={pod.spec.node_name}" - ) - - print_header("Endpoints") - try: - ep = core.read_namespaced_endpoints(name=args.service, namespace=args.namespace) - if not ep.subsets: - print("No endpoint subsets.") - else: - for subset in ep.subsets: - addrs = [a.ip for a in (subset.addresses or [])] - not_ready = [a.ip for a in (subset.not_ready_addresses or [])] - ports = [f"{p.name or ''}:{p.port}/{p.protocol}" for p in (subset.ports or [])] - print(f"addresses={addrs or []} not_ready={not_ready or []} ports={ports or []}") - except ApiException as exc: - print(f"Failed to read Endpoints: {exc.status} {exc.reason}") - - print_header("EndpointSlices") - try: - slices = discovery.list_namespaced_endpoint_slice( - namespace=args.namespace, - label_selector=f"kubernetes.io/service-name={args.service}", - ).items - if not slices: - print("No EndpointSlices found.") - for eps in slices: - addresses: List[str] = [] - for endpoint in eps.endpoints or []: - addresses.extend(endpoint.addresses or []) - ports = [f"{p.name or ''}:{p.port}/{p.protocol}" for p in (eps.ports or [])] - print(f"- {eps.metadata.name} addresses={addresses or []} ports={ports or []}") - except ApiException as exc: - print(f"Failed to list EndpointSlices: {exc.status} {exc.reason}") - - print_header("Ingress / HTTPRoute references") - try: - ingresses = net.list_namespaced_ingress(namespace=args.namespace).items - except ApiException as exc: - print(f"Failed to list Ingress: {exc.status} {exc.reason}") - ingresses = [] - ingress_hits = [ing.metadata.name for ing in ingresses if ingress_refs_service(ing, args.service)] - print(f"Ingress refs: {ingress_hits or []}") - - route_hits: List[str] = [] - try: - routes = custom.list_namespaced_custom_object( - group="gateway.networking.k8s.io", - version="v1", - namespace=args.namespace, - plural="httproutes", - ) - for route in routes.get("items", []): - if http_route_refs_service(route, args.namespace, args.service): - route_hits.append(route.get("metadata", {}).get("name", "")) - except ApiException as exc: - if exc.status != 404: - print(f"Failed to list HTTPRoutes: {exc.status} {exc.reason}") - print(f"HTTPRoute refs: {route_hits or []}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/jupyter-notebooks/llm-observability-in-action/python/03_workload_health.py b/jupyter-notebooks/llm-observability-in-action/python/03_workload_health.py deleted file mode 100755 index 3622001..0000000 --- a/jupyter-notebooks/llm-observability-in-action/python/03_workload_health.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3.11 -"""Check rollout/availability health for namespace workloads.""" - -from __future__ import annotations - -import argparse -import sys -from typing import List, Tuple - -from kubernetes import client, config -from kubernetes.client import ApiException - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Workload health checks") - parser.add_argument("--namespace", default="llm-observability", help="Namespace") - parser.add_argument("--context", default=None, help="Optional kubeconfig context") - parser.add_argument("--kubeconfig", default=None, help="Optional kubeconfig file") - parser.add_argument("--in-cluster", action="store_true", help="Use in-cluster auth") - return parser.parse_args() - - -def load_cfg(args: argparse.Namespace) -> None: - if args.in_cluster: - config.load_incluster_config() - else: - config.load_kube_config(config_file=args.kubeconfig, context=args.context) - - -def print_rows(title: str, rows: List[Tuple[str, str, str]]) -> None: - print(f"\n=== {title} ===") - if not rows: - print("") - return - print("kind/name | status | details") - print("----------+--------+--------") - for name, status, detail in rows: - print(f"{name} | {status} | {detail}") - - -def main() -> int: - args = parse_args() - try: - load_cfg(args) - except Exception as exc: - print(f"Failed to load kube config: {exc}", file=sys.stderr) - return 1 - - apps = client.AppsV1Api() - batch = client.BatchV1Api() - - overall_ok = True - deployment_rows: List[Tuple[str, str, str]] = [] - stateful_rows: List[Tuple[str, str, str]] = [] - daemon_rows: List[Tuple[str, str, str]] = [] - job_rows: List[Tuple[str, str, str]] = [] - - try: - deployments = apps.list_namespaced_deployment(args.namespace).items - for d in deployments: - desired = d.spec.replicas or 0 - available = d.status.available_replicas or 0 - observed = d.status.observed_generation or 0 - generation = d.metadata.generation or 0 - ok = available >= desired and observed >= generation - status = "OK" if ok else "FAIL" - if not ok: - overall_ok = False - deployment_rows.append( - ( - f"deploy/{d.metadata.name}", - status, - f"desired={desired} available={available} observedGen={observed} gen={generation}", - ) - ) - - statefulsets = apps.list_namespaced_stateful_set(args.namespace).items - for s in statefulsets: - desired = s.spec.replicas or 0 - ready = s.status.ready_replicas or 0 - ok = ready >= desired - status = "OK" if ok else "FAIL" - if not ok: - overall_ok = False - stateful_rows.append((f"sts/{s.metadata.name}", status, f"desired={desired} ready={ready}")) - - daemonsets = apps.list_namespaced_daemon_set(args.namespace).items - for ds in daemonsets: - desired = ds.status.desired_number_scheduled or 0 - ready = ds.status.number_ready or 0 - ok = ready >= desired - status = "OK" if ok else "FAIL" - if not ok: - overall_ok = False - daemon_rows.append((f"ds/{ds.metadata.name}", status, f"desired={desired} ready={ready}")) - - jobs = batch.list_namespaced_job(args.namespace).items - for j in jobs: - failed = j.status.failed or 0 - succeeded = j.status.succeeded or 0 - completions = j.spec.completions or 1 - ok = failed == 0 and succeeded >= completions - status = "OK" if ok else "WARN" - if failed > 0: - overall_ok = False - status = "FAIL" - job_rows.append((f"job/{j.metadata.name}", status, f"succeeded={succeeded} failed={failed} completions={completions}")) - - except ApiException as exc: - print(f"API error: {exc.status} {exc.reason}", file=sys.stderr) - return 2 - - print_rows("Deployments", deployment_rows) - print_rows("StatefulSets", stateful_rows) - print_rows("DaemonSets", daemon_rows) - print_rows("Jobs", job_rows) - - if overall_ok: - print("\nOverall: OK") - return 0 - - print("\nOverall: FAIL") - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/jupyter-notebooks/llm-observability-in-action/python/04_networking_report.py b/jupyter-notebooks/llm-observability-in-action/python/04_networking_report.py deleted file mode 100755 index 49649a8..0000000 --- a/jupyter-notebooks/llm-observability-in-action/python/04_networking_report.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python3.11 -"""Build a namespace networking report and optional JSON output.""" - -from __future__ import annotations - -import argparse -import json -import sys -from typing import Dict, List - -from kubernetes import client, config -from kubernetes.client import ApiException - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Networking report") - parser.add_argument("--namespace", default="llm-observability", help="Namespace") - parser.add_argument("--context", default=None, help="Optional kubeconfig context") - parser.add_argument("--kubeconfig", default=None, help="Optional kubeconfig file") - parser.add_argument("--in-cluster", action="store_true", help="Use in-cluster auth") - parser.add_argument("--json", action="store_true", help="Print JSON report") - return parser.parse_args() - - -def load_cfg(args: argparse.Namespace) -> None: - if args.in_cluster: - config.load_incluster_config() - else: - config.load_kube_config(config_file=args.kubeconfig, context=args.context) - - -def match_selector(labels: Dict[str, str] | None, selector: Dict[str, str] | None) -> bool: - if not selector: - return False - labels = labels or {} - return all(labels.get(k) == v for k, v in selector.items()) - - -def main() -> int: - args = parse_args() - try: - load_cfg(args) - except Exception as exc: - print(f"Failed to load kube config: {exc}", file=sys.stderr) - return 1 - - core = client.CoreV1Api() - discovery = client.DiscoveryV1Api() - net = client.NetworkingV1Api() - - try: - pods = core.list_namespaced_pod(args.namespace).items - svcs = core.list_namespaced_service(args.namespace).items - endpoints = core.list_namespaced_endpoints(args.namespace).items - endpoint_slices = discovery.list_namespaced_endpoint_slice(args.namespace).items - ingresses = net.list_namespaced_ingress(args.namespace).items - except ApiException as exc: - print(f"API error: {exc.status} {exc.reason}", file=sys.stderr) - return 2 - - pod_index = { - p.metadata.name: { - "name": p.metadata.name, - "labels": p.metadata.labels or {}, - "pod_ip": p.status.pod_ip, - "node": p.spec.node_name, - "phase": p.status.phase, - } - for p in pods - } - - endpoints_by_name: Dict[str, List[str]] = {} - for ep in endpoints: - addrs: List[str] = [] - for subset in ep.subsets or []: - addrs.extend([a.ip for a in (subset.addresses or [])]) - endpoints_by_name[ep.metadata.name] = addrs - - epslice_by_service: Dict[str, List[str]] = {} - for eps in endpoint_slices: - svc_name = (eps.metadata.labels or {}).get("kubernetes.io/service-name") - if not svc_name: - continue - epslice_by_service.setdefault(svc_name, []) - for endpoint in eps.endpoints or []: - epslice_by_service[svc_name].extend(endpoint.addresses or []) - - ingress_map: Dict[str, List[str]] = {} - for ing in ingresses: - for rule in ing.spec.rules or []: - if not rule.http: - continue - for path in rule.http.paths or []: - backend = path.backend.service if path.backend else None - if not backend: - continue - ingress_map.setdefault(backend.name, []) - ingress_map[backend.name].append(ing.metadata.name) - - report: Dict[str, object] = { - "namespace": args.namespace, - "services": [], - } - - for svc in svcs: - selector = svc.spec.selector or {} - selected_pods = [pod["name"] for pod in pod_index.values() if match_selector(pod["labels"], selector)] - ports = [{"name": p.name, "port": p.port, "targetPort": p.target_port, "protocol": p.protocol} for p in (svc.spec.ports or [])] - svc_record = { - "service": svc.metadata.name, - "type": svc.spec.type, - "cluster_ip": svc.spec.cluster_ip, - "dns": f"{svc.metadata.name}.{args.namespace}.svc.cluster.local", - "selector": selector, - "ports": ports, - "selected_pods": selected_pods, - "endpoints": endpoints_by_name.get(svc.metadata.name, []), - "endpoint_slices": epslice_by_service.get(svc.metadata.name, []), - "ingress_refs": ingress_map.get(svc.metadata.name, []), - } - report["services"].append(svc_record) - - if args.json: - print(json.dumps(report, indent=2, sort_keys=True)) - return 0 - - print(f"Namespace: {args.namespace}") - for svc in report["services"]: # type: ignore[index] - print("\n---") - print(f"service: {svc['service']}") - print(f"type: {svc['type']} cluster_ip: {svc['cluster_ip']}") - print(f"dns: {svc['dns']}") - print(f"selector: {svc['selector'] or ''}") - print(f"ports: {svc['ports']}") - print(f"selected_pods: {svc['selected_pods']}") - print(f"endpoints: {svc['endpoints']}") - print(f"endpoint_slices: {svc['endpoint_slices']}") - print(f"ingress_refs: {svc['ingress_refs']}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/jupyter-notebooks/llm-observability-in-action/python/05_watch_events.py b/jupyter-notebooks/llm-observability-in-action/python/05_watch_events.py deleted file mode 100755 index 302dafc..0000000 --- a/jupyter-notebooks/llm-observability-in-action/python/05_watch_events.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3.11 -"""Watch Kubernetes events in a namespace for troubleshooting.""" - -from __future__ import annotations - -import argparse -import datetime as dt -import sys - -from kubernetes import client, config, watch -from kubernetes.client import ApiException - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Watch namespace events") - parser.add_argument("--namespace", default="llm-observability", help="Namespace") - parser.add_argument("--context", default=None, help="Optional kubeconfig context") - parser.add_argument("--kubeconfig", default=None, help="Optional kubeconfig file") - parser.add_argument("--in-cluster", action="store_true", help="Use in-cluster auth") - parser.add_argument("--timeout", type=int, default=300, help="Watch timeout seconds") - parser.add_argument("--types", default="Normal,Warning", help="Comma-separated event types") - return parser.parse_args() - - -def load_cfg(args: argparse.Namespace) -> None: - if args.in_cluster: - config.load_incluster_config() - else: - config.load_kube_config(config_file=args.kubeconfig, context=args.context) - - -def main() -> int: - args = parse_args() - try: - load_cfg(args) - except Exception as exc: - print(f"Failed to load kube config: {exc}", file=sys.stderr) - return 1 - - allowed = {x.strip() for x in args.types.split(",") if x.strip()} - core = client.CoreV1Api() - event_watch = watch.Watch() - - print(f"Watching events in namespace={args.namespace} timeout={args.timeout}s types={sorted(allowed)}") - print("Press Ctrl+C to stop.") - - try: - for event in event_watch.stream( - core.list_namespaced_event, - namespace=args.namespace, - timeout_seconds=args.timeout, - ): - etype = event.get("type", "UNKNOWN") - obj: client.CoreV1Event = event["object"] - event_type = obj.type or "Unknown" - if allowed and event_type not in allowed: - continue - now = dt.datetime.now(dt.timezone.utc).isoformat() - involved = obj.involved_object - ref = f"{involved.kind}/{involved.name}" if involved else "" - print(f"[{now}] {etype} eventType={event_type} reason={obj.reason} object={ref} msg={obj.message}") - except KeyboardInterrupt: - print("Stopped by user.") - except ApiException as exc: - print(f"Watch API error: {exc.status} {exc.reason}", file=sys.stderr) - return 2 - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/jupyter-notebooks/llm-observability-in-action/python/README.md b/jupyter-notebooks/llm-observability-in-action/python/README.md deleted file mode 100644 index 4f149ad..0000000 --- a/jupyter-notebooks/llm-observability-in-action/python/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Python Kubernetes scripts - -These scripts use the official Kubernetes Python client and are focused on your local `llm-observability` namespace. - -## Install - -```bash -PYTHON_BIN="${PYTHON_BIN:-python3.11}" -"${PYTHON_BIN}" -m pip install -r python/requirements.txt -``` - -## Scripts - -1. `01_namespace_inventory.py` -- Full inventory across workloads, services, endpoints, config, and storage. - -2. `02_service_path_inspector.py` -- Service flow tracing from selectors to pods/endpoints and ingress/http route references. - -3. `03_workload_health.py` -- Rollout and availability health checks; exits non-zero if issues are detected. - -4. `04_networking_report.py` -- Networking graph-style report of services, selectors, endpoints, and DNS names. - -5. `05_watch_events.py` -- Live event streaming in namespace for troubleshooting. - -## Examples - -```bash -PYTHON_BIN="${PYTHON_BIN:-python3.11}" -"${PYTHON_BIN}" python/01_namespace_inventory.py --namespace llm-observability -"${PYTHON_BIN}" python/02_service_path_inspector.py --namespace llm-observability --service ollama -"${PYTHON_BIN}" python/03_workload_health.py --namespace llm-observability -"${PYTHON_BIN}" python/04_networking_report.py --namespace llm-observability --json -"${PYTHON_BIN}" python/05_watch_events.py --namespace llm-observability --timeout 180 -``` diff --git a/jupyter-notebooks/llm-observability-in-action/python/requirements.txt b/jupyter-notebooks/llm-observability-in-action/python/requirements.txt deleted file mode 100644 index 53d7f14..0000000 --- a/jupyter-notebooks/llm-observability-in-action/python/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -kubernetes>=29.0.0,<35.0.0 diff --git a/jupyter-notebooks/llm-observability-in-action/runbooks/run_go_suite.sh b/jupyter-notebooks/llm-observability-in-action/runbooks/run_go_suite.sh new file mode 100755 index 0000000..74fd90c --- /dev/null +++ b/jupyter-notebooks/llm-observability-in-action/runbooks/run_go_suite.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUNBOOK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd "${RUNBOOK_DIR}/../../.." && pwd -P)" +K8S_NAMESPACE="${K8S_NAMESPACE:-llm-observability}" +K8S_RELEASE="${K8S_RELEASE:-llm-observability-stack}" +OLLAMA_GATEWAY_SERVICE="${OLLAMA_GATEWAY_SERVICE:-ollama-gateway}" + +if [[ -f "${RUNBOOK_DIR}/../config.env" ]]; then + # shellcheck disable=SC1091 + source "${RUNBOOK_DIR}/../config.env" +fi + +cd "${REPO_ROOT}" +go run ./cmd/llm-observability status \ + --namespace "${K8S_NAMESPACE}" --release "${K8S_RELEASE}" +go run ./cmd/llm-observability validate \ + --namespace "${K8S_NAMESPACE}" --release "${K8S_RELEASE}" +go run ./cmd/llm-observability network --namespace "${K8S_NAMESPACE}" +for service in open-webui ollama "${OLLAMA_GATEWAY_SERVICE}"; do + if ! kubectl get service -n "${K8S_NAMESPACE}" "${service}" >/dev/null 2>&1; then + printf 'Skipping optional Service %s: not installed\n' "${service}" + continue + fi + go run ./cmd/llm-observability service-path \ + --namespace "${K8S_NAMESPACE}" --service "${service}" +done diff --git a/jupyter-notebooks/llm-observability-in-action/runbooks/run_python_suite.sh b/jupyter-notebooks/llm-observability-in-action/runbooks/run_python_suite.sh deleted file mode 100755 index 475e81c..0000000 --- a/jupyter-notebooks/llm-observability-in-action/runbooks/run_python_suite.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -RUNBOOK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" -ROOT_DIR="$(cd "${RUNBOOK_DIR}/.." && pwd -P)" - -K8S_NAMESPACE="llm-observability" -OPENWEBUI_SERVICE="open-webui" -OLLAMA_SERVICE="ollama" -LANGCHAIN_SERVICE="langchain-demo" -PYTHON_BIN="${PYTHON_BIN:-python3.11}" - -if [[ -f "${ROOT_DIR}/config.env" ]]; then - # shellcheck disable=SC1091 - source "${ROOT_DIR}/config.env" -fi - -"${PYTHON_BIN}" "${ROOT_DIR}/python/01_namespace_inventory.py" --namespace "${K8S_NAMESPACE:-llm-observability}" -"${PYTHON_BIN}" "${ROOT_DIR}/python/03_workload_health.py" --namespace "${K8S_NAMESPACE:-llm-observability}" -"${PYTHON_BIN}" "${ROOT_DIR}/python/02_service_path_inspector.py" --namespace "${K8S_NAMESPACE:-llm-observability}" --service "${OPENWEBUI_SERVICE:-open-webui}" -"${PYTHON_BIN}" "${ROOT_DIR}/python/02_service_path_inspector.py" --namespace "${K8S_NAMESPACE:-llm-observability}" --service "${OLLAMA_SERVICE:-ollama}" -"${PYTHON_BIN}" "${ROOT_DIR}/python/02_service_path_inspector.py" --namespace "${K8S_NAMESPACE:-llm-observability}" --service "${LANGCHAIN_SERVICE:-langchain-demo}" -"${PYTHON_BIN}" "${ROOT_DIR}/python/04_networking_report.py" --namespace "${K8S_NAMESPACE:-llm-observability}" diff --git a/jupyter-notebooks/llm-observability-stack-example-3.ipynb b/jupyter-notebooks/llm-observability-stack-example-3.ipynb index c41d598..78f78cc 100644 --- a/jupyter-notebooks/llm-observability-stack-example-3.ipynb +++ b/jupyter-notebooks/llm-observability-stack-example-3.ipynb @@ -35,7 +35,7 @@ "outputs": [], "source": [ "#Run this command in linux terminal only. \n", - "#!kubectl exec -it -n llm-observability langchain-demo-5dc95b9dbf-8ttrx -- /bin/bash" + "#!kubectl exec -it -n llm-observability ollama-gateway-5dc95b9dbf-8ttrx -- /bin/bash" ] }, { @@ -75,7 +75,7 @@ "metadata": {}, "outputs": [], "source": [ - "!kubectl describe pod langchain-demo-5dc95b9dbf-8ttrx -n llm-observability" + "!kubectl describe pod ollama-gateway-5dc95b9dbf-8ttrx -n llm-observability" ] }, { diff --git a/jupyter-notebooks/llm-observability-stack-example-4.ipynb b/jupyter-notebooks/llm-observability-stack-example-4.ipynb index 9bf5142..ee991d6 100644 --- a/jupyter-notebooks/llm-observability-stack-example-4.ipynb +++ b/jupyter-notebooks/llm-observability-stack-example-4.ipynb @@ -35,7 +35,7 @@ "outputs": [], "source": [ "#Run this command in linux terminal only. \n", - "#!kubectl exec -it -n llm-observability langchain-demo-5dc95b9dbf-8ttrx -- /bin/bash" + "#!kubectl exec -it -n llm-observability ollama-gateway-5dc95b9dbf-8ttrx -- /bin/bash" ] }, { @@ -75,7 +75,7 @@ "metadata": {}, "outputs": [], "source": [ - "!kubectl describe pod langchain-demo-5dc95b9dbf-8ttrx -n llm-observability" + "!kubectl describe pod ollama-gateway-5dc95b9dbf-8ttrx -n llm-observability" ] }, { @@ -105,7 +105,7 @@ "metadata": {}, "outputs": [], "source": [ - "#!kubectl -n llm-observability rollout restart deploy/langchain-demo\n" + "#!kubectl -n llm-observability rollout restart deploy/ollama-gateway\n" ] }, { diff --git a/jupyter-notebooks/llm-observability-stack-example-5.ipynb b/jupyter-notebooks/llm-observability-stack-example-5.ipynb index 14ed166..a60af0d 100644 --- a/jupyter-notebooks/llm-observability-stack-example-5.ipynb +++ b/jupyter-notebooks/llm-observability-stack-example-5.ipynb @@ -46,7 +46,7 @@ "outputs": [], "source": [ "#Run this command in linux terminal only. \n", - "#!kubectl exec -it -n llm-observability langchain-demo-5dc95b9dbf-8ttrx -- /bin/bash" + "#!kubectl exec -it -n llm-observability ollama-gateway-5dc95b9dbf-8ttrx -- /bin/bash" ] }, { @@ -66,7 +66,7 @@ "metadata": {}, "outputs": [], "source": [ - "!kubectl describe pod langchain-demo-5dc95b9dbf-fz9gz -n llm-observability" + "!kubectl describe pod ollama-gateway-5dc95b9dbf-fz9gz -n llm-observability" ] }, { @@ -76,7 +76,7 @@ "metadata": {}, "outputs": [], "source": [ - "!kubectl logs langchain-demo-5dc95b9dbf-fz9gz -n llm-observability" + "!kubectl logs ollama-gateway-5dc95b9dbf-fz9gz -n llm-observability" ] }, { @@ -86,7 +86,7 @@ "metadata": {}, "outputs": [], "source": [ - "!kubectl rollout status deployment/langchain-demo -n llm-observability" + "!kubectl rollout status deployment/ollama-gateway -n llm-observability" ] }, { @@ -96,7 +96,7 @@ "metadata": {}, "outputs": [], "source": [ - "!kubectl get deployment langchain-demo -n llm-observability -o yaml | grep -A 10 readinessProbe" + "!kubectl get deployment ollama-gateway -n llm-observability -o yaml | grep -A 10 readinessProbe" ] }, { diff --git a/langchain-demo/Dockerfile b/langchain-demo/Dockerfile deleted file mode 100644 index 0b4d548..0000000 --- a/langchain-demo/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.11-slim - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 - -WORKDIR /app - -COPY requirements.txt . -RUN python -m pip install --no-cache-dir --upgrade pip \ - && python -m pip install --no-cache-dir -r requirements.txt - -COPY app.py . - -EXPOSE 8000 - -RUN groupadd --gid 10001 appuser \ - && useradd --uid 10001 --gid 10001 --no-create-home appuser \ - && chown -R 10001:10001 /app -USER 10001:10001 - -CMD ["python", "-m", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"] diff --git a/langchain-demo/README.md b/langchain-demo/README.md deleted file mode 100644 index 7aa44ba..0000000 --- a/langchain-demo/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# langchain-demo - -`langchain-demo` is the local FastAPI application that provides: - -- a simple `/invoke` API for prompt testing -- health/config endpoints for notebook and cluster checks -- an Ollama-compatible proxy path at `/ollama/*` -- optional OpenTelemetry tracing for proxied Open WebUI traffic - -## Why It Exists - -This service is the observability bridge in the stack. Instead of sending browser traffic directly from Open WebUI to Ollama, Open WebUI can send Ollama-compatible requests through this service so the project can: - -- inspect proxy health independently -- keep a stable API surface for notebooks and smoke tests -- emit OpenTelemetry traces for local demo traffic - -## Source Files - -- `app.py` - FastAPI app and traced proxy logic -- `requirements.txt` - Python dependencies -- `Dockerfile` - local image build input - -## Main Endpoints - -- `GET /` -- `GET /healthz` -- `GET /config` -- `POST /invoke` -- `ANY /ollama/{path}` for upstream Ollama proxying - -## Important Environment Variables - -- `OLLAMA_MODEL` -- `OLLAMA_BASE_URL` -- `OLLAMA_UPSTREAM_BASE_URL` -- `OLLAMA_TEMPERATURE` -- `OLLAMA_PROXY_TIMEOUT_SECONDS` -- `OLLAMA_PROXY_TRACE_OTEL` -- `OTEL_TRACES_ENABLED` -- `OTEL_EXPORTER_OTLP_ENDPOINT` -- `OTEL_SERVICE_NAME` -- `OTEL_SERVICE_NAMESPACE` - -## Local Image Workflow - -Build: - -```bash -../hack/build-local-image.sh langchain-demo 0.1.1 ./langchain-demo -``` - -Import into k3s: - -```bash -../hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 -``` - -Restart deployment: - -```bash -kubectl rollout restart deploy/langchain-demo -n llm-observability -kubectl rollout status deploy/langchain-demo -n llm-observability -``` - -## Useful Checks - -Port-forward: - -```bash -kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 -``` - -Health: - -```bash -curl -s http://localhost:8000/healthz | jq -curl -s http://localhost:8000/config | jq -``` - -Invoke: - -```bash -curl -s http://localhost:8000/invoke \ - -H 'Content-Type: application/json' \ - -d '{"prompt":"Explain Kubernetes rollout strategy in three lines."}' | jq -``` - -Proxy to Ollama: - -```bash -curl -s http://localhost:8000/ollama/api/tags | jq -``` - -## Operational Notes - -- This service is intended to run from a prebuilt local image, not from mutable runtime installs. -- If notebook cells fail on `localhost:8000`, verify the port-forward first. -- If OpenTelemetry traces are missing, the proxy still works; check the OTLP endpoint and collector logs. diff --git a/langchain-demo/app.py b/langchain-demo/app.py deleted file mode 100644 index 1aa25b0..0000000 --- a/langchain-demo/app.py +++ /dev/null @@ -1,530 +0,0 @@ -import json -import os -import time -from typing import Any, Optional - -import httpx -from fastapi import FastAPI, HTTPException, Request -from fastapi.responses import Response, StreamingResponse -from langchain_ollama import ChatOllama -from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.trace import Span, Status, StatusCode -from prometheus_client import CONTENT_TYPE_LATEST, Counter, Gauge, Histogram, generate_latest -from pydantic import BaseModel, Field - -APP_TITLE = "k3s-ollama-opentelemetry-demo" -DEFAULT_MODEL = "gemma3-1b-it-gguf-local" -DEFAULT_BASE_URL = "http://ollama:11434" -DEFAULT_TEMPERATURE = 0.2 -DEFAULT_PROXY_PATH_PREFIX = "/ollama" -DEFAULT_PROXY_TIMEOUT_SECONDS = 180.0 -TRACE_PREVIEW_LIMIT = 8000 - -app = FastAPI(title=APP_TITLE) -_TRACER_CONFIGURED = False - -HTTP_REQUESTS = Counter( - "llm_observability_http_requests_total", - "HTTP requests handled by the LLM observability API.", - ["method", "route", "status"], -) -HTTP_REQUEST_DURATION = Histogram( - "llm_observability_http_request_duration_seconds", - "End-to-end HTTP request latency.", - ["method", "route"], - buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300), -) -LLM_REQUESTS = Counter( - "llm_observability_llm_requests_total", - "Completed LLM requests.", - ["model", "route", "outcome"], -) -LLM_ACTIVE_REQUESTS = Gauge( - "llm_observability_llm_active_requests", - "LLM requests currently being processed.", - ["model", "route"], -) -LLM_REQUEST_DURATION = Histogram( - "llm_observability_llm_request_duration_seconds", - "End-to-end LLM request latency.", - ["model", "route"], - buckets=(0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300), -) -LLM_TTFT = Histogram( - "llm_observability_time_to_first_token_seconds", - "Time from proxy request start to the first streamed response bytes.", - ["model", "route"], - buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60), -) -LLM_ITL = Histogram( - "llm_observability_inter_token_latency_seconds", - "Average inter-token latency derived from Ollama evaluation duration and token count.", - ["model", "route"], - buckets=(0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5), -) -LLM_PROMPT_TOKENS = Counter( - "llm_observability_prompt_tokens_total", - "Prompt tokens reported by the inference backend.", - ["model", "route"], -) -LLM_GENERATED_TOKENS = Counter( - "llm_observability_generated_tokens_total", - "Generated tokens reported by the inference backend.", - ["model", "route"], -) -LLM_TOKEN_THROUGHPUT = Histogram( - "llm_observability_generated_tokens_per_second", - "Generated token throughput reported by Ollama.", - ["model", "route"], - buckets=(0.5, 1, 2, 5, 10, 20, 40, 80, 160, 320), -) - - -class PromptIn(BaseModel): - prompt: str = Field(..., min_length=1) - system: Optional[str] = None - - -class InvokeOut(BaseModel): - response: str - model: str - ollama_base_url: str - - -def get_env(name: str, default: str) -> str: - value = os.environ.get(name) - return value if value not in (None, "") else default - - -def get_env_bool(name: str, default: bool) -> bool: - value = os.environ.get(name) - if value is None: - return default - return value.strip().lower() in {"1", "true", "yes", "on"} - - -def get_proxy_timeout_seconds() -> float: - try: - return float(get_env("OLLAMA_PROXY_TIMEOUT_SECONDS", str(DEFAULT_PROXY_TIMEOUT_SECONDS))) - except ValueError: - return DEFAULT_PROXY_TIMEOUT_SECONDS - - -def get_ollama_upstream_base_url() -> str: - return get_env("OLLAMA_UPSTREAM_BASE_URL", get_env("OLLAMA_BASE_URL", DEFAULT_BASE_URL)).rstrip("/") - - -def get_model_name(payload: Optional[Any] = None) -> str: - if isinstance(payload, dict) and payload.get("model"): - return str(payload["model"]) - return get_env("OLLAMA_MODEL", DEFAULT_MODEL) - - -def observe_ollama_payload(payload: Optional[Any], model: str, route: str) -> None: - if not isinstance(payload, dict): - return - - prompt_tokens = payload.get("prompt_eval_count") - generated_tokens = payload.get("eval_count") - eval_duration_ns = payload.get("eval_duration") - - if isinstance(prompt_tokens, (int, float)) and prompt_tokens >= 0: - LLM_PROMPT_TOKENS.labels(model=model, route=route).inc(float(prompt_tokens)) - if isinstance(generated_tokens, (int, float)) and generated_tokens >= 0: - LLM_GENERATED_TOKENS.labels(model=model, route=route).inc(float(generated_tokens)) - if ( - isinstance(generated_tokens, (int, float)) - and isinstance(eval_duration_ns, (int, float)) - and generated_tokens >= 0 - and eval_duration_ns > 0 - ): - tokens_per_second = float(generated_tokens) / (float(eval_duration_ns) / 1_000_000_000) - LLM_TOKEN_THROUGHPUT.labels(model=model, route=route).observe(tokens_per_second) - LLM_ITL.labels(model=model, route=route).observe(1.0 / tokens_per_second) - - -def safe_json(value: bytes) -> Optional[Any]: - if not value: - return None - try: - return json.loads(value.decode("utf-8")) - except Exception: - return None - - -def truncate_text(value: str, limit: int = TRACE_PREVIEW_LIMIT) -> str: - if len(value) <= limit: - return value - return value[:limit] - - -def configure_tracing() -> None: - global _TRACER_CONFIGURED - if _TRACER_CONFIGURED or not get_env_bool("OTEL_TRACES_ENABLED", True): - return - - endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") - resource = Resource.create( - { - "service.name": get_env("OTEL_SERVICE_NAME", "langchain-demo"), - "service.namespace": get_env("OTEL_SERVICE_NAMESPACE", "llm-observability"), - "deployment.environment": get_env("OTEL_DEPLOYMENT_ENVIRONMENT", "k3s-nvidia-edge"), - } - ) - provider = TracerProvider(resource=resource) - if endpoint: - provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))) - trace.set_tracer_provider(provider) - _TRACER_CONFIGURED = True - - -def get_tracer() -> trace.Tracer: - configure_tracing() - return trace.get_tracer("llm-observability-stack.langchain-demo") - - -def set_span_tokens(span: Span, payload: Optional[Any]) -> None: - if not isinstance(payload, dict): - return - prompt_tokens = payload.get("prompt_eval_count") - generated_tokens = payload.get("eval_count") - if isinstance(prompt_tokens, (int, float)) and prompt_tokens >= 0: - span.set_attribute("gen_ai.usage.input_tokens", int(prompt_tokens)) - if isinstance(generated_tokens, (int, float)) and generated_tokens >= 0: - span.set_attribute("gen_ai.usage.output_tokens", int(generated_tokens)) - - -def start_genai_span(name: str, model: str, route: str, extra: Optional[dict[str, Any]] = None) -> Span: - attributes: dict[str, Any] = { - "gen_ai.system": "ollama", - "gen_ai.operation.name": "chat", - "gen_ai.request.model": model, - "llm.route": route, - "server.address": get_ollama_upstream_base_url(), - } - if extra: - attributes.update(extra) - return get_tracer().start_span(name, attributes=attributes) - - -def start_proxy_trace( - method: str, upstream_path: str, query: str, payload: Optional[Any] -) -> Optional[Span]: - if not get_env_bool("OLLAMA_PROXY_TRACE_OTEL", True): - return None - - model = get_model_name(payload) - attrs: dict[str, Any] = { - "http.request.method": method, - "url.path": upstream_path, - } - if query: - attrs["url.query"] = query - if payload is not None: - attrs["llm.request.preview"] = truncate_text(json.dumps(payload, ensure_ascii=True)) - return start_genai_span(f"ollama {method.lower()} {upstream_path}", model, "ollama_proxy", attrs) - - -def finish_proxy_trace( - span: Optional[Span], - status_code: Optional[int], - outputs: Optional[dict[str, Any]] = None, - error: Optional[str] = None, -) -> None: - if span is None: - return - - if status_code is not None: - span.set_attribute("http.response.status_code", status_code) - if status_code >= 500: - span.set_status(Status(StatusCode.ERROR)) - for key, value in (outputs or {}).items(): - if isinstance(value, (str, int, float, bool)): - span.set_attribute(f"llm.response.{key}", value) - elif isinstance(value, dict): - span.set_attribute(f"llm.response.{key}", truncate_text(json.dumps(value, ensure_ascii=True))) - if error: - span.record_exception(Exception(error)) - span.set_status(Status(StatusCode.ERROR, error)) - span.end() - - - -def extract_response_payload(resp: Any) -> dict[str, Any]: - content_type = resp.headers.get("content-type", "") - if "application/json" in content_type: - try: - return {"response_json": resp.json()} - except Exception: - pass - return {"response_text_preview": truncate_text(resp.text)} - - -def build_upstream_headers(request: Request) -> dict[str, str]: - forwarded: dict[str, str] = {} - skip_headers = {"host", "content-length", "connection"} - for key, value in request.headers.items(): - if key.lower() in skip_headers: - continue - forwarded[key] = value - return forwarded - - -def get_llm() -> ChatOllama: - return ChatOllama( - model=get_env("OLLAMA_MODEL", DEFAULT_MODEL), - base_url=get_env("OLLAMA_BASE_URL", DEFAULT_BASE_URL), - temperature=float(get_env("OLLAMA_TEMPERATURE", str(DEFAULT_TEMPERATURE))), - ) - - -@app.middleware("http") -async def observe_http_request(request: Request, call_next): # type: ignore[no-untyped-def] - if request.url.path == "/metrics": - return await call_next(request) - - started = time.perf_counter() - status_code = 500 - try: - response = await call_next(request) - status_code = response.status_code - return response - finally: - route = getattr(request.scope.get("route"), "path", request.url.path) - HTTP_REQUESTS.labels( - method=request.method, - route=route, - status=str(status_code), - ).inc() - HTTP_REQUEST_DURATION.labels(method=request.method, route=route).observe( - time.perf_counter() - started - ) - - -@app.get("/") -def root() -> dict: - return { - "name": APP_TITLE, - "health": "/healthz", - "invoke": "/invoke", - "config": "/config", - "ollama_proxy": f"{get_env('OLLAMA_PROXY_PATH_PREFIX', DEFAULT_PROXY_PATH_PREFIX)}/api/*", - } - - -@app.get("/healthz") -def healthz() -> dict: - return { - "status": "ok", - "model": get_env("OLLAMA_MODEL", DEFAULT_MODEL), - "ollama_base_url": get_env("OLLAMA_BASE_URL", DEFAULT_BASE_URL), - "ollama_upstream_base_url": get_ollama_upstream_base_url(), - "otel_traces_enabled": get_env("OTEL_TRACES_ENABLED", "true"), - "otel_exporter_otlp_endpoint": os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT"), - "etcd_endpoints": os.environ.get("ETCD_ENDPOINTS"), - } - - -@app.get("/metrics", include_in_schema=False) -def metrics() -> Response: - return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) - - -@app.get("/config") -def config() -> dict: - return { - "model": get_env("OLLAMA_MODEL", DEFAULT_MODEL), - "ollama_base_url": get_env("OLLAMA_BASE_URL", DEFAULT_BASE_URL), - "ollama_upstream_base_url": get_ollama_upstream_base_url(), - "temperature": float(get_env("OLLAMA_TEMPERATURE", str(DEFAULT_TEMPERATURE))), - "otel_service_name": get_env("OTEL_SERVICE_NAME", "langchain-demo"), - "otel_exporter_otlp_endpoint": os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT"), - "etcd_endpoints": os.environ.get("ETCD_ENDPOINTS"), - } - - -@app.post("/invoke", response_model=InvokeOut) -def invoke(req: PromptIn) -> InvokeOut: - model = get_model_name() - route = "invoke" - started = time.perf_counter() - LLM_ACTIVE_REQUESTS.labels(model=model, route=route).inc() - span = start_genai_span( - "ollama invoke", - model, - route, - {"gen_ai.request.temperature": float(get_env("OLLAMA_TEMPERATURE", str(DEFAULT_TEMPERATURE)))}, - ) - try: - llm = get_llm() - prompt = req.prompt if not req.system else f"System: {req.system}\n\nUser: {req.prompt}" - span.set_attribute("llm.request.prompt_preview", truncate_text(prompt)) - response = llm.invoke(prompt) - content = getattr(response, "content", response) - response_metadata = getattr(response, "response_metadata", None) - observe_ollama_payload(response_metadata, model, route) - set_span_tokens(span, response_metadata) - span.set_attribute("gen_ai.response.model", model) - span.set_attribute("llm.response.preview", truncate_text(str(content))) - LLM_REQUESTS.labels(model=model, route=route, outcome="success").inc() - return InvokeOut( - response=str(content), - model=model, - ollama_base_url=get_env("OLLAMA_BASE_URL", DEFAULT_BASE_URL), - ) - except Exception as exc: # pragma: no cover - runtime surface for demo support drills - LLM_REQUESTS.labels(model=model, route=route, outcome="error").inc() - span.record_exception(exc) - span.set_status(Status(StatusCode.ERROR, str(exc))) - raise HTTPException(status_code=500, detail=str(exc)) from exc - finally: - span.end() - LLM_ACTIVE_REQUESTS.labels(model=model, route=route).dec() - LLM_REQUEST_DURATION.labels(model=model, route=route).observe(time.perf_counter() - started) - - -@app.api_route("/ollama/{upstream_path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) -async def ollama_proxy(upstream_path: str, request: Request): - upstream_base_url = get_ollama_upstream_base_url() - normalized_path = f"/{upstream_path.lstrip('/')}" - upstream_url = f"{upstream_base_url}{normalized_path}" - if request.url.query: - upstream_url = f"{upstream_url}?{request.url.query}" - - request_body = await request.body() - body_json = safe_json(request_body) - model = get_model_name(body_json) - route = "ollama_proxy" - started = time.perf_counter() - stream_owns_metrics = False - LLM_ACTIVE_REQUESTS.labels(model=model, route=route).inc() - proxy_headers = build_upstream_headers(request) - - span = start_proxy_trace( - method=request.method, - upstream_path=normalized_path, - query=request.url.query, - payload=body_json, - ) - - stream_requested = bool(isinstance(body_json, dict) and body_json.get("stream") is True) - timeout_seconds = get_proxy_timeout_seconds() - timeout = httpx.Timeout(timeout_seconds, connect=10.0) - - try: - if stream_requested: - async_client = httpx.AsyncClient(timeout=timeout) - upstream_request = async_client.build_request( - method=request.method, - url=upstream_url, - headers=proxy_headers, - content=request_body or None, - ) - try: - upstream_response = await async_client.send(upstream_request, stream=True) - except httpx.HTTPError: - await async_client.aclose() - raise - content_type = upstream_response.headers.get("content-type", "application/octet-stream") - upstream_status_code = upstream_response.status_code - - async def iter_stream(): - preview_parts: list[str] = [] - preview_chars = 0 - stream_buffer = b"" - final_payload: Optional[Any] = None - first_chunk_at: Optional[float] = None - try: - async for chunk in upstream_response.aiter_bytes(chunk_size=8192): - if not chunk: - continue - now = time.perf_counter() - if first_chunk_at is None: - first_chunk_at = now - LLM_TTFT.labels(model=model, route=route).observe(now - started) - stream_buffer += chunk - lines = stream_buffer.split(b"\n") - stream_buffer = lines.pop() - for line in lines: - parsed = safe_json(line.strip()) - if isinstance(parsed, dict) and parsed.get("done") is True: - final_payload = parsed - if preview_chars < TRACE_PREVIEW_LIMIT: - decoded = chunk.decode("utf-8", errors="ignore") - remaining = TRACE_PREVIEW_LIMIT - preview_chars - sample = decoded[:remaining] - preview_parts.append(sample) - preview_chars += len(sample) - yield chunk - if stream_buffer.strip(): - parsed = safe_json(stream_buffer.strip()) - if isinstance(parsed, dict) and parsed.get("done") is True: - final_payload = parsed - observe_ollama_payload(final_payload, model, route) - set_span_tokens(span, final_payload) - outcome = "success" if upstream_status_code < 500 else "error" - LLM_REQUESTS.labels(model=model, route=route, outcome=outcome).inc() - finish_proxy_trace( - span, - upstream_status_code, - outputs={"streamed": True, "response_preview": "".join(preview_parts)}, - ) - except Exception as exc: - LLM_REQUESTS.labels(model=model, route=route, outcome="error").inc() - finish_proxy_trace( - span, - upstream_status_code, - outputs={"streamed": True}, - error=str(exc), - ) - raise - finally: - LLM_ACTIVE_REQUESTS.labels(model=model, route=route).dec() - LLM_REQUEST_DURATION.labels(model=model, route=route).observe( - time.perf_counter() - started - ) - await upstream_response.aclose() - await async_client.aclose() - - stream_owns_metrics = True - return StreamingResponse( - iter_stream(), - media_type=content_type, - status_code=upstream_status_code, - ) - - async with httpx.AsyncClient(timeout=timeout) as async_client: - upstream_response = await async_client.request( - method=request.method, - url=upstream_url, - headers=proxy_headers, - content=request_body or None, - ) - - response_payload = safe_json(upstream_response.content) - observe_ollama_payload(response_payload, model, route) - set_span_tokens(span, response_payload) - finish_proxy_trace( - span, - upstream_response.status_code, - outputs=extract_response_payload(upstream_response), - ) - outcome = "success" if upstream_response.status_code < 500 else "error" - LLM_REQUESTS.labels(model=model, route=route, outcome=outcome).inc() - return Response( - content=upstream_response.content, - status_code=upstream_response.status_code, - media_type=upstream_response.headers.get("content-type"), - ) - except httpx.HTTPError as exc: - LLM_REQUESTS.labels(model=model, route=route, outcome="error").inc() - finish_proxy_trace(span, None, error=str(exc)) - raise HTTPException(status_code=502, detail=f"Ollama proxy request failed: {exc}") from exc - finally: - if not stream_owns_metrics: - LLM_ACTIVE_REQUESTS.labels(model=model, route=route).dec() - LLM_REQUEST_DURATION.labels(model=model, route=route).observe(time.perf_counter() - started) diff --git a/langchain-demo/requirements.txt b/langchain-demo/requirements.txt deleted file mode 100644 index 32fe054..0000000 --- a/langchain-demo/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -fastapi==0.115.12 -uvicorn[standard]==0.34.0 -pydantic==2.10.6 -langchain==0.3.21 -langchain-ollama==0.3.0 -httpx==0.28.1 -prometheus-client==0.22.1 -opentelemetry-api==1.34.1 -opentelemetry-sdk==1.34.1 -opentelemetry-exporter-otlp-proto-grpc==1.34.1 diff --git a/llm-observability-stack-run-instructions-with-git.txt b/llm-observability-stack-run-instructions-with-git.txt index 54d61a9..6aa8913 100644 --- a/llm-observability-stack-run-instructions-with-git.txt +++ b/llm-observability-stack-run-instructions-with-git.txt @@ -69,11 +69,11 @@ curl -s http://127.0.0.1:11434/api/generate \ Before enabling the full profile, build and import the local images: -./hack/build-local-image.sh langchain-demo 0.1.1 ./langchain-demo -./hack/import-local-image-to-k3s.sh langchain-demo 0.1.1 +./hack/build-local-image.sh ollama-gateway 0.2.0 . ollama-gateway/Dockerfile +./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 -./hack/build-local-image.sh python-toolbox 0.2.0 ./python-toolbox -./hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 +./hack/build-local-image.sh edge-toolbox 0.2.0 . edge-toolbox/Dockerfile +./hack/import-local-image-to-k3s.sh edge-toolbox 0.2.0 Install the full local profile: @@ -91,8 +91,8 @@ helm upgrade --install llm-observability-stack . \ kubectl get pods,svc,pvc -n llm-observability -o wide kubectl rollout status deploy/ollama -n llm-observability --timeout=10m -kubectl rollout status deploy/langchain-demo -n llm-observability --timeout=5m -kubectl rollout status deploy/python-toolbox -n llm-observability --timeout=5m +kubectl rollout status deploy/ollama-gateway -n llm-observability --timeout=5m +kubectl rollout status deploy/edge-toolbox -n llm-observability --timeout=5m kubectl rollout status deploy/opentelemetry-collector -n llm-observability --timeout=5m 7. Access services locally @@ -101,9 +101,9 @@ Ollama: kubectl port-forward -n llm-observability svc/ollama 11434:11434 -LangChain proxy: +Ollama gateway: -kubectl port-forward -n llm-observability svc/langchain-demo 8000:8000 +kubectl port-forward -n llm-observability svc/ollama-gateway 8000:8000 Open WebUI: @@ -137,7 +137,7 @@ helm template llm-observability-stack . \ -f values.enterprise-pilot-k3s.yaml \ --set kube-prometheus-stack.crds.enabled=false >/tmp/rendered-local-full-stack.yaml -python3 -m pytest -q tests +go test ./... Optional local stack validation: @@ -148,7 +148,7 @@ Optional local stack validation: With Ollama port-forward active: -./benchmarks/ollama_benchmark.py \ +bin/llm-observability benchmark \ --model gemma3-1b-it-gguf-local \ --warmup-runs 1 \ --runs 10 \ @@ -158,8 +158,8 @@ With Ollama port-forward active: kubectl get all -n llm-observability kubectl logs -n llm-observability deploy/ollama --tail=100 -kubectl logs -n llm-observability deploy/langchain-demo --tail=100 -kubectl logs -n llm-observability deploy/python-toolbox --tail=100 +kubectl logs -n llm-observability deploy/ollama-gateway --tail=100 +kubectl logs -n llm-observability deploy/edge-toolbox --tail=100 kubectl logs -n llm-observability deploy/opentelemetry-collector --tail=100 kubectl get events -n llm-observability --sort-by=.lastTimestamp @@ -189,7 +189,7 @@ git diff --stat Run the normal repository validation: helm lint . -python3 -m pytest -q tests +go test ./... 13. Commit project changes diff --git a/ollama-gateway/Dockerfile b/ollama-gateway/Dockerfile new file mode 100644 index 0000000..c999628 --- /dev/null +++ b/ollama-gateway/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.26-bookworm AS build +WORKDIR /src +COPY go.mod go.sum ./ +COPY cmd/ollama-gateway ./cmd/ollama-gateway +COPY internal/gateway ./internal/gateway +RUN go mod edit -droprequire github.com/Edge-Computing-LLM/k3s-nvidia-edge \ + -dropreplace github.com/Edge-Computing-LLM/k3s-nvidia-edge \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/ollama-gateway ./cmd/ollama-gateway + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/ollama-gateway /usr/local/bin/ollama-gateway +EXPOSE 8000 +ENTRYPOINT ["/usr/local/bin/ollama-gateway"] diff --git a/ollama-gateway/README.md b/ollama-gateway/README.md new file mode 100644 index 0000000..fd442d8 --- /dev/null +++ b/ollama-gateway/README.md @@ -0,0 +1,15 @@ +# ollama-gateway + +`ollama-gateway` is a native Go HTTP gateway for the local Ollama service. It +provides `/healthz`, `/config`, `/invoke`, `/ollama/*`, and `/metrics`, preserves +streaming responses, and exports OpenTelemetry traces when an OTLP endpoint is +configured. + +Build the local image from the repository root: + +```bash +./hack/build-local-image.sh ollama-gateway 0.2.0 . ollama-gateway/Dockerfile +./hack/import-local-image-to-k3s.sh ollama-gateway 0.2.0 +``` + +The image contains one non-root static Go binary and no Python runtime. diff --git a/python-toolbox/Dockerfile b/python-toolbox/Dockerfile deleted file mode 100644 index 9e97dfe..0000000 --- a/python-toolbox/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM python:3.11-slim - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=1 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - dnsutils \ - iputils-ping \ - netcat-openbsd \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /workspace -COPY requirements.txt /tmp/requirements.txt -RUN python -m pip install --upgrade pip \ - && pip install --no-cache-dir -r /tmp/requirements.txt -COPY examples/ /workspace/examples/ - -CMD ["sleep", "infinity"] diff --git a/python-toolbox/README.md b/python-toolbox/README.md deleted file mode 100644 index be64110..0000000 --- a/python-toolbox/README.md +++ /dev/null @@ -1,92 +0,0 @@ -# python-toolbox - -`python-toolbox` is a minimal in-cluster helper image for diagnostics, connectivity checks, and notebook-backed Kubernetes troubleshooting. - -## Purpose - -This component exists to give the project a safe in-cluster execution point for: - -- DNS resolution checks -- internal Service connectivity checks -- Ollama smoke tests -- Redis checks -- OpenTelemetry API health and optional trace seeding - -It is intentionally lighter than a full notebook image and is meant for shell/script execution inside the cluster. - -## Local Profile Status - -- `pythonToolbox.enabled: true` -- continuous OpenTelemetry seeder jobs remain disabled by default - -## Included Example Scripts - -- `service_dns_check.py` -- `ollama_smoke.py` -- `redis_ping.py` -- `otel_genai_inference_traces.py` -- `otel_genai_trace_seed_every_5m.py` - -Scripts are copied into: - -- `/workspace/examples` - -## Build and Refresh - -Build: - -```bash -../hack/build-local-image.sh python-toolbox 0.2.0 ./python-toolbox -``` - -Import: - -```bash -../hack/import-local-image-to-k3s.sh python-toolbox 0.2.0 -``` - -Restart: - -```bash -kubectl rollout restart deploy/python-toolbox -n llm-observability -kubectl rollout status deploy/python-toolbox -n llm-observability -``` - -## Daily Usage - -Quick shell: - -```bash -kubectl exec -it -n llm-observability deploy/python-toolbox -- bash -``` - -Run individual scripts: - -```bash -kubectl exec -it -n llm-observability deploy/python-toolbox -- \ - python /workspace/examples/service_dns_check.py - -kubectl exec -it -n llm-observability deploy/python-toolbox -- \ - python /workspace/examples/ollama_smoke.py -``` - -Seed OpenTelemetry chart data: - -```bash -kubectl exec -it -n llm-observability deploy/python-toolbox -- \ - python /workspace/examples/otel_genai_inference_traces.py -``` - -Run the continuous 5-minute seeder only when you explicitly want chart activity: - -```bash -kubectl exec -it -n llm-observability deploy/python-toolbox -- \ - env OBS_CALL_COUNT_PER_CYCLE=4 OBS_INTERVAL_SECONDS=300 \ - python /workspace/examples/otel_genai_trace_seed_every_5m.py -``` - -## Operational Notes - -- This image is intentionally small and does not include JupyterLab. -- The notebooks use this pod as an in-cluster probe target, especially for DNS and service reachability drills. -- If a notebook says a script is missing, rebuild and re-import the image so the running deployment matches the local source tree. diff --git a/python-toolbox/examples/ollama_smoke.py b/python-toolbox/examples/ollama_smoke.py deleted file mode 100644 index 618e2ab..0000000 --- a/python-toolbox/examples/ollama_smoke.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -import time -import requests - -base_url = os.getenv("OLLAMA_BASE_URL", "http://ollama:11434") -model = os.getenv("OLLAMA_MODEL", "gemma3-1b-it-gguf-local") - -resp = requests.get(f"{base_url}/api/tags", timeout=10) -resp.raise_for_status() -print("models:") -print(resp.text) - -payload = { - "model": model, - "messages": [{"role": "user", "content": "Reply with one short sentence saying the stack is healthy."}], - "stream": False, -} -start = time.perf_counter() -resp = requests.post(f"{base_url}/api/chat", json=payload, timeout=60) -resp.raise_for_status() -elapsed = time.perf_counter() - start -print(f"chat completed in {elapsed:.2f}s") -print(resp.json()) diff --git a/python-toolbox/examples/otel_genai_inference_traces.py b/python-toolbox/examples/otel_genai_inference_traces.py deleted file mode 100644 index 6094fb0..0000000 --- a/python-toolbox/examples/otel_genai_inference_traces.py +++ /dev/null @@ -1,173 +0,0 @@ -import json -import os -import time -from typing import Any - -import requests -from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.trace import Span, Status, StatusCode - -DEFAULT_API_URL = "http://ollama:11434/api/chat" -DEFAULT_MODEL = "gemma3-1b-it-gguf-local" -DEFAULT_TIMEOUT_SECONDS = 180 -DEFAULT_CALL_COUNT = 12 -DEFAULT_OTLP_ENDPOINT = "http://opentelemetry-collector:4317" -TRACE_PREVIEW_LIMIT = 1500 - -DEFAULT_PROMPTS = [ - "Explain what Kubernetes readiness probe means in one short paragraph.", - "Give one example where GPU acceleration helps inference latency.", - "What should an OpenTelemetry GenAI span contain for local LLM inference?", - "Write three bullet points for debugging Ollama in k3s.", -] - - -def truncate(value: str, limit: int = TRACE_PREVIEW_LIMIT) -> str: - return value if len(value) <= limit else value[:limit] - - -def configure_tracing(endpoint: str) -> None: - provider = TracerProvider( - resource=Resource.create( - { - "service.name": os.getenv("OTEL_SERVICE_NAME", "otel-trace-seeder"), - "service.namespace": os.getenv("OTEL_SERVICE_NAMESPACE", "llm-observability"), - "deployment.environment": os.getenv("OTEL_DEPLOYMENT_ENVIRONMENT", "k3s-nvidia-edge"), - } - ) - ) - provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))) - trace.set_tracer_provider(provider) - - -def load_prompts() -> list[str]: - raw_json = os.getenv("OBS_PROMPTS_JSON") - if raw_json: - try: - payload = json.loads(raw_json) - except json.JSONDecodeError as exc: - raise SystemExit(f"OBS_PROMPTS_JSON is not valid JSON: {exc}") from exc - if not isinstance(payload, list) or not all(isinstance(item, str) and item.strip() for item in payload): - raise SystemExit("OBS_PROMPTS_JSON must be a non-empty JSON array of strings") - return [item.strip() for item in payload] - - raw_split = os.getenv("OBS_PROMPTS") - if raw_split: - prompts = [item.strip() for item in raw_split.split("||") if item.strip()] - if prompts: - return prompts - - return list(DEFAULT_PROMPTS) - - -def extract_answer(payload: dict[str, Any]) -> str: - message = payload.get("message") - if isinstance(message, dict): - content = message.get("content") - if isinstance(content, str): - return content - response = payload.get("response") - if isinstance(response, str): - return response - return json.dumps(payload, ensure_ascii=True)[:TRACE_PREVIEW_LIMIT] - - -def set_token_attributes(span: Span, payload: dict[str, Any]) -> None: - prompt_tokens = payload.get("prompt_eval_count") - generated_tokens = payload.get("eval_count") - if isinstance(prompt_tokens, (int, float)) and prompt_tokens >= 0: - span.set_attribute("gen_ai.usage.input_tokens", int(prompt_tokens)) - if isinstance(generated_tokens, (int, float)) and generated_tokens >= 0: - span.set_attribute("gen_ai.usage.output_tokens", int(generated_tokens)) - - -def seed_inference_runs( - model: str, - api_url: str, - timeout_seconds: int, - call_count: int, - prompts: list[str], -) -> tuple[int, int]: - tracer = trace.get_tracer("llm-observability-stack.otel-trace-seeder") - success = 0 - failures = 0 - - for index in range(call_count): - prompt = prompts[index % len(prompts)] - started = time.perf_counter() - with tracer.start_as_current_span( - "ollama seed chat", - attributes={ - "gen_ai.system": "ollama", - "gen_ai.operation.name": "chat", - "gen_ai.request.model": model, - "server.address": api_url, - "llm.request.prompt_preview": truncate(prompt), - "llm.seed.index": index + 1, - }, - ) as span: - try: - response = requests.post( - api_url, - json={ - "model": model, - "stream": False, - "messages": [{"role": "user", "content": prompt}], - }, - timeout=(10, timeout_seconds), - ) - span.set_attribute("http.response.status_code", response.status_code) - response.raise_for_status() - payload = response.json() - answer = extract_answer(payload) - elapsed_ms = (time.perf_counter() - started) * 1000 - set_token_attributes(span, payload) - span.set_attribute("gen_ai.response.model", model) - span.set_attribute("llm.response.latency_ms", round(elapsed_ms, 2)) - span.set_attribute("llm.response.preview", truncate(answer)) - success += 1 - print(f"[{index + 1}/{call_count}] OK latency_ms={elapsed_ms:.2f}") - except Exception as exc: - elapsed_ms = (time.perf_counter() - started) * 1000 - span.record_exception(exc) - span.set_status(Status(StatusCode.ERROR, str(exc))) - span.set_attribute("llm.response.latency_ms", round(elapsed_ms, 2)) - failures += 1 - print(f"[{index + 1}/{call_count}] ERROR error={exc}") - - print("\nCompleted OpenTelemetry GenAI trace seeding.") - print(f"Success: {success}") - print(f"Failed : {failures}") - return success, failures - - -def main() -> None: - endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", DEFAULT_OTLP_ENDPOINT) - model = os.getenv("OLLAMA_MODEL", DEFAULT_MODEL) - api_url = os.getenv("OBS_INFERENCE_API_URL", DEFAULT_API_URL).rstrip("/") - timeout_seconds = int(os.getenv("OBS_TIMEOUT_SECONDS", str(DEFAULT_TIMEOUT_SECONDS))) - call_count = int(os.getenv("OBS_CALL_COUNT", str(DEFAULT_CALL_COUNT))) - prompts = load_prompts() - - configure_tracing(endpoint) - print(f"OTLP endpoint : {endpoint}") - print(f"Inference API URL : {api_url}") - print(f"Model : {model}") - print(f"Calls : {call_count}") - print(f"Prompts loaded : {len(prompts)}") - - seed_inference_runs( - model=model, - api_url=api_url, - timeout_seconds=timeout_seconds, - call_count=call_count, - prompts=prompts, - ) - - -if __name__ == "__main__": - main() diff --git a/python-toolbox/examples/otel_genai_trace_seed_every_5m.py b/python-toolbox/examples/otel_genai_trace_seed_every_5m.py deleted file mode 100644 index 286abc9..0000000 --- a/python-toolbox/examples/otel_genai_trace_seed_every_5m.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -import time -from datetime import datetime, timezone - -from otel_genai_inference_traces import ( - DEFAULT_CALL_COUNT, - DEFAULT_MODEL, - DEFAULT_OTLP_ENDPOINT, - DEFAULT_TIMEOUT_SECONDS, - configure_tracing, - load_prompts, - seed_inference_runs, -) - -DEFAULT_INTERVAL_SECONDS = 300 - - -def now_utc() -> str: - return datetime.now(timezone.utc).isoformat() - - -def main() -> None: - endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", DEFAULT_OTLP_ENDPOINT) - model = os.getenv("OLLAMA_MODEL", DEFAULT_MODEL) - api_url = os.getenv("OBS_INFERENCE_API_URL", "http://ollama:11434/api/chat").rstrip("/") - timeout_seconds = int(os.getenv("OBS_TIMEOUT_SECONDS", str(DEFAULT_TIMEOUT_SECONDS))) - prompts = load_prompts() - - calls_per_cycle = int(os.getenv("OBS_CALL_COUNT_PER_CYCLE", os.getenv("OBS_CALL_COUNT", str(DEFAULT_CALL_COUNT)))) - interval_seconds = int(os.getenv("OBS_INTERVAL_SECONDS", str(DEFAULT_INTERVAL_SECONDS))) - max_cycles = int(os.getenv("OBS_MAX_CYCLES", "0")) - - configure_tracing(endpoint) - - print("Starting OpenTelemetry GenAI trace seed scheduler.") - print(f"Started at UTC : {now_utc()}") - print(f"OTLP endpoint : {endpoint}") - print(f"Inference API URL : {api_url}") - print(f"Model : {model}") - print(f"Calls per cycle : {calls_per_cycle}") - print(f"Interval (seconds) : {interval_seconds}") - print(f"Max cycles (0=inf) : {max_cycles}") - print(f"Prompts loaded : {len(prompts)}") - - cycle = 0 - while True: - cycle += 1 - cycle_started = time.time() - print(f"\n[{now_utc()}] Cycle {cycle} started") - - seed_inference_runs( - model=model, - api_url=api_url, - timeout_seconds=timeout_seconds, - call_count=calls_per_cycle, - prompts=prompts, - ) - - if max_cycles > 0 and cycle >= max_cycles: - print(f"[{now_utc()}] Reached OBS_MAX_CYCLES={max_cycles}. Exiting.") - break - - elapsed = time.time() - cycle_started - sleep_seconds = max(0, interval_seconds - int(elapsed)) - print(f"[{now_utc()}] Sleeping {sleep_seconds}s before next cycle.") - time.sleep(sleep_seconds) - - -if __name__ == "__main__": - main() diff --git a/python-toolbox/examples/redis_ping.py b/python-toolbox/examples/redis_ping.py deleted file mode 100644 index a1a893b..0000000 --- a/python-toolbox/examples/redis_ping.py +++ /dev/null @@ -1,7 +0,0 @@ -import os -import redis - -host = os.getenv("REDIS_HOST", "open-webui-redis") -port = int(os.getenv("REDIS_PORT", "6379")) -client = redis.Redis(host=host, port=port, socket_connect_timeout=3, socket_timeout=3) -print("PING:", client.ping()) diff --git a/python-toolbox/examples/service_dns_check.py b/python-toolbox/examples/service_dns_check.py deleted file mode 100644 index a93813d..0000000 --- a/python-toolbox/examples/service_dns_check.py +++ /dev/null @@ -1,95 +0,0 @@ -import os -import socket -import time -from urllib.parse import urlparse - -import requests - - -def resolve_port(default: int = 11434) -> int: - raw = os.getenv("OLLAMA_PORT") - if not raw: - return default - - # Kubernetes service env vars may look like: tcp://10.43.54.34:11434 - if raw.startswith("tcp://") or "://" in raw: - parsed = urlparse(raw) - if parsed.port: - return int(parsed.port) - - # Plain integer string case - try: - return int(raw) - except ValueError: - return default - - -def resolve_host(default: str = "ollama") -> str: - raw_host = os.getenv("OLLAMA_SERVICE") - if raw_host: - return raw_host - - # If Kubernetes injected OLLAMA_PORT like tcp://10.43.54.34:11434, use hostname from there - raw_port = os.getenv("OLLAMA_PORT") - if raw_port and "://" in raw_port: - parsed = urlparse(raw_port) - if parsed.hostname: - return parsed.hostname - - return default - - -def check_endpoint(name: str, host: str, port: int, path: str): - url = f"http://{host}:{port}{path}" - print(f"\n=== {name} ===") - print("Host:", host) - print("Port:", port) - print("URL :", url) - - try: - resolved_ip = socket.gethostbyname(host) - print("DNS :", resolved_ip) - except Exception as e: - print("DNS resolution failed:", repr(e)) - return - - try: - start = time.perf_counter() - response = requests.get(url, timeout=10) - elapsed = time.perf_counter() - start - print("HTTP status:", response.status_code) - print("Elapsed :", f"{elapsed:.3f}s") - body = response.text[:500].strip() - print("Body :", body if body else "") - except Exception as e: - print("HTTP request failed:", repr(e)) - - -if __name__ == "__main__": - ollama_host = resolve_host("ollama") - ollama_port = resolve_port(11434) - - targets = [ - ("Ollama tags", ollama_host, ollama_port, "/api/tags"), - ("Langchain demo health", "langchain-demo", 8000, "/healthz"), - ("Open WebUI root", "open-webui", 8080, "/"), - ("Redis ping via TCP check target", "open-webui-redis", 6379, "/"), - ] - - for name, host, port, path in targets: - # HTTP only for HTTP services - if port == 6379: - print(f"\n=== {name} ===") - print("Host:", host) - print("Port:", port) - try: - resolved_ip = socket.gethostbyname(host) - print("DNS :", resolved_ip) - sock = socket.create_connection((host, port), timeout=5) - print("TCP :", "connected") - sock.close() - except Exception as e: - print("TCP check failed:", repr(e)) - continue - - check_endpoint(name, host, port, path) diff --git a/python-toolbox/requirements.txt b/python-toolbox/requirements.txt deleted file mode 100644 index efca68f..0000000 --- a/python-toolbox/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -langchain>=0.3.0,<0.4.0 -langchain-ollama>=0.3.0,<0.4.0 -requests>=2.32.0,<3.0.0 -httpx>=0.28.0,<0.29.0 -dnspython>=2.7.0,<3.0.0 -redis>=6.0.0,<7.0.0 -opentelemetry-api>=1.34.0,<2.0.0 -opentelemetry-sdk>=1.34.0,<2.0.0 -opentelemetry-exporter-otlp-proto-grpc>=1.34.0,<2.0.0 diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index db29937..0000000 --- a/requirements-test.txt +++ /dev/null @@ -1,3 +0,0 @@ --r langchain-demo/requirements.txt -pytest==8.3.5 -httpx==0.28.1 diff --git a/templates/NOTES.txt b/templates/NOTES.txt index a62854d..fff2b5c 100644 --- a/templates/NOTES.txt +++ b/templates/NOTES.txt @@ -16,9 +16,9 @@ Endpoints after install: {{- if .Values.openWebUI.enabled }} - Open WebUI: http://open-webui:8080 {{- end }} -{{- if .Values.langchainDemo.enabled }} -- LangChain: http://langchain-demo:8000 -- Open WebUI traced proxy path: http://langchain-demo:8000/ollama/api/* +{{- if .Values.ollamaGateway.enabled }} +- LangChain: http://ollama-gateway:8000 +- Open WebUI traced proxy path: http://ollama-gateway:8000/ollama/api/* {{- end }} {{- if .Values.otelTraceSeeder.enabled }} - OpenTelemetry trace seeder CronJob: otel-trace-seeder (schedule: {{ .Values.otelTraceSeeder.schedule }}) diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index 802f669..0dd5828 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -2,11 +2,11 @@ {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{- define "llm-observability-stack.langchainServiceAccountName" -}} -{{- if .Values.langchainDemo.serviceAccount.create -}} -{{- default "langchain-demo" .Values.langchainDemo.serviceAccount.name -}} +{{- define "llm-observability-stack.ollamaGatewayServiceAccountName" -}} +{{- if .Values.ollamaGateway.serviceAccount.create -}} +{{- default "ollama-gateway" .Values.ollamaGateway.serviceAccount.name -}} {{- else -}} -{{- default "default" .Values.langchainDemo.serviceAccount.name -}} +{{- default "default" .Values.ollamaGateway.serviceAccount.name -}} {{- end -}} {{- end -}} diff --git a/templates/python-toolbox-deployment.yaml b/templates/edge-toolbox-deployment.yaml similarity index 52% rename from templates/python-toolbox-deployment.yaml rename to templates/edge-toolbox-deployment.yaml index acafd4d..24ebb94 100644 --- a/templates/python-toolbox-deployment.yaml +++ b/templates/edge-toolbox-deployment.yaml @@ -1,35 +1,35 @@ -{{- if .Values.pythonToolbox.enabled }} +{{- if .Values.edgeToolbox.enabled }} apiVersion: apps/v1 kind: Deployment metadata: - name: python-toolbox + name: edge-toolbox namespace: {{ include "llm-observability-stack.namespace" . }} spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: python-toolbox + app.kubernetes.io/name: edge-toolbox template: metadata: labels: - app.kubernetes.io/name: python-toolbox + app.kubernetes.io/name: edge-toolbox spec: -{{- if .Values.pythonToolbox.runtimeClassName }} - runtimeClassName: {{ .Values.pythonToolbox.runtimeClassName }} +{{- if .Values.edgeToolbox.runtimeClassName }} + runtimeClassName: {{ .Values.edgeToolbox.runtimeClassName }} {{- end }} containers: - name: toolbox - image: {{ .Values.pythonToolbox.image.repository }}:{{ .Values.pythonToolbox.image.tag }} - imagePullPolicy: {{ .Values.pythonToolbox.image.pullPolicy }} + image: {{ .Values.edgeToolbox.image.repository }}:{{ .Values.edgeToolbox.image.tag }} + imagePullPolicy: {{ .Values.edgeToolbox.image.pullPolicy }} command: -{{ toYaml .Values.pythonToolbox.command | indent 12 }} +{{ toYaml .Values.edgeToolbox.command | indent 12 }} env: - name: OLLAMA_BASE_URL value: http://ollama:11434 - name: OLLAMA_MODEL value: {{ .Values.ollamaModel.name | quote }} - name: OTEL_SERVICE_NAME - value: python-toolbox + value: edge-toolbox - name: OTEL_SERVICE_NAMESPACE value: {{ include "llm-observability-stack.namespace" . | quote }} - name: OTEL_EXPORTER_OTLP_ENDPOINT @@ -38,10 +38,24 @@ spec: value: {{ ternary (default "redis" .Values.redis.fullnameOverride) "open-webui-redis" .Values.redis.enabled | quote }} - name: REDIS_PORT value: "6379" -{{- range .Values.pythonToolbox.extraEnv }} +{{- range .Values.edgeToolbox.extraEnv }} - name: {{ .name }} value: {{ .value | quote }} {{- end }} resources: -{{ toYaml .Values.pythonToolbox.resources | indent 12 }} +{{ toYaml .Values.edgeToolbox.resources | indent 12 }} + ports: + - name: health + containerPort: 8080 + readinessProbe: + httpGet: + path: /healthz + port: health + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: health + initialDelaySeconds: 10 + periodSeconds: 20 {{- end }} diff --git a/templates/langchain-demo-configmap.yaml b/templates/langchain-demo-configmap.yaml deleted file mode 100644 index 96cd2fd..0000000 --- a/templates/langchain-demo-configmap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{- if and .Values.langchainDemo.enabled .Values.langchainDemo.appFromConfigMap.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: langchain-demo-app - namespace: {{ include "llm-observability-stack.namespace" . }} -data: - app.py: | -{{ .Files.Get "langchain-demo/app.py" | indent 4 }} -{{- end }} diff --git a/templates/langchain-demo-service.yaml b/templates/langchain-demo-service.yaml deleted file mode 100644 index e4a7fa8..0000000 --- a/templates/langchain-demo-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if .Values.langchainDemo.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: langchain-demo - namespace: {{ include "llm-observability-stack.namespace" . }} - labels: - app.kubernetes.io/name: langchain-demo - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - type: {{ .Values.langchainDemo.service.type }} - selector: - app.kubernetes.io/name: langchain-demo - ports: - - name: http - port: {{ .Values.langchainDemo.service.port }} - targetPort: http -{{- end }} diff --git a/templates/langchain-demo-deployment.yaml b/templates/ollama-gateway-deployment.yaml similarity index 73% rename from templates/langchain-demo-deployment.yaml rename to templates/ollama-gateway-deployment.yaml index b19759f..4787718 100644 --- a/templates/langchain-demo-deployment.yaml +++ b/templates/ollama-gateway-deployment.yaml @@ -1,5 +1,5 @@ -{{- if .Values.langchainDemo.enabled }} -{{- $probes := .Values.langchainDemo.probes | default dict -}} +{{- if .Values.ollamaGateway.enabled }} +{{- $probes := .Values.ollamaGateway.probes | default dict -}} {{- $startupProbe := get $probes "startup" | default dict -}} {{- $readinessProbe := get $probes "readiness" | default dict -}} {{- $livenessProbe := get $probes "liveness" | default dict -}} @@ -10,40 +10,30 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: langchain-demo + name: ollama-gateway namespace: {{ include "llm-observability-stack.namespace" . }} spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: langchain-demo + app.kubernetes.io/name: ollama-gateway template: metadata: labels: - app.kubernetes.io/name: langchain-demo + app.kubernetes.io/name: ollama-gateway spec: - serviceAccountName: {{ include "llm-observability-stack.langchainServiceAccountName" . }} + serviceAccountName: {{ include "llm-observability-stack.ollamaGatewayServiceAccountName" . }} automountServiceAccountToken: false securityContext: -{{ toYaml .Values.langchainDemo.podSecurityContext | indent 8 }} +{{ toYaml .Values.ollamaGateway.podSecurityContext | indent 8 }} containers: - name: app - image: {{ .Values.langchainDemo.image.repository }}:{{ .Values.langchainDemo.image.tag }} - imagePullPolicy: {{ .Values.langchainDemo.image.pullPolicy }} + image: {{ .Values.ollamaGateway.image.repository }}:{{ .Values.ollamaGateway.image.tag }} + imagePullPolicy: {{ .Values.ollamaGateway.image.pullPolicy }} securityContext: -{{ toYaml .Values.langchainDemo.securityContext | indent 12 }} - workingDir: /app +{{ toYaml .Values.ollamaGateway.securityContext | indent 12 }} command: - - python - - -m - - uvicorn - - app:app - - --host - - {{ .Values.langchainDemo.app.host | quote }} - - --port - - {{ .Values.langchainDemo.app.port | quote }} - - --workers - - {{ .Values.langchainDemo.app.workers | quote }} + - /usr/local/bin/ollama-gateway env: - name: OLLAMA_BASE_URL value: http://ollama:11434 @@ -55,13 +45,15 @@ spec: value: "/ollama" - name: OLLAMA_PROXY_TRACE_OTEL value: {{ ternary "true" "false" .Values.opentelemetry.tracing.enabled | quote }} - - name: OLLAMA_PROXY_TIMEOUT_SECONDS - value: "180" + - name: OLLAMA_PROXY_TIMEOUT + value: "180s" + - name: PORT + value: {{ .Values.ollamaGateway.app.port | quote }} - name: OTEL_TRACES_ENABLED value: {{ ternary "true" "false" .Values.opentelemetry.tracing.enabled | quote }} {{- if $otelEnabled }} - name: OTEL_SERVICE_NAME - value: langchain-demo + value: ollama-gateway - name: OTEL_SERVICE_NAMESPACE value: {{ include "llm-observability-stack.namespace" . | quote }} - name: OTEL_EXPORTER_OTLP_ENDPOINT @@ -75,15 +67,15 @@ spec: - name: ETCD_ENDPOINTS value: http://{{ include "llm-observability-stack.etcdFullname" . }}:{{ .Values.etcd.service.clientPort }} {{- end }} -{{- range .Values.langchainDemo.extraEnv }} +{{- range .Values.ollamaGateway.extraEnv }} - name: {{ .name }} value: {{ .value | quote }} {{- end }} ports: - name: http - containerPort: {{ .Values.langchainDemo.app.port }} + containerPort: {{ .Values.ollamaGateway.app.port }} resources: -{{ toYaml .Values.langchainDemo.resources | indent 12 }} +{{ toYaml .Values.ollamaGateway.resources | indent 12 }} {{- if (default true (get $startupProbe "enabled")) }} startupProbe: httpGet: @@ -114,15 +106,4 @@ spec: timeoutSeconds: {{ default 3 (get $livenessProbe "timeoutSeconds") }} failureThreshold: {{ default 6 (get $livenessProbe "failureThreshold") }} {{- end }} -{{- if .Values.langchainDemo.appFromConfigMap.enabled }} - volumeMounts: - - name: app-code - mountPath: /app/app.py - subPath: app.py - readOnly: true - volumes: - - name: app-code - configMap: - name: langchain-demo-app -{{- end }} {{- end }} diff --git a/templates/ollama-gateway-service.yaml b/templates/ollama-gateway-service.yaml new file mode 100644 index 0000000..b01bbf3 --- /dev/null +++ b/templates/ollama-gateway-service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.ollamaGateway.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: ollama-gateway + namespace: {{ include "llm-observability-stack.namespace" . }} + labels: + app.kubernetes.io/name: ollama-gateway + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + type: {{ .Values.ollamaGateway.service.type }} + selector: + app.kubernetes.io/name: ollama-gateway + ports: + - name: http + port: {{ .Values.ollamaGateway.service.port }} + targetPort: http +{{- end }} diff --git a/templates/langchain-demo-serviceaccount.yaml b/templates/ollama-gateway-serviceaccount.yaml similarity index 55% rename from templates/langchain-demo-serviceaccount.yaml rename to templates/ollama-gateway-serviceaccount.yaml index d88e4a8..ba50553 100644 --- a/templates/langchain-demo-serviceaccount.yaml +++ b/templates/ollama-gateway-serviceaccount.yaml @@ -1,8 +1,8 @@ -{{- if and .Values.langchainDemo.enabled .Values.langchainDemo.serviceAccount.create }} +{{- if and .Values.ollamaGateway.enabled .Values.ollamaGateway.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "llm-observability-stack.langchainServiceAccountName" . }} + name: {{ include "llm-observability-stack.ollamaGatewayServiceAccountName" . }} namespace: {{ include "llm-observability-stack.namespace" . }} automountServiceAccountToken: false {{- end }} diff --git a/templates/langchain-demo-servicemonitor.yaml b/templates/ollama-gateway-servicemonitor.yaml similarity index 82% rename from templates/langchain-demo-servicemonitor.yaml rename to templates/ollama-gateway-servicemonitor.yaml index 9c3bd34..bda4376 100644 --- a/templates/langchain-demo-servicemonitor.yaml +++ b/templates/ollama-gateway-servicemonitor.yaml @@ -1,15 +1,15 @@ -{{- if and .Values.monitoring.enabled .Values.monitoring.serviceMonitor.enabled .Values.langchainDemo.enabled }} +{{- if and .Values.monitoring.enabled .Values.monitoring.serviceMonitor.enabled .Values.ollamaGateway.enabled }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: langchain-demo + name: ollama-gateway namespace: {{ include "llm-observability-stack.namespace" . }} labels: {{ toYaml .Values.monitoring.labels | indent 4 }} spec: selector: matchLabels: - app.kubernetes.io/name: langchain-demo + app.kubernetes.io/name: ollama-gateway app.kubernetes.io/instance: {{ .Release.Name }} endpoints: - port: http diff --git a/templates/otel-trace-seeder-cronjob.yaml b/templates/otel-trace-seeder-cronjob.yaml index 0c45196..51df7e4 100644 --- a/templates/otel-trace-seeder-cronjob.yaml +++ b/templates/otel-trace-seeder-cronjob.yaml @@ -1,7 +1,7 @@ {{- if .Values.otelTraceSeeder.enabled }} -{{- $imageRepository := default .Values.pythonToolbox.image.repository .Values.otelTraceSeeder.image.repository -}} -{{- $imageTag := default .Values.pythonToolbox.image.tag .Values.otelTraceSeeder.image.tag -}} -{{- $imagePullPolicy := default .Values.pythonToolbox.image.pullPolicy .Values.otelTraceSeeder.image.pullPolicy -}} +{{- $imageRepository := default .Values.edgeToolbox.image.repository .Values.otelTraceSeeder.image.repository -}} +{{- $imageTag := default .Values.edgeToolbox.image.tag .Values.otelTraceSeeder.image.tag -}} +{{- $imagePullPolicy := default .Values.edgeToolbox.image.pullPolicy .Values.otelTraceSeeder.image.pullPolicy -}} apiVersion: batch/v1 kind: CronJob metadata: @@ -59,13 +59,4 @@ spec: {{- end }} resources: {{ toYaml .Values.otelTraceSeeder.resources | indent 16 }} - volumeMounts: - - name: seeder-scripts - mountPath: /opt/otel-trace-seeder - readOnly: true - volumes: - - name: seeder-scripts - configMap: - name: otel-trace-seeder-scripts - defaultMode: 0555 {{- end }} diff --git a/templates/otel-trace-seeder-scripts-configmap.yaml b/templates/otel-trace-seeder-scripts-configmap.yaml deleted file mode 100644 index c8cfce7..0000000 --- a/templates/otel-trace-seeder-scripts-configmap.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.otelTraceSeeder.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: otel-trace-seeder-scripts - namespace: {{ include "llm-observability-stack.namespace" . }} -data: - otel_genai_inference_traces.py: | -{{ .Files.Get "python-toolbox/examples/otel_genai_inference_traces.py" | indent 4 }} - otel_genai_trace_seed_every_5m.py: | -{{ .Files.Get "python-toolbox/examples/otel_genai_trace_seed_every_5m.py" | indent 4 }} -{{- end }} diff --git a/tests/helm_smoke_test.go b/tests/helm_smoke_test.go new file mode 100644 index 0000000..58ad433 --- /dev/null +++ b/tests/helm_smoke_test.go @@ -0,0 +1,144 @@ +package tests + +import ( + "archive/tar" + "compress/gzip" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func root(t *testing.T) string { + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("cannot locate test source") + } + return filepath.Dir(filepath.Dir(file)) +} +func helm(t *testing.T, args ...string) (string, error) { + t.Helper() + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is unavailable") + } + command := exec.Command("helm", args...) + command.Dir = root(t) + output, err := command.CombinedOutput() + return string(output), err +} +func requireContains(t *testing.T, value string, expected ...string) { + t.Helper() + for _, item := range expected { + if !strings.Contains(value, item) { + t.Errorf("render is missing %q", item) + } + } +} +func requireAbsent(t *testing.T, value string, forbidden ...string) { + t.Helper() + for _, item := range forbidden { + if strings.Contains(value, item) { + t.Errorf("render unexpectedly contains %q", item) + } + } +} + +func TestLocalProfileUsesNativeGoWorkloads(t *testing.T) { + manifest, err := helm(t, "template", "llm-observability-stack", ".", "-f", "values.local-k3s.example.yaml") + if err != nil { + t.Fatal(manifest) + } + requireContains(t, manifest, "name: ollama", "name: open-webui", "name: ollama-gateway", "name: edge-toolbox", "/usr/local/bin/ollama-gateway", "/usr/local/bin/edge-toolbox") + requireAbsent(t, manifest, "python", "uvicorn", "app.py", "langchain") +} + +func TestGeForceProfileContract(t *testing.T) { + manifest, err := helm(t, "template", "llm-observability-stack", ".", "-f", "values.geforce-940m-k3s.yaml") + if err != nil { + t.Fatal(manifest) + } + requireContains(t, manifest, "qwen-1.8b-chat-q4_K_M.gguf", "nvidia.com/gpu.present: \"true\"", "nvidia.com/gpu: 1", "OLLAMA_KEEP_ALIVE", "PARAMETER num_gpu 23", "PARAMETER num_ctx 256", "PARAMETER num_batch 1", "helm.sh/resource-policy: keep", "name: open-webui", "name: open-webui-redis", "name: llm-observability-dashboards", "edge-llm-observability.json", "DCGM_FI_DEV_MEM_COPY_UTIL", "kind: ServiceMonitor") + requireAbsent(t, manifest, "name: ollama-gateway", "name: edge-toolbox", "/bin/ollama rm", "kind: ClusterPolicy") +} + +func TestCPUProfileHasNoNVIDIAScheduling(t *testing.T) { + manifest, err := helm(t, "template", "llm-observability-stack", ".", "-f", "values.cpu-k3s.yaml") + if err != nil { + t.Fatal(manifest) + } + requireContains(t, manifest, "name: ollama", "name: ollama-gateway", "/usr/local/bin/ollama-gateway") + requireAbsent(t, manifest, "runtimeClassName: nvidia", "runtimeClassName: \"nvidia\"", "nvidia.com/gpu: 1", "nvidia.com/gpu.present", "python") +} + +func TestDestructiveModelCleanupIsRejected(t *testing.T) { + output, err := helm(t, "template", "llm-observability-stack", ".", "--set", "ollama.ollama.models.clean=true") + if err == nil || !strings.Contains(output, "models.clean must stay false") { + t.Fatalf("expected cleanup rejection, err=%v output=%s", err, output) + } +} +func TestBaseLayerChartsAreRejected(t *testing.T) { + for _, key := range []string{"gpu-operator.enabled=true", "nvidia-device-plugin.enabled=true", "dcgm-exporter.enabled=true"} { + output, err := helm(t, "template", "llm-observability-stack", ".", "--set", key) + if err == nil || !strings.Contains(output, "k3s-nvidia-edge") { + t.Fatalf("expected %s rejection, err=%v output=%s", key, err, output) + } + } +} +func TestSecretMismatchIsRejected(t *testing.T) { + output, err := helm(t, "template", "llm-observability-stack", ".", "--set", "openWebUI.existingSecret=legacy-secret", "--set", "open-webui.webuiSecret.existingSecretName=subchart-secret") + if err == nil || !strings.Contains(output, "Secret name mismatch") { + t.Fatalf("expected secret rejection, err=%v output=%s", err, output) + } +} + +func TestChartPackageIsSmallAndExcludesDevelopmentSources(t *testing.T) { + directory := t.TempDir() + output, err := helm(t, "package", ".", "-d", directory) + if err != nil { + t.Fatal(output) + } + matches, _ := filepath.Glob(filepath.Join(directory, "llm-observability-stack-*.tgz")) + if len(matches) != 1 { + t.Fatalf("package matches: %#v", matches) + } + info, err := os.Stat(matches[0]) + if err != nil { + t.Fatal(err) + } + if info.Size() >= 3_000_000 { + t.Fatalf("chart package is %d bytes", info.Size()) + } + file, err := os.Open(matches[0]) + if err != nil { + t.Fatal(err) + } + defer file.Close() + compressed, err := gzip.NewReader(file) + if err != nil { + t.Fatal(err) + } + defer compressed.Close() + archive := tar.NewReader(compressed) + names := map[string]bool{} + for { + header, err := archive.Next() + if err != nil { + break + } + names[header.Name] = true + } + for _, forbidden := range []string{"llm-observability-stack/.git/", "llm-observability-stack/docs/", "llm-observability-stack/tests/", "llm-observability-stack/internal/", "llm-observability-stack/ollama-gateway/", "llm-observability-stack/edge-toolbox/"} { + for name := range names { + if strings.HasPrefix(name, forbidden) { + t.Errorf("package contains %s", name) + } + } + } + for _, required := range []string{"llm-observability-stack/templates/ollama-gateway-deployment.yaml", "llm-observability-stack/templates/edge-toolbox-deployment.yaml", "llm-observability-stack/dashboards/edge-llm-observability.json"} { + if !names[required] { + t.Errorf("package is missing %s", required) + } + } +} diff --git a/tests/test_helm_smoke.py b/tests/test_helm_smoke.py deleted file mode 100644 index 7610a20..0000000 --- a/tests/test_helm_smoke.py +++ /dev/null @@ -1,382 +0,0 @@ -from __future__ import annotations - -import shutil -import subprocess -import tarfile -from pathlib import Path - -import pytest - - -REPO_ROOT = Path(__file__).resolve().parents[1] - - -def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]: - return subprocess.run( - cmd, - cwd=REPO_ROOT, - text=True, - capture_output=True, - check=False, - ) - - -def _combined_output(proc: subprocess.CompletedProcess[str]) -> str: - return f"{proc.stdout}\n{proc.stderr}".strip() - - -def _is_cluster_unreachable(proc: subprocess.CompletedProcess[str]) -> bool: - output = _combined_output(proc) - return "Kubernetes cluster unreachable" in output - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_helm_template_renders_core_resources() -> None: - render = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "-f", - "values.local-k3s.example.yaml", - ] - ) - assert render.returncode == 0, render.stderr or render.stdout - - manifest = render.stdout - assert "kind: Deployment" in manifest - assert "name: langchain-demo" in manifest - assert "name: ollama" in manifest - assert "name: open-webui" in manifest - assert "kind: StatefulSet" in manifest - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_helm_install_dry_run_client_succeeds() -> None: - namespace = "llm-observability-smoke-check" - install = _run( - [ - "helm", - "upgrade", - "--install", - "llm-observability-smoke", - ".", - "--namespace", - namespace, - "--create-namespace", - "--dry-run=client", - "--debug", - "-f", - "values.local-k3s.example.yaml", - "--set", - f"namespace.name={namespace}", - ] - ) - if install.returncode != 0 and _is_cluster_unreachable(install): - pytest.skip("Skipping install dry-run smoke test: Kubernetes cluster is unreachable in this environment.") - assert install.returncode == 0, _combined_output(install) - assert "llm-observability-smoke" in install.stdout - assert namespace in install.stdout - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_full_stack_nvidia_profile_renders_observability_resources() -> None: - render = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "-f", - "values.full-stack-nvidia.example.yaml", - "--set", - "opentelemetry.tracing.enabled=true", - "--set", - "openWebUI.existingSecret=", - "--set", - "open-webui.webuiSecret.existingSecretName=", - ] - ) - assert render.returncode == 0, _combined_output(render) - manifest = render.stdout - assert "kind: ServiceMonitor" in manifest - assert "kind: Probe" in manifest - assert 'url: "prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115"' in manifest - assert "url: http://prometheus-blackbox-exporter" not in manifest - assert "kind: PrometheusRule" in manifest - assert "name: llm-observability-dashboards" in manifest - assert "llm_observability_time_to_first_token_seconds" in manifest - assert "name: nvidia-device-plugin" not in manifest - assert "name: dcgm-exporter" not in manifest - assert "kind: ClusterPolicy" not in manifest - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_geforce_profile_uses_repository_modelfile_and_gpu_label() -> None: - render = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "-f", - "values.geforce-940m-k3s.yaml", - ] - ) - assert render.returncode == 0, _combined_output(render) - manifest = render.stdout - assert "qwen-1.8b-chat-q4_K_M.gguf" in manifest - assert "nvidia.com/gpu.present: \"true\"" in manifest - assert "node-role.kubernetes.io/worker" not in manifest - assert "nvidia.com/gpu: 1" in manifest - assert "OLLAMA_KEEP_ALIVE" in manifest - assert 'value: "-1"' in manifest - assert '/bin/ollama run qwen-1-8b-chat-q4-k-m-local "Reply with exactly: model ready"' in manifest - assert "PARAMETER num_gpu 23" in manifest - assert "PARAMETER num_ctx 256" in manifest - assert "PARAMETER num_batch 1" in manifest - assert 'helm.sh/resource-policy: keep' in manifest - assert "readOnly: true" in manifest - assert "name: open-webui" in manifest - assert "name: open-webui-redis" in manifest - assert "http://ollama:11434" in manifest - assert "http://langchain-demo:8000/ollama" not in manifest - assert "name: langchain-demo" not in manifest - assert "/bin/ollama rm" not in manifest - assert "name: nvidia-device-plugin" not in manifest - assert "name: dcgm-exporter" not in manifest - assert "kind: ClusterPolicy" not in manifest - - default_render = _run(["helm", "template", "llm-observability-stack", "."]) - assert default_render.returncode == 0, _combined_output(default_render) - assert "FROM /models/gguf/gemma-3-1b-it-gguf.gguf" in default_render.stdout - assert "/bin/ollama rm" not in default_render.stdout - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_cpu_profile_disables_nvidia_scheduling() -> None: - render = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "-f", - "values.cpu-k3s.yaml", - ] - ) - assert render.returncode == 0, _combined_output(render) - manifest = render.stdout - assert "name: ollama" in manifest - assert "runtimeClassName: \"nvidia\"" not in manifest - assert "runtimeClassName: nvidia" not in manifest - assert "nvidia.com/gpu: 1" not in manifest - assert "nvidia.com/gpu.present" not in manifest - assert "name: dcgm-exporter" not in manifest - assert "readOnly: true" in manifest - assert "/bin/ollama rm" not in manifest - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_generated_cpu_overlay_can_override_enterprise_gpu_profile() -> None: - render = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "-f", - "values.enterprise-pilot-k3s.yaml", - "--set", - "runtime.accelerator=cpu", - "--set", - "nvidia.runtimeClassName=", - "--set", - "nvidia.gpuCount=0", - "--set", - "dcgm-exporter.enabled=false", - "--set", - "monitoring.dcgmExporter.serviceMonitor.enabled=false", - "--set", - "ollama.runtimeClassName=", - "--set", - "ollama.nodeSelector=null", - "--set", - "ollama.ollama.gpu.enabled=false", - "--set", - "ollama.ollama.gpu.number=0", - ] - ) - assert render.returncode == 0, _combined_output(render) - manifest = render.stdout - assert "runtimeClassName: \"nvidia\"" not in manifest - assert "runtimeClassName: nvidia" not in manifest - assert "nvidia.com/gpu: 1" not in manifest - assert "nvidia.com/gpu.present" not in manifest - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_generated_nvidia_overlay_uses_gpu_resource_without_static_node_label() -> None: - render = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "-f", - "values.enterprise-pilot-k3s.yaml", - "--set", - "runtime.accelerator=nvidia", - "--set", - "nvidia.runtimeClassName=nvidia", - "--set", - "nvidia.gpuCount=1", - "--set", - "ollama.runtimeClassName=nvidia", - "--set", - "ollama.nodeSelector=null", - "--set", - "ollama.ollama.gpu.enabled=true", - "--set", - "ollama.ollama.gpu.number=1", - ] - ) - assert render.returncode == 0, _combined_output(render) - manifest = render.stdout - assert "runtimeClassName: \"nvidia\"" in manifest - assert "nvidia.com/gpu: 1" in manifest - assert "nvidia.com/gpu.present" not in manifest - assert "name: nvidia-device-plugin" not in manifest - assert "name: dcgm-exporter" not in manifest - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_model_cleanup_is_rejected() -> None: - render = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "--set", - "ollama.ollama.models.clean=true", - ] - ) - assert render.returncode != 0 - assert "models.clean must stay false" in _combined_output(render) - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_nvidia_profile_rejects_missing_startup_model_residency() -> None: - missing_run = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "-f", - "values.geforce-940m-k3s.yaml", - "--set", - "ollama.ollama.models.run={}", - ] - ) - assert missing_run.returncode != 0 - assert "ollama.ollama.models.run must include" in _combined_output(missing_run) - - finite_keep_alive = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "-f", - "values.geforce-940m-k3s.yaml", - "--set", - "ollama.extraEnv[0].name=OLLAMA_KEEP_ALIVE", - "--set", - "ollama.extraEnv[0].value=10m", - ] - ) - assert finite_keep_alive.returncode != 0 - assert "OLLAMA_KEEP_ALIVE" in _combined_output(finite_keep_alive) - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_base_layer_chart_enables_are_rejected() -> None: - for key in ("gpu-operator.enabled", "nvidia-device-plugin.enabled", "dcgm-exporter.enabled"): - render = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "--set", - f"{key}=true", - ] - ) - assert render.returncode != 0 - assert "k3s-nvidia-edge" in _combined_output(render) - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_helm_package_stays_below_secret_limit_budget(tmp_path: Path) -> None: - package = _run(["helm", "package", ".", "-d", str(tmp_path)]) - assert package.returncode == 0, _combined_output(package) - - archive = next(tmp_path.glob("llm-observability-stack-*.tgz")) - assert archive.stat().st_size < 3_000_000 - - with tarfile.open(archive, "r:gz") as tgz: - names = tgz.getnames() - - forbidden_prefixes = [ - "llm-observability-stack/.git/", - "llm-observability-stack/jupyter-notebooks-2/", - "llm-observability-stack/docs/", - "llm-observability-stack/tests/", - ] - for prefix in forbidden_prefixes: - assert not any(name.startswith(prefix) for name in names), prefix - - required_files = [ - "llm-observability-stack/langchain-demo/app.py", - "llm-observability-stack/Modelfile.gemma-3-1b-it-gguf", - "llm-observability-stack/Modelfile.qwen-1.8b-chat-q4_K_M", - "llm-observability-stack/dashboards/llm-overview.json", - "llm-observability-stack/dashboards/nvidia-gpu.json", - "llm-observability-stack/dashboards/benchmark-results.json", - "llm-observability-stack/python-toolbox/examples/otel_genai_inference_traces.py", - "llm-observability-stack/python-toolbox/examples/otel_genai_trace_seed_every_5m.py", - "llm-observability-stack/charts/kube-prometheus-stack/Chart.yaml", - "llm-observability-stack/charts/opentelemetry-collector/Chart.yaml", - "llm-observability-stack/charts/opentelemetry-operator/Chart.yaml", - ] - for required_file in required_files: - assert required_file in names, required_file - - removed_substrate_files = [ - "llm-observability-stack/charts/gpu-operator/Chart.yaml", - "llm-observability-stack/charts/nvidia-device-plugin/Chart.yaml", - "llm-observability-stack/charts/dcgm-exporter/Chart.yaml", - ] - for removed_file in removed_substrate_files: - assert removed_file not in names, removed_file - - -@pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary is not available") -def test_secret_wiring_validation_fails_on_mismatched_legacy_and_subchart_values() -> None: - render = _run( - [ - "helm", - "template", - "llm-observability-stack", - ".", - "--set", - "openWebUI.existingSecret=legacy-secret", - "--set", - "open-webui.webuiSecret.existingSecretName=subchart-secret", - ] - ) - assert render.returncode != 0 - assert "Secret name mismatch" in (render.stderr + render.stdout) diff --git a/tests/test_langchain_demo_smoke.py b/tests/test_langchain_demo_smoke.py deleted file mode 100644 index 692f46b..0000000 --- a/tests/test_langchain_demo_smoke.py +++ /dev/null @@ -1,203 +0,0 @@ -from __future__ import annotations - -import importlib.util -from pathlib import Path -from types import SimpleNamespace - -import pytest -from fastapi.testclient import TestClient - - -REPO_ROOT = Path(__file__).resolve().parents[1] -APP_PATH = REPO_ROOT / "langchain-demo" / "app.py" - -_SPEC = importlib.util.spec_from_file_location("langchain_demo_app", APP_PATH) -assert _SPEC is not None and _SPEC.loader is not None -app_module = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(app_module) - - -@pytest.fixture(autouse=True) -def reset_env_and_state(monkeypatch: pytest.MonkeyPatch) -> None: - # Keep each test deterministic and independent from host environment. - for name in [ - "OLLAMA_BASE_URL", - "OLLAMA_UPSTREAM_BASE_URL", - "OLLAMA_MODEL", - "OLLAMA_TEMPERATURE", - "OLLAMA_PROXY_TRACE_OTEL", - "OLLAMA_PROXY_TIMEOUT_SECONDS", - "OTEL_TRACES_ENABLED", - "OTEL_EXPORTER_OTLP_ENDPOINT", - "OTEL_SERVICE_NAME", - ]: - monkeypatch.delenv(name, raising=False) - monkeypatch.setenv("OTEL_TRACES_ENABLED", "false") - - -def test_root_and_healthz_endpoints() -> None: - with TestClient(app_module.app) as client: - root_resp = client.get("/") - assert root_resp.status_code == 200 - payload = root_resp.json() - assert payload["invoke"] == "/invoke" - assert payload["health"] == "/healthz" - assert payload["ollama_proxy"] == "/ollama/api/*" - - health_resp = client.get("/healthz") - assert health_resp.status_code == 200 - health = health_resp.json() - assert health["status"] == "ok" - assert health["ollama_base_url"] == "http://ollama:11434" - assert health["ollama_upstream_base_url"] == "http://ollama:11434" - - metrics_resp = client.get("/metrics") - assert metrics_resp.status_code == 200 - assert "llm_observability_http_requests_total" in metrics_resp.text - assert "llm_observability_llm_request_duration_seconds" in metrics_resp.text - - -def test_invoke_success_and_error_paths(monkeypatch: pytest.MonkeyPatch) -> None: - class FakeLLM: - def invoke(self, prompt: str) -> SimpleNamespace: - assert "User: hello from test" in prompt - return SimpleNamespace(content="synthetic-response") - - with TestClient(app_module.app) as client: - monkeypatch.setattr(app_module, "get_llm", lambda: FakeLLM()) - ok_resp = client.post("/invoke", json={"prompt": "hello from test", "system": "be brief"}) - assert ok_resp.status_code == 200 - assert ok_resp.json()["response"] == "synthetic-response" - - def raising_llm() -> FakeLLM: - raise RuntimeError("llm unavailable") - - monkeypatch.setattr(app_module, "get_llm", raising_llm) - err_resp = client.post("/invoke", json={"prompt": "hello from test"}) - assert err_resp.status_code == 500 - assert "llm unavailable" in err_resp.text - - -def test_ollama_proxy_forwards_non_streaming_requests(monkeypatch: pytest.MonkeyPatch) -> None: - captured: dict[str, object] = {} - - class FakeResponse: - status_code = 200 - headers = {"content-type": "application/json"} - content = b'{"ok":true}' - text = '{"ok":true}' - - @staticmethod - def json() -> dict[str, bool]: - return {"ok": True} - - class FakeAsyncClient: - def __init__(self, *args: object, **kwargs: object) -> None: - captured["client_init"] = kwargs - - async def __aenter__(self) -> "FakeAsyncClient": - return self - - async def __aexit__(self, exc_type, exc, tb) -> None: # type: ignore[no-untyped-def] - return None - - async def request(self, **kwargs: object) -> FakeResponse: - captured.update(kwargs) - return FakeResponse() - - monkeypatch.setenv("OLLAMA_UPSTREAM_BASE_URL", "http://ollama:11434") - monkeypatch.setattr(app_module.httpx, "AsyncClient", FakeAsyncClient) - - with TestClient(app_module.app) as client: - resp = client.post( - "/ollama/api/chat?trace_id=abc", - json={"stream": False, "messages": [{"role": "user", "content": "hello"}]}, - headers={"x-test-header": "smoke"}, - ) - - assert resp.status_code == 200 - assert resp.json() == {"ok": True} - assert captured["url"] == "http://ollama:11434/api/chat?trace_id=abc" - assert captured["method"] == "POST" - client_init = captured["client_init"] - assert isinstance(client_init, dict) - assert "timeout" in client_init - assert isinstance(captured["content"], (bytes, bytearray)) - headers = captured["headers"] - assert isinstance(headers, dict) - assert "host" not in {k.lower() for k in headers.keys()} - - -def test_ollama_proxy_forwards_streaming_requests(monkeypatch: pytest.MonkeyPatch) -> None: - captured: dict[str, object] = {} - - class FakeStreamResponse: - status_code = 200 - headers = {"content-type": "application/x-ndjson"} - - async def aiter_bytes(self, chunk_size: int = 8192): # type: ignore[no-untyped-def] - _ = chunk_size - for chunk in [b'{"token":"hello"}\n', b'{"token":"world"}\n']: - yield chunk - - async def aclose(self) -> None: - captured["response_closed"] = True - - class FakeAsyncClient: - def __init__(self, *args: object, **kwargs: object) -> None: - captured["client_init"] = kwargs - - def build_request(self, **kwargs: object) -> dict[str, object]: - captured["build_request"] = kwargs - return {"built": True, **kwargs} - - async def send(self, request: object, stream: bool = False) -> FakeStreamResponse: - captured["send_request"] = request - captured["send_stream"] = stream - return FakeStreamResponse() - - async def aclose(self) -> None: - captured["client_closed"] = True - - monkeypatch.setenv("OLLAMA_UPSTREAM_BASE_URL", "http://ollama:11434") - monkeypatch.setattr(app_module.httpx, "AsyncClient", FakeAsyncClient) - - with TestClient(app_module.app) as client: - resp = client.post( - "/ollama/api/chat", - json={"stream": True, "messages": [{"role": "user", "content": "hello"}]}, - ) - - assert resp.status_code == 200 - assert resp.content == b'{"token":"hello"}\n{"token":"world"}\n' - assert captured["send_stream"] is True - assert captured["response_closed"] is True - assert captured["client_closed"] is True - - with TestClient(app_module.app) as client: - metrics = client.get("/metrics").text - assert "llm_observability_time_to_first_token_seconds_count" in metrics - assert 'route="ollama_proxy"' in metrics - - -def test_ollama_proxy_surfaces_upstream_failures(monkeypatch: pytest.MonkeyPatch) -> None: - class FailingAsyncClient: - async def __aenter__(self) -> "FailingAsyncClient": - return self - - async def __aexit__(self, exc_type, exc, tb) -> None: # type: ignore[no-untyped-def] - return None - - async def request(self, **_: object) -> object: - raise app_module.httpx.RequestError( - "network-failure", - request=app_module.httpx.Request("POST", "http://ollama:11434/api/chat"), - ) - - monkeypatch.setattr(app_module.httpx, "AsyncClient", lambda *args, **kwargs: FailingAsyncClient()) - - with TestClient(app_module.app) as client: - resp = client.post("/ollama/api/chat", json={"stream": False}) - - assert resp.status_code == 502 - assert "network-failure" in resp.text diff --git a/validation/deployment-proof.md b/validation/deployment-proof.md index 2333cbf..ae7b119 100644 --- a/validation/deployment-proof.md +++ b/validation/deployment-proof.md @@ -24,7 +24,7 @@ helm list -A kubectl get pods,svc,pvc -n llm-observability -o wide kubectl get servicemonitors,probes,prometheusrules -A kubectl logs -n llm-observability deployment/ollama --tail=100 -kubectl logs -n llm-observability deployment/langchain-demo --tail=100 +kubectl logs -n llm-observability deployment/ollama-gateway --tail=100 kubectl logs -n llm-observability statefulset/open-webui --tail=100 ``` diff --git a/validation/local-run-report-2026-06-22.md b/validation/local-run-report-2026-06-22.md index 66717db..182c215 100644 --- a/validation/local-run-report-2026-06-22.md +++ b/validation/local-run-report-2026-06-22.md @@ -101,7 +101,7 @@ DCGM exporter was deployed and served metrics through `http://dcgm-exporter:9400 The following monitoring resources were present: - `ServiceMonitor/dcgm-exporter` -- `ServiceMonitor/langchain-demo` +- `ServiceMonitor/ollama-gateway` - `ServiceMonitor/opentelemetry-collector` - kube-prometheus-stack ServiceMonitors - `Probe/llm-stack-http` @@ -120,7 +120,7 @@ Open WebUI initially refused HTTP while downloading first-start Hugging Face ass Benchmark command: ```bash -./benchmarks/ollama_benchmark.py \ +bin/llm-observability benchmark \ --model gemma3-1b-it-gguf-local \ --warmup-runs 1 \ --runs 3 \ diff --git a/validation/pilot-report.md b/validation/pilot-report.md index e43a0d6..abc0eed 100644 --- a/validation/pilot-report.md +++ b/validation/pilot-report.md @@ -4,7 +4,7 @@ This report summarizes the current local technical validation of `llm-observabil ## Summary -`llm-observability-stack` deploys local LLM inference and observability components on k3s/Kubernetes. The verified local environment is Xubuntu 24 with k3s, NVIDIA GPU scheduling, Ollama, Open WebUI, a LangChain-compatible FastAPI proxy, OpenTelemetry Collector, Prometheus, Grafana, Alertmanager, DCGM exporter, and diagnostic workloads. +`llm-observability-stack` deploys local LLM inference and observability components on k3s/Kubernetes. The verified local environment is Xubuntu 24 with k3s, NVIDIA GPU scheduling, Ollama, Open WebUI, a native Go Ollama gateway, OpenTelemetry Collector, Prometheus, Grafana, Alertmanager, DCGM exporter, and diagnostic workloads. ## Verified Local Environment diff --git a/validation/stack-one-page-summary.md b/validation/stack-one-page-summary.md index 07fcab5..39d75fe 100644 --- a/validation/stack-one-page-summary.md +++ b/validation/stack-one-page-summary.md @@ -1,6 +1,6 @@ # llm-observability-stack One-Page Summary -`llm-observability-stack` is a Helm-based local LLM observability stack for k3s and Kubernetes. It packages Ollama/GGUF model serving, Open WebUI, a LangChain-compatible FastAPI proxy, Prometheus metrics, Grafana dashboards, OpenTelemetry Collector, endpoint probes, benchmark tooling, and optional NVIDIA GPU monitoring. +`llm-observability-stack` is a Helm-based local LLM observability stack for k3s and Kubernetes. It packages Ollama/GGUF model serving, Open WebUI, a native Go Ollama gateway, Prometheus metrics, Grafana dashboards, OpenTelemetry Collector, endpoint probes, benchmark tooling, and optional NVIDIA GPU monitoring. ## Problem @@ -12,7 +12,7 @@ The stack provides one reproducible deployment path for local inference and obse - Ollama serves local GGUF models. - Open WebUI provides browser access. -- `langchain-demo` exposes a FastAPI proxy with LLM metrics. +- `ollama-gateway` exposes a native Go Ollama gateway with LLM metrics. - Prometheus and Grafana collect and visualize application, cluster, benchmark, and GPU metrics. - OpenTelemetry Collector accepts OTLP telemetry. - CPU-only and NVIDIA GPU profiles support development, local validation, and edge deployment. @@ -24,7 +24,7 @@ The stack provides one reproducible deployment path for local inference and obse - `nvidia.com/gpu: 1` advertised by the node. - GPU Operator deployed separately in namespace `gpu-operator`. - `llm-observability-stack` deployed in namespace `llm-observability`. -- Full local stack pods Running, including Ollama, Open WebUI, LangChain proxy, OpenTelemetry Collector, Prometheus, Grafana, Alertmanager, DCGM exporter, and Python toolbox. +- Full local stack pods Running, including Ollama, Open WebUI, Ollama gateway, OpenTelemetry Collector, Prometheus, Grafana, Alertmanager, DCGM exporter, and Go edge toolbox. ## Key References diff --git a/values.cpu-k3s.yaml b/values.cpu-k3s.yaml index 853717d..466235b 100644 --- a/values.cpu-k3s.yaml +++ b/values.cpu-k3s.yaml @@ -128,16 +128,12 @@ ollama: - name: OLLAMA_CONTEXT_LENGTH value: "1024" -langchainDemo: +ollamaGateway: enabled: true image: - repository: langchain-demo - tag: 0.1.1 + repository: ollama-gateway + tag: 0.2.0 pullPolicy: IfNotPresent - app: - workers: 1 - appFromConfigMap: - enabled: true service: type: ClusterIP port: 8000 @@ -152,7 +148,7 @@ langchainDemo: openWebUI: enabled: true auth: false - ollamaBaseUrl: http://langchain-demo:8000/ollama + ollamaBaseUrl: http://ollama-gateway:8000/ollama webuiSecretKey: replace-with-a-32-plus-char-secret service: type: ClusterIP @@ -165,10 +161,10 @@ open-webui: port: 8080 containerPort: 8080 ollamaUrls: - - http://langchain-demo:8000/ollama + - http://ollama-gateway:8000/ollama extraEnvVars: - name: OLLAMA_BASE_URL - value: http://langchain-demo:8000/ollama + value: http://ollama-gateway:8000/ollama - name: ENABLE_PERSISTENT_CONFIG value: "False" - name: WEBUI_AUTH @@ -189,10 +185,10 @@ open-webui: redis: enabled: false -pythonToolbox: +edgeToolbox: enabled: true image: - repository: python-toolbox + repository: edge-toolbox tag: 0.2.0 pullPolicy: IfNotPresent diff --git a/values.enterprise-pilot-k3s.yaml b/values.enterprise-pilot-k3s.yaml index 638be05..1a0884a 100644 --- a/values.enterprise-pilot-k3s.yaml +++ b/values.enterprise-pilot-k3s.yaml @@ -127,16 +127,12 @@ ollama: - name: OLLAMA_CONTEXT_LENGTH value: "1024" -langchainDemo: +ollamaGateway: enabled: true image: - repository: langchain-demo - tag: 0.1.1 + repository: ollama-gateway + tag: 0.2.0 pullPolicy: IfNotPresent - app: - workers: 1 - appFromConfigMap: - enabled: true service: type: ClusterIP port: 8000 @@ -151,7 +147,7 @@ langchainDemo: openWebUI: enabled: true auth: false - ollamaBaseUrl: http://langchain-demo:8000/ollama + ollamaBaseUrl: http://ollama-gateway:8000/ollama webuiSecretKey: replace-with-a-32-plus-char-secret service: type: ClusterIP @@ -164,10 +160,10 @@ open-webui: port: 8080 containerPort: 8080 ollamaUrls: - - http://langchain-demo:8000/ollama + - http://ollama-gateway:8000/ollama extraEnvVars: - name: OLLAMA_BASE_URL - value: http://langchain-demo:8000/ollama + value: http://ollama-gateway:8000/ollama - name: ENABLE_PERSISTENT_CONFIG value: "False" - name: WEBUI_AUTH @@ -188,10 +184,10 @@ open-webui: redis: enabled: false -pythonToolbox: +edgeToolbox: enabled: true image: - repository: python-toolbox + repository: edge-toolbox tag: 0.2.0 pullPolicy: IfNotPresent diff --git a/values.full-stack-nvidia.example.yaml b/values.full-stack-nvidia.example.yaml index fca6997..14ad8d2 100644 --- a/values.full-stack-nvidia.example.yaml +++ b/values.full-stack-nvidia.example.yaml @@ -83,7 +83,7 @@ openWebUI: open-webui: extraEnvVars: - name: OLLAMA_BASE_URL - value: http://langchain-demo:8000/ollama + value: http://ollama-gateway:8000/ollama - name: ENABLE_PERSISTENT_CONFIG value: "False" - name: WEBUI_AUTH diff --git a/values.geforce-940m-k3s.yaml b/values.geforce-940m-k3s.yaml index 2e34b0c..2203b93 100644 --- a/values.geforce-940m-k3s.yaml +++ b/values.geforce-940m-k3s.yaml @@ -3,10 +3,107 @@ namespace: name: llm-observability monitoring: - enabled: false + enabled: true + blackbox: + # This compact local profile does not deploy a blackbox exporter. + enabled: false + dcgmExporter: + serviceMonitor: + enabled: true + namespace: gpu-operator + interval: 15s + port: gpu-metrics + path: /metrics + selector: + app: nvidia-dcgm-exporter + grafanaDashboards: + enabled: true networkPolicy: enabled: false +# A resource-conscious Prometheus/Grafana deployment for the validated +# single-node GeForce 940M profile. The dashboards under dashboards/*.json are +# discovered by the Grafana sidecar from the root chart's ConfigMap. +kube-prometheus-stack: + enabled: true + crds: + enabled: true + defaultRules: + create: false + alertmanager: + enabled: false + grafana: + enabled: true + defaultDashboardsEnabled: false + adminUser: admin + adminPassword: admin + persistence: + enabled: false + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + labelValue: "1" + searchNamespace: ALL + provider: + allowUiUpdates: false + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + prometheus: + enabled: true + prometheusSpec: + retention: 6h + retentionSize: 1GB + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + probeSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + scrapeConfigSelectorNilUsesHelmValues: false + resources: + requests: + cpu: 100m + memory: 384Mi + limits: + cpu: 1000m + memory: 1536Mi + prometheusOperator: + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 300m + memory: 256Mi + kube-state-metrics: + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 300m + memory: 256Mi + prometheus-node-exporter: + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + kubeEtcd: + enabled: false + kubeControllerManager: + enabled: false + kubeScheduler: + enabled: false + kubeProxy: + enabled: false + nvidia: runtimeClassName: nvidia gpuResourceName: nvidia.com/gpu @@ -165,9 +262,9 @@ open-webui: value: "False" redis: enabled: false -langchainDemo: +ollamaGateway: enabled: false -pythonToolbox: +edgeToolbox: enabled: false otelTraceSeeder: enabled: false diff --git a/values.local-k3s.example.yaml b/values.local-k3s.example.yaml index e34277e..5f48100 100644 --- a/values.local-k3s.example.yaml +++ b/values.local-k3s.example.yaml @@ -94,13 +94,11 @@ ollama: - name: OLLAMA_CONTEXT_LENGTH value: "1024" -langchainDemo: +ollamaGateway: image: - repository: langchain-demo - tag: 0.1.1 + repository: ollama-gateway + tag: 0.2.0 pullPolicy: IfNotPresent - app: - workers: 2 probes: startup: timeoutSeconds: 3 @@ -112,8 +110,6 @@ langchainDemo: initialDelaySeconds: 60 timeoutSeconds: 3 failureThreshold: 6 - appFromConfigMap: - enabled: true service: type: ClusterIP port: 8000 @@ -122,7 +118,7 @@ etcd: enabled: false openWebUI: - ollamaBaseUrl: http://langchain-demo:8000/ollama + ollamaBaseUrl: http://ollama-gateway:8000/ollama webuiSecretKey: "replace-with-a-32-plus-char-secret" service: type: ClusterIP @@ -139,24 +135,24 @@ open-webui: port: 8080 containerPort: 8080 ollamaUrls: - - http://langchain-demo:8000/ollama + - http://ollama-gateway:8000/ollama extraEnvVars: - name: OLLAMA_BASE_URL - value: http://langchain-demo:8000/ollama + value: http://ollama-gateway:8000/ollama - name: ENABLE_PERSISTENT_CONFIG value: "False" - name: WEBUI_AUTH value: "False" -pythonToolbox: +edgeToolbox: enabled: true image: - repository: python-toolbox + repository: edge-toolbox tag: 0.2.0 pullPolicy: IfNotPresent command: - - sleep - - infinity + - /usr/local/bin/edge-toolbox + - serve otelTraceSeeder: enabled: false diff --git a/values.validation-k3s.yaml b/values.validation-k3s.yaml index 9ab477e..88a9e31 100644 --- a/values.validation-k3s.yaml +++ b/values.validation-k3s.yaml @@ -13,9 +13,9 @@ open-webui: enabled: false redis: enabled: false -langchainDemo: +ollamaGateway: enabled: false -pythonToolbox: +edgeToolbox: enabled: false otelTraceSeeder: enabled: false diff --git a/values.yaml b/values.yaml index 1320004..b461702 100644 --- a/values.yaml +++ b/values.yaml @@ -31,7 +31,7 @@ monitoring: module: http_2xx interval: 30s targets: - - 'http://langchain-demo.{{ .Values.namespace.name }}.svc.cluster.local:8000/healthz' + - 'http://ollama-gateway.{{ .Values.namespace.name }}.svc.cluster.local:8000/healthz' - 'http://ollama.{{ .Values.namespace.name }}.svc.cluster.local:11434/api/tags' - 'http://open-webui.{{ .Values.namespace.name }}.svc.cluster.local:8080/health' prometheusRules: @@ -400,7 +400,7 @@ openWebUI: enabled: true fullnameOverride: open-webui # Route Open WebUI through the OpenTelemetry-instrumented Ollama proxy. - ollamaBaseUrl: http://langchain-demo:8000/ollama + ollamaBaseUrl: http://ollama-gateway:8000/ollama # Leave empty to auto-generate a strong value when chart-managed secret mode is used. # If you provide a value here, it is used as-is. webuiSecretKey: "" @@ -440,7 +440,7 @@ open-webui: redis: enabled: true ollamaUrls: - - http://langchain-demo:8000/ollama + - http://ollama-gateway:8000/ollama enableOpenaiApi: false openaiBaseApiUrl: "" openaiApiKey: "" @@ -470,7 +470,7 @@ open-webui: memory: 2Gi extraEnvVars: - name: OLLAMA_BASE_URL - value: http://langchain-demo:8000/ollama + value: http://ollama-gateway:8000/ollama - name: ENABLE_PERSISTENT_CONFIG value: "False" - name: WEBUI_AUTH @@ -528,9 +528,9 @@ redis: cpu: 500m memory: 512Mi -langchainDemo: +ollamaGateway: enabled: true - nameOverride: langchain-demo + nameOverride: ollama-gateway serviceAccount: create: true name: "" @@ -548,14 +548,11 @@ langchainDemo: drop: - ALL image: - repository: langchain-demo - tag: 0.1.1 + repository: ollama-gateway + tag: 0.2.0 pullPolicy: IfNotPresent app: - host: 0.0.0.0 port: 8000 - # prometheus-client uses an in-process registry; keep one worker unless multiprocess mode is configured. - workers: 1 probes: startup: enabled: true @@ -580,9 +577,6 @@ langchainDemo: periodSeconds: 20 timeoutSeconds: 3 failureThreshold: 6 - appFromConfigMap: - # Mount langchain-demo/app.py from chart files so proxy/tracing code can be updated without rebuilding image. - enabled: true service: type: ClusterIP port: 8000 @@ -661,16 +655,16 @@ opentelemetry: - debug -pythonToolbox: +edgeToolbox: enabled: true runtimeClassName: "" image: - repository: python-toolbox + repository: edge-toolbox tag: 0.2.0 pullPolicy: IfNotPresent command: - - sleep - - infinity + - /usr/local/bin/edge-toolbox + - serve resources: requests: cpu: 50m @@ -682,7 +676,7 @@ pythonToolbox: otelTraceSeeder: enabled: false - # Runs a short Python job every 5 minutes to keep OpenTelemetry GenAI traces flowing. + # Runs a short native Go job every 5 minutes to keep OpenTelemetry GenAI traces flowing. schedule: "*/5 * * * *" suspend: false concurrencyPolicy: Forbid @@ -697,8 +691,8 @@ otelTraceSeeder: tag: "" pullPolicy: "" command: - - python - - /opt/otel-trace-seeder/otel_genai_inference_traces.py + - /usr/local/bin/edge-toolbox + - seed env: inferenceApiUrl: http://ollama:11434/api/chat callCount: "2"