From 3d97d28a9ba246843a25766ea38b1dfdc853702b Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sat, 4 Jul 2026 17:20:13 +0200 Subject: [PATCH 01/11] feat: migrate Tempo to 3.0 Per the official monolithic migration: ingester and compactor blocks removed (live-store/backend-worker handle both), local-blocks processor dropped, traces_storage removed; block retention (7d) moves under overrides.defaults.compaction. Boot-tested standalone, then verified in place against the live stack: existing 2.x blocks readable, traces queryable, span metrics regenerating. Dependabot's Tempo major-version ignore is removed now the migration is done. --- .github/dependabot.yml | 5 ----- compose.yml | 6 +----- config/tempo.yaml | 20 ++++++++------------ docs/RUNBOOK.md | 1 - 4 files changed, 9 insertions(+), 23 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4149d6c..6d6cc41 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,11 +8,6 @@ updates: monitoring-stack-updates: patterns: - "*" - ignore: - # Tempo 3.x is a re-architecture requiring a config migration; upgrade - # deliberately, not via a routine bump. See comment in compose.yml. - - dependency-name: "grafana/tempo" - update-types: ["version-update:semver-major"] - package-ecosystem: "docker" directory: "/demo" diff --git a/compose.yml b/compose.yml index 43238ee..21dc661 100644 --- a/compose.yml +++ b/compose.yml @@ -43,11 +43,7 @@ services: logging: *default-logging tempo: - # Held at 2.x: Tempo 3.0 is a re-architecture (ingester/compactor removed, - # metrics-generator changed) needing a real config migration — see - # https://grafana.com/docs/tempo/latest/set-up-for-tracing/setup-tempo/migrate-to-3/ - # Dependabot ignores 3.x majors until that migration is done deliberately. - image: grafana/tempo:2.10.4 + image: grafana/tempo:3.0.2 restart: unless-stopped volumes: - ./config/tempo.yaml:/etc/tempo/tempo.yaml:ro diff --git a/config/tempo.yaml b/config/tempo.yaml index ba7b78b..245ef57 100644 --- a/config/tempo.yaml +++ b/config/tempo.yaml @@ -1,5 +1,8 @@ -# Tempo accepts OTLP directly. We only forward traces *from* the collector +# Tempo 3.x single-binary. Traces arrive only from the collector # (collector → tempo:4317) so external apps never talk to Tempo directly. +# Migrated from 2.x: the ingester/compactor blocks and the local-blocks +# processor are gone (handled internally by the live-store/backend worker); +# block retention now lives under overrides. server: http_listen_port: 3200 @@ -12,13 +15,6 @@ distributor: grpc: endpoint: 0.0.0.0:4317 -ingester: - max_block_duration: 5m - -compactor: - compaction: - block_retention: 168h # 7 days — traces are bulky, tune to disk budget - storage: trace: backend: local @@ -28,7 +24,7 @@ storage: path: /var/tempo/blocks # Derive RED metrics (requests/errors/duration) + a service graph from traces. -# These land in Prometheus and power Grafana's built-in Service Graph panel. +# These land in Prometheus and power the Service Health dashboard. metrics_generator: registry: external_labels: @@ -38,13 +34,13 @@ metrics_generator: remote_write: - url: http://prometheus:9090/api/v1/write send_exemplars: true - traces_storage: - path: /var/tempo/generator/traces overrides: defaults: metrics_generator: - processors: [service-graphs, span-metrics, local-blocks] + processors: [service-graphs, span-metrics] + compaction: + block_retention: 168h # 7 days — traces are bulky, tune to disk budget usage_report: reporting_enabled: false diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 4d1624a..cf29ec6 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -89,4 +89,3 @@ on the host: `git pull && just pull && just up`. The `just check` validators (promtool, otelcol, amtool) read their image versions from `compose.yml`, so every bump is validated with the exact binaries the stack will run — CI fails loudly when config syntax drifts between versions, which is the point. -Tempo majors are deliberately held back (see the comment in `compose.yml`). From afb2dde980eed113726439b66f2eda16b96e144e Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sat, 4 Jul 2026 17:20:13 +0200 Subject: [PATCH 02/11] feat: put Cloudflare Access in front of Grafana MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Email one-time-PIN at the edge for the public Grafana hostname, allowed addresses via the grafana_allowed_emails variable. The OTLP hostname stays outside Access — machines authenticate with the bearer token. Validated against the provider v5 schema. --- infra/main.tf | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/infra/main.tf b/infra/main.tf index 20fbd5c..83f1c06 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -39,6 +39,11 @@ variable "domain" { description = "Apex domain, e.g. example.org → grafana.example.org, otlp.example.org." } +variable "grafana_allowed_emails" { + type = list(string) + description = "Email addresses allowed through Cloudflare Access to Grafana (one-time PIN)." +} + resource "cloudflare_zero_trust_tunnel_cloudflared" "monitoring" { account_id = var.account_id name = "monitoring" @@ -86,6 +91,33 @@ resource "cloudflare_dns_record" "otlp" { ttl = 1 } +# Cloudflare Access in front of Grafana: email one-time-PIN at the edge, so +# the public hostname never reaches Grafana's login page unauthenticated. +# The OTLP hostname is NOT behind Access — machines authenticate with the +# bearer token instead. +resource "cloudflare_zero_trust_access_application" "grafana" { + account_id = var.account_id + name = "Grafana (monitoring)" + domain = "grafana.${var.domain}" + type = "self_hosted" + session_duration = "24h" + + policies = [{ + id = cloudflare_zero_trust_access_policy.grafana_emails.id + precedence = 1 + }] +} + +resource "cloudflare_zero_trust_access_policy" "grafana_emails" { + account_id = var.account_id + name = "monitoring: allowed emails" + decision = "allow" + + include = [for email in var.grafana_allowed_emails : { + email = { email = email } + }] +} + data "cloudflare_zero_trust_tunnel_cloudflared_token" "monitoring" { account_id = var.account_id tunnel_id = cloudflare_zero_trust_tunnel_cloudflared.monitoring.id From eb15d34f884a3d3589ce4cc804f5a29c1ace560b Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sat, 4 Jul 2026 17:22:39 +0200 Subject: [PATCH 03/11] feat: add Logs Overview dashboard, alerts table, and home dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Logs Overview covers the OTLP log path (volume by service and level, error tail, live tail — verified against live data); Stack Health gains an Active Alerts table (Watchdog excluded, it fires by design) and becomes Grafana's home dashboard so operators land on stack state instead of the welcome page. --- CHANGELOG.md | 16 +++-- README.md | 9 +-- compose.yml | 2 + dashboards/logs-overview.json | 130 ++++++++++++++++++++++++++++++++++ dashboards/stack-health.json | 37 ++++++++++ 5 files changed, 183 insertions(+), 11 deletions(-) create mode 100644 dashboards/logs-overview.json diff --git a/CHANGELOG.md b/CHANGELOG.md index d255bbc..7c38e27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,18 +22,20 @@ Reliability hardening: the stack becomes operable, not just runnable. - `docs/RUNBOOK.md` and `docs/ONBOARDING.md` with verified telemetry templates (zero-code Python, plain OTLP, Loki Docker driver, Grafana Alloy). -- `infra/`: OpenTofu for the Cloudflare tunnel, ingress routes, and DNS. +- `infra/`: OpenTofu for the Cloudflare tunnel, ingress routes, and DNS, + plus Cloudflare Access (email one-time PIN) in front of Grafana. +- **Logs Overview** dashboard for OTLP-ingested logs (volume by service + and level, errors, live tail); Stack Health gains an Active Alerts + table and becomes the Grafana home dashboard. ### Changed - Prometheus storage now bounded by size as well as time (`--storage.tsdb.retention.size=15GB`). - -### Fixed - -- Tempo held at 2.x: a routine bump to 3.0 crash-looped on the 2.x config - (3.0 is a re-architecture). Dependabot now skips Tempo majors so the - migration happens deliberately. +- **Tempo migrated to 3.0** (after a routine bump initially crash-looped on + the 2.x config): ingester/compactor blocks removed per the monolithic + migration guide, retention moved under overrides, verified in place + against live traffic. ## [0.1.0] - 2026-07-03 diff --git a/README.md b/README.md index a32f8c6..05bf26b 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,8 @@ Grafana at (admin / change-me): - **Dashboards → Service Health (RED)** — request rate, error rate, and latency for the demo service. Latency dots are exemplars: click one to open that exact trace in Tempo. -- **Explore → Loki** — `{service_name="demo-api"}`; expand an error line - and follow its `trace_id` to the trace. +- **Dashboards → Logs Overview** — log volume by service and level, + errors, and a live tail for everything the stack ingests over OTLP. - **Alerting → Alert rules** — stack-health and error-rate rules, evaluated by Prometheus. @@ -88,8 +88,9 @@ just up-tunnel # core stack + Cloudflare Tunnel (needs CLOUDFLARE_TUNNE Grafana: (admin / whatever you set). -The Cloudflare side of the tunnel (hostnames, DNS, the tunnel itself) is -code too: `infra/` holds a small OpenTofu config whose output is the +The Cloudflare side of the tunnel (hostnames, DNS, the tunnel itself, and +Cloudflare Access gating Grafana behind email one-time-PIN) is code too: +`infra/` holds a small OpenTofu config whose output is the `CLOUDFLARE_TUNNEL_TOKEN` the overlay needs — bootstrap instructions in [infra/main.tf](infra/main.tf). diff --git a/compose.yml b/compose.yml index 21dc661..e66b70b 100644 --- a/compose.yml +++ b/compose.yml @@ -119,6 +119,8 @@ services: GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env} GF_USERS_ALLOW_SIGN_UP: "false" GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3000} + # Land on Stack Health instead of the empty welcome page. + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/stack-health.json logging: *default-logging volumes: diff --git a/dashboards/logs-overview.json b/dashboards/logs-overview.json new file mode 100644 index 0000000..36eb78c --- /dev/null +++ b/dashboards/logs-overview.json @@ -0,0 +1,130 @@ +{ + "title": "Logs Overview", + "uid": "logs-overview", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-3h", "to": "now" }, + "graphTooltip": 1, + "tags": ["logs", "otlp"], + "templating": { + "list": [ + { + "name": "service", + "label": "Service", + "type": "query", + "datasource": { "type": "loki", "uid": "loki" }, + "definition": "label_values(service_name)", + "query": { "type": "labelValues", "label": "service_name" }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { "text": "All", "value": "$__all" }, + "options": [], + "hide": 0 + } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "Log Rate by Service", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "id": 1, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { + "expr": "sum by (service_name) (rate({service_name=~\"$service\"}[$__auto]))", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "logs/s", + "custom": { "lineWidth": 2, "fillOpacity": 10 } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Log Rate by Level", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "id": 2, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { + "expr": "sum by (detected_level) (rate({service_name=~\"$service\"}[$__auto]))", + "legendFormat": "{{detected_level}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "logs/s", + "custom": { "lineWidth": 2, "fillOpacity": 10 } + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "(?i)error" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] + }, + { + "matcher": { "id": "byRegexp", "options": "(?i)(critical|fatal)" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "dark-red" } }] + }, + { + "matcher": { "id": "byRegexp", "options": "(?i)warn.*" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }] + }, + { + "matcher": { "id": "byRegexp", "options": "(?i)info" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] + } + ] + } + }, + { + "type": "logs", + "title": "Errors", + "description": "Error-and-worse log lines. Expand one and follow trace_id to the trace in Tempo.", + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 8 }, + "id": 3, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { + "expr": "{service_name=~\"$service\", detected_level=~\"(?i)(error|critical|fatal)\"}", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending" + } + }, + { + "type": "logs", + "title": "All Logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 17 }, + "id": 4, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { + "expr": "{service_name=~\"$service\"}", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending" + } + } + ] +} diff --git a/dashboards/stack-health.json b/dashboards/stack-health.json index c8f74cf..422bfc0 100644 --- a/dashboards/stack-health.json +++ b/dashboards/stack-health.json @@ -134,6 +134,43 @@ }, "overrides": [] } + }, + { + "type": "table", + "title": "Active Alerts", + "description": "Pending and firing alerts (Watchdog excluded — it fires by design).", + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 16 }, + "id": 6, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "ALERTS{alertname!=\"Watchdog\"}", + "instant": true, + "format": "table", + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { "Time": true, "Value": true, "__name__": true }, + "indexByName": { "alertname": 0, "alertstate": 1, "severity": 2 } + } + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { "id": "byName", "options": "alertstate" }, + "properties": [ + { "id": "custom.cellOptions", "value": { "type": "color-text" } }, + { "id": "mappings", "value": [{ "type": "value", "options": { "firing": { "color": "red" }, "pending": { "color": "orange" } } }] } + ] + } + ] + } } ] } From 317efdf07a536156cc791e2d98430a491a73cd41 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sun, 5 Jul 2026 17:10:47 +0200 Subject: [PATCH 04/11] docs: add terraform.tfvars.example for the infra variables The .env.example equivalent for infra/: the four input variables (account_id, zone_id, domain, grafana_allowed_emails) with a note that the API token comes from CLOUDFLARE_API_TOKEN, not tfvars. Un-ignore *.tfvars.example so it's tracked while real tfvars stay ignored. --- .gitignore | 1 + infra/main.tf | 3 ++- infra/terraform.tfvars.example | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 infra/terraform.tfvars.example diff --git a/.gitignore b/.gitignore index d29ad9d..6968e63 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ backups/ infra/.terraform/ infra/terraform.tfstate* infra/*.tfvars +!infra/*.tfvars.example diff --git a/infra/main.tf b/infra/main.tf index 83f1c06..fcf2d72 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -3,7 +3,8 @@ # click-ops in the Zero Trust dashboard. # # Bootstrap (owner-run, once): -# export CLOUDFLARE_API_TOKEN=... # needs Tunnel:Edit, DNS:Edit +# cp terraform.tfvars.example terraform.tfvars # then fill it in +# export CLOUDFLARE_API_TOKEN=... # needs Tunnel:Edit, DNS:Edit, Access:Edit # cd infra && tofu init && tofu apply # tofu output -raw tunnel_token # → CLOUDFLARE_TUNNEL_TOKEN in ../.env # diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example new file mode 100644 index 0000000..5ef4f33 --- /dev/null +++ b/infra/terraform.tfvars.example @@ -0,0 +1,15 @@ +# Copy to terraform.tfvars (gitignored) and fill in. The Cloudflare API token +# is NOT set here — export it: `export CLOUDFLARE_API_TOKEN=...` +# (needs Tunnel:Edit, DNS:Edit, and Access: Apps and Policies:Edit). + +# Cloudflare account ID (dash.cloudflare.com → any domain → overview sidebar). +account_id = "your-account-id" + +# Zone ID of the domain the hostnames live under (same overview sidebar). +zone_id = "your-zone-id" + +# Apex domain → grafana., otlp.. +domain = "example.org" + +# Emails allowed through Cloudflare Access to Grafana (one-time PIN). +grafana_allowed_emails = ["you@example.org"] From c692ba194254c69b5ff98ac5a55f9c1fb5bc5ea2 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sun, 5 Jul 2026 17:38:33 +0200 Subject: [PATCH 05/11] docs: editorial pass for a broader CML audience Defines jargon on first use (OTLP, RED, exemplars, cardinality, dead man's switch), breaks up dense sentences, and keeps the technical detail. No factual changes. --- README.md | 121 ++++++++++++++++----------- docs/ONBOARDING.md | 65 ++++++++------ docs/RUNBOOK.md | 108 ++++++++++++++---------- docs/adr/0001-observability-stack.md | 6 +- 4 files changed, 173 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 05bf26b..e2ea2b8 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,23 @@ # Observability stack -One host running Grafana, Loki, Tempo, Prometheus, and an OpenTelemetry -Collector, wired so logs, traces, and metrics cross-link. Point any number of -projects at it over OTLP and their telemetry lands in one place. +Central monitoring for CML's research software. One host runs Grafana, +Loki, Tempo, Prometheus, and an OpenTelemetry Collector, wired together so +logs, traces, and metrics cross-reference each other. Projects send their +telemetry here over OTLP (the OpenTelemetry protocol) and it lands in one +place, queryable side by side. ## Architecture -Everything enters through one gateway — the OTel Collector — so a project -configures a single OTLP endpoint and a backend can be swapped without -touching any app. Tempo derives RED metrics from spans, so a service that -only sends traces still gets dashboards and error alerting. It all runs on -one host on purpose: at CML's telemetry volume, distributed ingest would add -operational weight for no gain. Rationale and alternatives: +Everything enters through a single gateway, the OpenTelemetry Collector. A +project only ever configures one endpoint, and we can swap a storage +backend later without touching any application. Tempo also derives +request-rate, error-rate, and duration ("RED") metrics from the traces it +receives, so a service that sends nothing but traces still gets a working +dashboard and error alerting. + +The stack deliberately runs on a single host. At CML's telemetry volume, +distributed ingestion would add operational weight for no gain. The +reasoning, and the alternatives we considered, are recorded in [ADR 0001](docs/adr/0001-observability-stack.md). ```mermaid @@ -31,9 +37,10 @@ flowchart LR end ``` -Solid arrows are the write path; dotted arrows are Grafana reading at query -time. Locally (`just up` / `just demo`) there is no tunnel — everything -talks over the compose network and Grafana is on `localhost:3000`. +Solid arrows show telemetry being written; dotted arrows show Grafana +reading at query time. Locally (`just up` or `just demo`) there is no +tunnel involved: everything talks over the compose network and Grafana is +at `localhost:3000`. ## Demo: see it work in one command @@ -42,21 +49,24 @@ cp .env.example .env just demo ``` -This starts the full stack plus a small auto-instrumented FastAPI service -under constant load (`compose.demo.yml`). Give it a minute, then open -Grafana at (admin / change-me): +This starts the full stack plus a small FastAPI service under constant +artificial load (`compose.demo.yml`). The service is instrumented with +OpenTelemetry auto-instrumentation and fails about one request in ten, on +purpose. Give it a minute, then open Grafana at +(admin / change-me) and look at: - **Dashboards → Service Health (RED)** — request rate, error rate, and - latency for the demo service. Latency dots are exemplars: click one to - open that exact trace in Tempo. -- **Dashboards → Logs Overview** — log volume by service and level, - errors, and a live tail for everything the stack ingests over OTLP. -- **Alerting → Alert rules** — stack-health and error-rate rules, - evaluated by Prometheus. + latency. The dots on the latency panel are exemplars: click one and + Grafana opens the exact trace behind that measurement. +- **Dashboards → Logs Overview** — log volume by service and level, an + error feed, and a live tail of everything arriving over OTLP. +- **Alerting → Alert rules** — the stack-health and error-rate rules + Prometheus is evaluating. ![Service Health (RED) dashboard](docs/img/service-health.png) -Tear it down with `just demo-down`. +`just demo-down` removes the demo services again; the rest of the stack +keeps running. ## Layout @@ -83,48 +93,57 @@ infra/ # OpenTofu: Cloudflare tunnel, ingress routes, DNS ```sh cp .env.example .env # set GRAFANA_ADMIN_PASSWORD just up # core stack, local only -just up-tunnel # core stack + Cloudflare Tunnel (needs CLOUDFLARE_TUNNEL_TOKEN) +just up-tunnel # production: core stack + Cloudflare Tunnel ``` Grafana: (admin / whatever you set). -The Cloudflare side of the tunnel (hostnames, DNS, the tunnel itself, and -Cloudflare Access gating Grafana behind email one-time-PIN) is code too: -`infra/` holds a small OpenTofu config whose output is the -`CLOUDFLARE_TUNNEL_TOKEN` the overlay needs — bootstrap instructions in -[infra/main.tf](infra/main.tf). +In production the stack sits behind a Cloudflare Tunnel, and that edge is +code too. The tunnel, its hostnames, DNS, and the Cloudflare Access rule +that puts an email one-time-PIN in front of Grafana all live in `infra/` +as a small OpenTofu configuration. Applying it produces the +`CLOUDFLARE_TUNNEL_TOKEN` that `just up-tunnel` needs; bootstrap steps are +at the top of [infra/main.tf](infra/main.tf). -`just check` validates everything (compose files, Prometheus config and -alert rules, collector config, YAML, workflows, dashboard JSON) in pinned -containers — no host installs. CI runs the same command on every push and -PR, plus a smoke test that boots the stack and waits for Grafana to report -healthy. +`just check` validates the whole repository: compose files, Prometheus +config and alert rules, collector config, YAML, workflows, and dashboard +JSON. Every validator runs in a pinned container, so nothing needs to be +installed on the host. CI runs the same command on every push and pull +request, plus a smoke test that boots the stack and waits for Grafana to +come up healthy. ## Sending telemetry from a project -One OTLP endpoint (`:4317` gRPC / `:4318` HTTP), bearer-token auth -(`OTLP_AUTH_TOKEN`), and three naming conventions. Copy-paste templates for -every ingestion form — zero-code Python/FastAPI, plain OTLP env vars, the -Loki Docker driver, and Grafana Alloy for file logs — live in -**[docs/ONBOARDING.md](docs/ONBOARDING.md)**. +You need three things: the OTLP endpoint, the bearer token +(`OTLP_AUTH_TOKEN`), and a few naming conventions. Copy-paste templates +for each route — zero-code Python/FastAPI, plain OTLP environment +variables, the Loki Docker driver, and Grafana Alloy for log files — are +in **[docs/ONBOARDING.md](docs/ONBOARDING.md)**. -Never publish 4317/4318 directly; the compose file binds them to -`127.0.0.1` and the tunnel is the exposure path. +Never publish ports 4317/4318 to the internet. The compose file binds them +to `127.0.0.1`; the tunnel is the way in. ## Alerting -Prometheus evaluates the rules in `config/alerts/` (target down, OTel -export failures, >5% span error rate, disk >80%); Alertmanager delivers -them to any webhook via `ALERT_WEBHOOK_URL`, and the always-firing -`Watchdog` posts to `HEARTBEAT_URL` every 5 minutes — point that at a dead -man's switch so you hear about it when the monitoring host itself dies. -Both are optional; unset means alerts are visible in Grafana only. -Operations (rotating tokens, disk pressure, backup/restore): -**[docs/RUNBOOK.md](docs/RUNBOOK.md)**. +Prometheus evaluates the rules in `config/alerts/`: scrape target down, +OTel export failures, error rate above 5%, disk above 80%. Alertmanager +delivers them to whatever webhook you set in `ALERT_WEBHOOK_URL` (ntfy, +Slack, and so on). + +One rule, `Watchdog`, fires permanently by design and posts to +`HEARTBEAT_URL` every five minutes. Point that at a dead man's switch such +as healthchecks.io — a service that alerts when the pings *stop* — and you +will also hear about the one failure the host cannot report itself: its own +death. Both variables are optional; leave them unset and alerts are simply +visible in Grafana. + +Day-to-day operations (rotating tokens, disk pressure, backup and restore) +are covered in **[docs/RUNBOOK.md](docs/RUNBOOK.md)**. ## Storage Everything persists to local Docker volumes (`loki_data`, `tempo_data`, -`prometheus_data`, `grafana_data`). Swap Loki / Tempo storage to S3-compatible -(Backblaze B2, Cloudflare R2, Hetzner, MinIO) when you outgrow local disk — -`compose.storage-s3.yml` documents the concrete shape of that change. +`prometheus_data`, `grafana_data`). When local disk stops fitting, Loki +and Tempo can move to any S3-compatible object store (Backblaze B2, +Cloudflare R2, Hetzner, MinIO); `compose.storage-s3.yml` documents the +concrete shape of that change. diff --git a/docs/ONBOARDING.md b/docs/ONBOARDING.md index c1705ae..e6e0a88 100644 --- a/docs/ONBOARDING.md +++ b/docs/ONBOARDING.md @@ -1,7 +1,10 @@ # Sending telemetry from your project -One endpoint, one token, three conventions — then pick the template that -matches how your project runs. +This stack collects logs, traces, and metrics from CML projects and shows +them side by side in one Grafana. Getting your project in takes an +endpoint, a token, and a couple of naming conventions. Then pick the +template that matches how your project runs — if it's a Python/FastAPI +service, Template 1 needs no code changes at all. ## The endpoint @@ -11,23 +14,27 @@ matches how your project runs. | Private network / same host | `:4317` (gRPC) or `:4318` (HTTP) | | Auth | `Authorization: Bearer ` (ask the stack operator) | -The tunnel routes only HTTPS to the collector's HTTP receiver — there is no -public gRPC path, so set `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` when -sending through it. gRPC works only on private paths (VPN/WireGuard, same -Docker network). Never expose 4317/4318 directly. +The tunnel only routes HTTPS to the collector's HTTP receiver; there is no +public gRPC path. When sending through it, set +`OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`. gRPC works on private paths +only (VPN/WireGuard, or the same Docker network). Never expose 4317/4318 +directly. ## The conventions -- **`service.name`** — required; one stable name per deployable unit - (`relab-api`, not `relab-api-prod-2`). This is what dashboards key on. -- **`env`** — `prod`, `staging`, `dev`, set via resource attributes. -- **Keep label cardinality low** — no user IDs, request IDs, or timestamps - in resource attributes or log labels; those belong in log lines and span - attributes, where they're query-time filters. +- **`service.name`** is required: one stable name per deployable unit + (`relab-api`, not `relab-api-prod-2`). Dashboards key on it. +- **`env`** is `prod`, `staging`, or `dev`, set as a resource attribute. +- **Keep labels low-cardinality.** Loki and Prometheus index labels, and + every distinct value creates a new stream or series. User IDs, request + IDs, and timestamps therefore don't belong in resource attributes or log + labels. Put them in the log line or in span attributes instead — you can + still filter on them at query time, without the storage blowing up. -Any service that sends **traces** automatically gets RED metrics, the -Service Health dashboard, and error-rate alerting — Tempo derives them from -spans. Send traces first; everything else is a bonus. +Traces are the most valuable signal to send. Tempo derives request-rate, +error-rate, and duration ("RED") metrics from them, so a service that +sends only traces already gets the Service Health dashboard and error +alerting. Start with traces; everything else is a bonus. ## Template 1 — Python/FastAPI, zero code changes @@ -47,13 +54,14 @@ export OTEL_SEMCONV_STABILITY_OPT_IN=http opentelemetry-instrument uvicorn app:app --host 0.0.0.0 --port 8000 ``` -Traces, RED metrics, and logs with trace context — no OTel code in the app. -The living example is this repo's own [`demo/`](../demo/) + +That's the whole integration: traces, RED metrics, and logs that carry +their trace context, without a line of OTel code in the app. The working +example is this repo's own [`demo/`](../demo/) service plus [`compose.demo.yml`](../compose.demo.yml). ## Template 2 — any language, plain OTLP -Every OTel SDK understands the same four env vars: +Every OpenTelemetry SDK understands the same four environment variables: ```sh OTEL_SERVICE_NAME=my-service @@ -64,11 +72,12 @@ OTEL_RESOURCE_ATTRIBUTES=env=prod ## Template 3 — Docker container logs (Loki driver) -For shipping container stdout/stderr without touching the app. Requires a -Loki push URL, which **this stack does not expose by default** — Loki has no -auth, so a push hostname must first be added to `infra/main.tf` and protected -(Cloudflare Access service token), or reached over a private network path -(VPN/WireGuard). If in doubt, use the OTLP log path above instead. +This ships container stdout/stderr without touching the app. It needs a +Loki push URL, which **this stack does not expose by default**: Loki has +no authentication of its own, so a push hostname must first be added to +`infra/main.tf` and protected with a Cloudflare Access service token, or +reached over a private network path (VPN/WireGuard). If in doubt, use the +OTLP log path from Templates 1–2 instead. ```sh # once per host @@ -84,12 +93,13 @@ logging: loki-external-labels: service={{.Name}},env=prod,host=myhost ``` -RELab's `compose.logging.loki.yml` overlay (auto-included when `LOKI_URL` -is set) is the reference implementation of this pattern. +RELab's `compose.logging.loki.yml` overlay, auto-included when `LOKI_URL` +is set, is the reference implementation of this pattern. ## Template 4 — host or file logs (Grafana Alloy) -For log files outside containers (Promtail is EOL; Alloy is its successor): +For log files that live outside containers. (Promtail is end-of-life; +Alloy is its successor.) ```alloy // alloy/config.alloy @@ -120,4 +130,5 @@ alloy: - /var/log/myapp:/var/log/myapp:ro ``` -Same caveat as Template 3: the Loki push URL needs a protected network path. +The same caveat as Template 3 applies: the Loki push URL needs a protected +network path. diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index cf29ec6..9bf3fb9 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -1,16 +1,16 @@ # Runbook -Operational procedures for this stack. Commands run from the repo root on -the monitoring host. `just ps` and the **Stack Health** dashboard are the -first stop for anything. +How to operate this stack. All commands run from the repo root on the +monitoring host. Start with `just ps` and the **Stack Health** dashboard; +between them they answer most "what is wrong" questions. ![Stack Health dashboard](img/stack-health.png) -(That capture is honest: the red export-failure spike is a Tempo outage -after a bad major-version bump, and the ingest gap is a backup/restore -drill — both procedures below.) +That capture is honest, not staged: the red export-failure spike is a real +Tempo outage after a bad major-version bump, and the gap in the ingest +panel is a backup/restore drill. Both procedures are below. -## A service is down / misbehaving +## A service is down or misbehaving ```sh just ps # what's running, what's restarting @@ -18,13 +18,15 @@ just logs # follow logs (otel-collector, loki, tempo, prometheu just restart ``` -`TargetDown` fires after 2 minutes if Prometheus can't scrape the collector, -node-exporter, or itself. `OtelExportFailures` means the collector is up but -a backend is rejecting data — check that backend's logs. +Two alerts point here. `TargetDown` fires after two minutes when +Prometheus can't scrape the collector, node-exporter, or itself. +`OtelExportFailures` means the collector is up but a backend is rejecting +its data, so look at that backend's logs, not the collector's. ## Disk filling up (`HostDiskSpaceLow`) -Retention is only partially size-bounded by design: +Retention is only partially size-bounded, and that's by design rather than +oversight: | Data | Time limit | Size limit | | --- | --- | --- | @@ -33,12 +35,14 @@ Retention is only partially size-bounded by design: | Loki chunks | 30d | none — Loki cannot cap total size | | Tempo blocks | 7d | none | -So when the 80% alert fires: check the Filesystem panel on Stack Health, -then either free space or shorten retention — `retention_period` in -`config/loki.yaml`, `block_retention` in `config/tempo.yaml`, the -`--storage.tsdb.retention.*` flags in `compose.yml` — and restart the -affected service. If disk pressure is chronic, move Loki/Tempo to object -storage: `compose.storage-s3.yml`. +Loki and Tempo simply have no total-size knob, which is why the disk alert +at 80% is the real backstop. When it fires: check the Filesystem panel on +Stack Health, then either free space or shorten a retention window +(`retention_period` in `config/loki.yaml`, `block_retention` in +`config/tempo.yaml`, the `--storage.tsdb.retention.*` flags in +`compose.yml`) and restart the affected service. If disk pressure keeps +coming back, the durable fix is moving Loki and Tempo to object storage — +see `compose.storage-s3.yml`. ## Backup and restore @@ -48,44 +52,56 @@ just restore backups/monitoring-.tar.gz # stops the stack, wipes volumes, just up ``` -Backups are crash-consistent (equivalent to a power loss; every component -recovers via its WAL). Tarballs are mode 0600 and contain secrets (Grafana -DB); copy them off-host over a private channel — a backup on the disk it -protects is a decoration. RPO = however often you run it; a daily cron on -the host is the intended setup. During the pause the collector keeps -accepting telemetry and retries for ~5 minutes; a backup that takes longer -than that will drop data, so on large volumes run it at a quiet hour. +Backups are crash-consistent: restoring one is equivalent to recovering +from a power loss, which every component in the stack does cleanly via its +write-ahead log. Two practical notes: + +- The tarball is mode 0600 and contains secrets (the Grafana database + among them). Copy it off-host over a private channel — a backup on the + disk it protects is a decoration. +- During the pause the collector keeps accepting telemetry and buffers it + for about five minutes. A backup that takes longer than that will drop + data, so on large volumes run it at a quiet hour. + +How much history you can lose equals how often you run it. A daily cron on +the host is the intended setup. ## Rotating secrets -- **OTLP token:** new value for `OTLP_AUTH_TOKEN` in `.env` → - `docker compose up -d otel-collector` → update every sender's - `OTEL_EXPORTER_OTLP_HEADERS`. Senders with the old token get 401s (visible - as their export errors) until updated. That includes a running demo - overlay — re-run `just demo` to recreate it with the new token. -- **Tunnel token:** rotate in Cloudflare Zero Trust → new - `CLOUDFLARE_TUNNEL_TOKEN` in `.env` → `just up-tunnel`. -- **Grafana admin password:** `GRAFANA_ADMIN_PASSWORD` in `.env` → - `docker compose up -d grafana`. +- **OTLP token:** set the new `OTLP_AUTH_TOKEN` in `.env`, then + `docker compose up -d otel-collector`, then update every sender's + `OTEL_EXPORTER_OTLP_HEADERS`. Senders still on the old token get 401s + (visible as export errors on their side) until they're updated. A + running demo overlay counts as a sender: re-run `just demo` to recreate + it with the new token. +- **Tunnel token:** rotate in Cloudflare Zero Trust, set the new + `CLOUDFLARE_TUNNEL_TOKEN` in `.env`, then `just up-tunnel`. +- **Grafana admin password:** change `GRAFANA_ADMIN_PASSWORD` in `.env`, + then `docker compose up -d grafana`. ## Alert delivery -Prometheus evaluates rules → Alertmanager delivers: +Prometheus evaluates the rules; Alertmanager delivers them. Two +environment variables control where: -- `ALERT_WEBHOOK_URL` — all alerts (any webhook receiver: ntfy, Slack, …). -- `HEARTBEAT_URL` — the always-firing `Watchdog` posts here every 5 minutes. - Point it at a dead man's switch (e.g. healthchecks.io) that alerts when - pings **stop**: that is the "monitoring host is dead" signal nothing - inside the host can send. +- `ALERT_WEBHOOK_URL` receives all alerts (any webhook: ntfy, Slack, …). +- `HEARTBEAT_URL` receives the always-firing `Watchdog` every five + minutes. Point it at a dead man's switch (e.g. healthchecks.io) that + raises the alarm when pings **stop** — that is the "monitoring host is + dead" signal nothing inside the host can send. -Both empty = no delivery; Alertmanager logs a notify error per cycle, which -is expected and harmless. After changing either, `docker compose up -d +Leaving both empty is fine: nothing is delivered, Alertmanager logs one +notify error per cycle (expected, harmless), and alerts remain visible in +Grafana. After changing either variable, `docker compose up -d alertmanager`. ## Upgrading images -Dependabot PRs bump the pins. For each: CI runs `just check`; after merge, -on the host: `git pull && just pull && just up`. The `just check` validators -(promtool, otelcol, amtool) read their image versions from `compose.yml`, so -every bump is validated with the exact binaries the stack will run — CI -fails loudly when config syntax drifts between versions, which is the point. +Dependabot opens PRs that bump the pinned versions, and CI runs `just +check` on each one. The validators (promtool, otelcol, amtool) read their +image versions from `compose.yml`, so every bump is checked with the exact +binaries the stack will run — when a new version changes its config +syntax, CI fails loudly before the change reaches the host. That is the +point. + +After merging: on the host, `git pull && just pull && just up`. diff --git a/docs/adr/0001-observability-stack.md b/docs/adr/0001-observability-stack.md index b69f096..bf0357c 100644 --- a/docs/adr/0001-observability-stack.md +++ b/docs/adr/0001-observability-stack.md @@ -18,12 +18,12 @@ endpoints via Cloudflare Tunnel; bind everything else to `127.0.0.1`. - **OTLP-native:** one endpoint, one protocol. Swap a backend or move it to a hosted equivalent without touching any application. -- **Single-host:** one compose file one person can audit and rebuild. +- **Single-host:** one compose file that one person can audit and rebuild. Distributed ingest (Kubernetes, Mimir, multi-tenant Loki) would add operational overhead for no benefit at this volume. - **Traces as the primary signal:** Tempo's metrics generator derives RED - metrics and a service graph from spans, so a service that only sends traces - still gets dashboards and error-rate alerting. + metrics (request rate, errors, duration) and a service graph from spans, so a + service that only sends traces still gets dashboards and error-rate alerting. ## Alternatives considered From b6101ad25fa54b32f78c297a2a12459bce90d60f Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sun, 5 Jul 2026 17:38:33 +0200 Subject: [PATCH 06/11] chore: switch demo to uv + Python 3.14; date the 0.2.0 release pyproject.toml replaces requirements.txt (Dependabot's pip ecosystem reads it too), the image builds on the uv base, and the container was verified to build and start. CHANGELOG's Unreleased section becomes 0.2.0 ahead of tagging. --- CHANGELOG.md | 2 +- demo/Dockerfile | 10 +++++----- demo/pyproject.toml | 11 +++++++++++ demo/requirements.txt | 5 ----- 4 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 demo/pyproject.toml delete mode 100644 demo/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c38e27..d1560cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Notable changes to this stack. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow [SemVer](https://semver.org/). -## [Unreleased] +## [0.2.0] - 2026-07-05 Reliability hardening: the stack becomes operable, not just runnable. diff --git a/demo/Dockerfile b/demo/Dockerfile index c55926a..f8476ed 100644 --- a/demo/Dockerfile +++ b/demo/Dockerfile @@ -1,9 +1,9 @@ -FROM python:3.13-slim - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +FROM ghcr.io/astral-sh/uv:python3.14-bookworm-slim WORKDIR /app +COPY pyproject.toml . +RUN uv sync + COPY app.py . -CMD ["opentelemetry-instrument", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uv", "run", "opentelemetry-instrument", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/demo/pyproject.toml b/demo/pyproject.toml new file mode 100644 index 0000000..7a86bbc --- /dev/null +++ b/demo/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "demo" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = [ + "fastapi==0.139.0", + "uvicorn==0.50.0", + "opentelemetry-distro==0.64b0", + "opentelemetry-exporter-otlp==1.43.0", + "opentelemetry-instrumentation-fastapi==0.64b0", +] diff --git a/demo/requirements.txt b/demo/requirements.txt deleted file mode 100644 index 3815ed8..0000000 --- a/demo/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -fastapi==0.139.0 -uvicorn==0.49.0 -opentelemetry-distro==0.64b0 -opentelemetry-exporter-otlp==1.43.0 -opentelemetry-instrumentation-fastapi==0.64b0 From 30a0753276da86ea22fd023164f596c349a9171c Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sun, 5 Jul 2026 18:10:35 +0200 Subject: [PATCH 07/11] docs: update changelog --- CHANGELOG.md | 50 +++++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1560cb..e9d0c34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,37 +5,33 @@ versions follow [SemVer](https://semver.org/). ## [0.2.0] - 2026-07-05 -Reliability hardening: the stack becomes operable, not just runnable. +Makes the stack operable, not just runnable: authentication, self-monitoring, +alert delivery, backups, and infrastructure as code. ### Added -- Bearer-token auth on OTLP ingestion (`OTLP_AUTH_TOKEN`); senders use - `OTEL_EXPORTER_OTLP_HEADERS`. -- Host self-monitoring: node-exporter, `HostDiskSpaceLow` alert (the - storage backstop — Loki/Tempo have no total-size cap), **Stack Health** - dashboard. -- Alert delivery: Alertmanager routes to `ALERT_WEBHOOK_URL`; an - always-firing `Watchdog` heartbeats to `HEARTBEAT_URL` (dead man's - switch). -- `just backup` / `just restore` — crash-consistent volume snapshots, - exercised end-to-end. -- `docs/RUNBOOK.md` and `docs/ONBOARDING.md` with verified telemetry - templates (zero-code Python, plain OTLP, Loki Docker driver, Grafana - Alloy). -- `infra/`: OpenTofu for the Cloudflare tunnel, ingress routes, and DNS, - plus Cloudflare Access (email one-time PIN) in front of Grafana. -- **Logs Overview** dashboard for OTLP-ingested logs (volume by service - and level, errors, live tail); Stack Health gains an Active Alerts - table and becomes the Grafana home dashboard. +- **Authenticated ingestion**: OTLP now requires a bearer token + (`OTLP_AUTH_TOKEN`). +- **Self-monitoring**: node-exporter, a **Stack Health** dashboard, and a + disk-space alert that backstops Loki and Tempo (neither can cap its own + size). +- **Alert delivery**: Alertmanager posts to any webhook + (`ALERT_WEBHOOK_URL`), and an always-firing `Watchdog` pings a dead man's + switch (`HEARTBEAT_URL`) so even a dead host gets noticed. +- **Backup and restore**: `just backup` / `just restore`, tested by wiping + every volume and recovering. +- **The Cloudflare edge as code**: `infra/` (OpenTofu) manages the tunnel, + DNS, and an email-PIN Access policy in front of Grafana. +- **Operator and onboarding docs**: a runbook plus copy-paste telemetry + templates for new projects. +- **Logs Overview dashboard** for OTLP logs, an Active Alerts table on Stack + Health, and Stack Health as the Grafana home page. ### Changed -- Prometheus storage now bounded by size as well as time - (`--storage.tsdb.retention.size=15GB`). -- **Tempo migrated to 3.0** (after a routine bump initially crash-looped on - the 2.x config): ingester/compactor blocks removed per the monolithic - migration guide, retention moved under overrides, verified in place - against live traffic. +- Prometheus storage is now capped by size as well as time (15 GB). +- **Tempo upgraded to 3.0**, migrating off the removed ingester/compactor + config and verified in place against live traffic. ## [0.1.0] - 2026-07-03 @@ -43,14 +39,14 @@ First tagged release: the stack is runnable, demoable, and validated in CI. ### Added -- `just demo`: one-command demo — an auto-instrumented FastAPI service under +- `just demo`: one-command demo: an auto-instrumented FastAPI service under constant load populates Grafana with correlated traces, metrics, and logs. - `just check`: validation gate (compose syntax, Prometheus config + alert rules, collector config, YAML, workflows, dashboard JSON), all in pinned containers. Run by GitHub Actions on every push/PR, plus a `just smoke` boot test. - Provisioned **Service Health (RED)** dashboard driven by Tempo span - metrics — works for any service that sends traces. + metrics: works for any service that sends traces. - Prometheus alert rules: target down, OTel export failures, high span error rate. - README: architecture diagram, demo quickstart, screenshot, design From 3e31b5375862f11eecc5c10d95aa419df7b1557e Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sun, 5 Jul 2026 18:10:40 +0200 Subject: [PATCH 08/11] feat: refactor Dockerfile to use multi-stage builds for improved efficiency --- demo/Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/demo/Dockerfile b/demo/Dockerfile index f8476ed..15dde83 100644 --- a/demo/Dockerfile +++ b/demo/Dockerfile @@ -1,9 +1,13 @@ -FROM ghcr.io/astral-sh/uv:python3.14-bookworm-slim - +FROM python:3.14-slim AS builder +COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/ WORKDIR /app COPY pyproject.toml . -RUN uv sync +RUN uv pip install --system --no-cache -r pyproject.toml +FROM python:3.14-slim +COPY --from=builder /usr/local/lib/python3.14/site-packages /usr/local/lib/python3.14/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin +WORKDIR /app COPY app.py . -CMD ["uv", "run", "opentelemetry-instrument", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["opentelemetry-instrument", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] From f2c47985a41441b0665067ba2bce13d778f30f85 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sun, 5 Jul 2026 19:31:25 +0200 Subject: [PATCH 09/11] fix: pin uv version to 0.11.26 in Dockerfile for consistency --- demo/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/Dockerfile b/demo/Dockerfile index 15dde83..5b429ee 100644 --- a/demo/Dockerfile +++ b/demo/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.14-slim AS builder -COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/ +COPY --from=ghcr.io/astral-sh/uv:0.11.26 /uv /bin/ WORKDIR /app COPY pyproject.toml . RUN uv pip install --system --no-cache -r pyproject.toml From 863b32c9b14fd14d1a3d3080d093a3621bbd4e15 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sun, 5 Jul 2026 19:31:31 +0200 Subject: [PATCH 10/11] feat: add validation for grafana_allowed_emails variable to ensure at least one email is set --- infra/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/main.tf b/infra/main.tf index fcf2d72..9510b66 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -43,6 +43,10 @@ variable "domain" { variable "grafana_allowed_emails" { type = list(string) description = "Email addresses allowed through Cloudflare Access to Grafana (one-time PIN)." + validation { + condition = length(var.grafana_allowed_emails) > 0 + error_message = "Set at least one email, or Cloudflare Access locks everyone out of Grafana." + } } resource "cloudflare_zero_trust_tunnel_cloudflared" "monitoring" { From 23a923cd2f98d6ff42d3d1695ee386b3328c34b2 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Sun, 5 Jul 2026 19:31:43 +0200 Subject: [PATCH 11/11] fix: update log query expressions in dashboards for correct syntax --- dashboards/logs-overview.json | 2 +- dashboards/relab-api.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/logs-overview.json b/dashboards/logs-overview.json index 36eb78c..90410e9 100644 --- a/dashboards/logs-overview.json +++ b/dashboards/logs-overview.json @@ -96,7 +96,7 @@ "datasource": { "type": "loki", "uid": "loki" }, "targets": [ { - "expr": "{service_name=~\"$service\", detected_level=~\"(?i)(error|critical|fatal)\"}", + "expr": "{service_name=~\"$service\"} | detected_level=~\"(?i)(error|critical|fatal)\"", "refId": "A" } ], diff --git a/dashboards/relab-api.json b/dashboards/relab-api.json index 7cd02ff..b0abee7 100644 --- a/dashboards/relab-api.json +++ b/dashboards/relab-api.json @@ -86,7 +86,7 @@ "datasource": { "type": "loki", "uid": "loki" }, "targets": [ { - "expr": "{service_name=\"relab-api\", detected_level=~\"error|critical\"}", + "expr": "{service_name=\"relab-api\"} | detected_level=~\"error|critical\"", "refId": "A" } ],