opsdeck-data.js

window.OPSDECK_DATA = {
  commands: [
    {
      id: "linux-release",
      title: "Show OS and kernel version",
      command: "cat /etc/os-release && uname -r",
      description: "Identify the Ubuntu release and running kernel before comparing behavior or packages.",
      category: "Linux",
      impact: "read",
      tags: ["version", "release", "kernel", "ubuntu", "inventory"]
    },
    {
      id: "linux-load",
      title: "Check load and uptime",
      command: "uptime && cat /proc/loadavg",
      description: "Show uptime, load averages, and runnable or blocked task counts.",
      category: "Linux",
      impact: "read",
      tags: ["cpu", "load", "performance", "pressure"]
    },
    {
      id: "linux-processes",
      title: "Top CPU-consuming processes",
      command: "ps -eo pid,ppid,user,stat,%cpu,%mem,etime,cmd --sort=-%cpu | head -25",
      description: "Rank processes by CPU consumption with state, age, and full command line.",
      category: "Linux",
      impact: "read",
      tags: ["process", "cpu", "top", "performance"]
    },
    {
      id: "linux-memory",
      title: "Inspect memory pressure",
      command: "free -h && vmstat 1 5",
      description: "Summarize memory and swap, then sample run queue, paging, I/O wait, and CPU activity.",
      category: "Linux",
      impact: "read",
      tags: ["ram", "swap", "oom", "memory", "vmstat"]
    },
    {
      id: "linux-oom",
      title: "Find out-of-memory kills",
      command: "journalctl -k -g 'Out of memory|Killed process' --since '-24 hours'",
      description: "Search recent kernel logs for OOM decisions and killed processes.",
      category: "Linux",
      impact: "read",
      tags: ["oom", "memory", "killed", "kernel", "logs"]
    },
    {
      id: "linux-file-descriptors",
      title: "Count open file descriptors",
      command: "find /proc/[0-9]*/fd -type l 2>/dev/null | wc -l",
      description: "Estimate system-wide open descriptors when investigating leaks or limit exhaustion.",
      category: "Linux",
      impact: "read",
      tags: ["fd", "files", "ulimit", "leak", "limits"]
    },
    {
      id: "linux-dmesg",
      title: "Read important kernel events",
      command: "journalctl -k -p warning..alert --since '-2 hours' --no-pager",
      description: "Show warning-or-higher kernel messages from the last two hours.",
      category: "Linux",
      impact: "read",
      tags: ["dmesg", "kernel", "hardware", "error", "logs"]
    },
    {
      id: "linux-time",
      title: "Verify time synchronization",
      command: "timedatectl status && timedatectl timesync-status",
      description: "Check timezone, NTP state, selected time server, offset, and synchronization.",
      category: "Linux",
      impact: "read",
      tags: ["ntp", "clock", "skew", "chrony", "time"]
    },
    {
      id: "linux-package-owner",
      title: "Find which package owns a file",
      command: "dpkg -S /path/to/file",
      description: "Resolve an installed file back to its Debian package.",
      category: "Linux",
      impact: "read",
      tags: ["apt", "dpkg", "package", "file"]
    },
    {
      id: "linux-upgradable",
      title: "List pending package upgrades",
      command: "apt list --upgradable",
      description: "Show installed packages for which a newer candidate version is available.",
      category: "Linux",
      impact: "read",
      tags: ["apt", "updates", "security", "patch"]
    },
    {
      id: "systemd-failed",
      title: "List failed systemd units",
      command: "systemctl --failed --no-pager",
      description: "Find services, mounts, and other units currently in a failed state.",
      category: "systemd",
      impact: "read",
      tags: ["service", "failed", "unit", "systemctl"]
    },
    {
      id: "systemd-status",
      title: "Inspect a service",
      command: "systemctl status <unit> --no-pager -l",
      description: "Show current service state, recent log lines, process tree, and failure reason.",
      category: "systemd",
      impact: "read",
      tags: ["service", "status", "unit", "daemon"]
    },
    {
      id: "systemd-logs",
      title: "Read recent service logs",
      command: "journalctl -u <unit> --since '-30 minutes' --no-pager",
      description: "Retrieve timestamped logs for one unit within a bounded time window.",
      category: "systemd",
      impact: "read",
      tags: ["journalctl", "logs", "service", "error"]
    },
    {
      id: "systemd-errors",
      title: "Show errors from current boot",
      command: "journalctl -b -p err..alert --no-pager",
      description: "Display error-or-higher messages from the current boot across all units.",
      category: "systemd",
      impact: "read",
      tags: ["boot", "journal", "error", "logs"]
    },
    {
      id: "systemd-restart",
      title: "Restart a service",
      command: "sudo systemctl restart <unit>",
      description: "Stop and start a unit. Check status and dependent services immediately afterward.",
      category: "systemd",
      impact: "change",
      tags: ["service", "restart", "daemon", "recovery"],
      note: "May interrupt active requests"
    },
    {
      id: "systemd-overrides",
      title: "Inspect service overrides",
      command: "systemctl cat <unit>",
      description: "Show the packaged unit and all active drop-in overrides in resolution order.",
      category: "systemd",
      impact: "read",
      tags: ["service", "config", "override", "unit"]
    },
    {
      id: "net-address",
      title: "Show addresses and routes",
      command: "ip -br address && ip route",
      description: "Get a compact interface overview followed by the active routing table.",
      category: "Network",
      impact: "read",
      tags: ["ip", "address", "route", "gateway", "interface"]
    },
    {
      id: "net-listen",
      title: "List listening sockets",
      command: "sudo ss -lntup",
      description: "Map listening TCP and UDP sockets to owning processes.",
      category: "Network",
      impact: "read",
      tags: ["ports", "socket", "listen", "tcp", "udp", "process"]
    },
    {
      id: "net-connect",
      title: "Test a TCP connection",
      command: "nc -vz -w 3 <host> <port>",
      description: "Verify TCP reachability with a three-second connection timeout.",
      category: "Network",
      impact: "read",
      tags: ["port", "connectivity", "firewall", "timeout", "nc"]
    },
    {
      id: "net-dns",
      title: "Trace DNS resolution",
      command: "resolvectl query <name> && dig +trace <name>",
      description: "Compare the local resolver result with an iterative DNS delegation trace.",
      category: "Network",
      impact: "read",
      tags: ["dns", "resolve", "systemd-resolved", "dig"]
    },
    {
      id: "net-path",
      title: "Inspect the network path",
      command: "mtr -rwzc 20 <host>",
      description: "Collect twenty route and packet-loss samples for each hop.",
      category: "Network",
      impact: "read",
      tags: ["latency", "loss", "route", "mtr", "network"]
    },
    {
      id: "net-counters",
      title: "Check interface error counters",
      command: "ip -s link show <interface> && ethtool -S <interface> | grep -Ei 'err|drop|crc|timeout'",
      description: "Inspect host-side drops and driver or NIC counters such as CRC and timeout errors.",
      category: "Network",
      impact: "read",
      tags: ["nic", "drop", "crc", "interface", "errors", "ceph"]
    },
    {
      id: "net-capture",
      title: "Capture traffic for one host and port",
      command: "sudo tcpdump -ni any host <host> and port <port> -c 200",
      description: "Capture a bounded packet sample without DNS or service-name lookups.",
      category: "Network",
      impact: "read",
      tags: ["packet", "capture", "tcpdump", "network"],
      note: "May expose payload metadata"
    },
    {
      id: "net-netplan",
      title: "Validate and apply Netplan safely",
      command: "sudo netplan generate && sudo netplan try",
      description: "Validate configuration and apply it with an automatic rollback prompt.",
      category: "Network",
      impact: "change",
      tags: ["netplan", "network", "config", "ubuntu"],
      note: "Use console access for remote changes"
    },
    {
      id: "storage-filesystems",
      title: "Check filesystem capacity and types",
      command: "df -hT && df -ih",
      description: "Show space and inode consumption; a filesystem can be full on either dimension.",
      category: "Storage",
      impact: "read",
      tags: ["disk", "full", "filesystem", "inode", "capacity"]
    },
    {
      id: "storage-largest",
      title: "Find large directories",
      command: "sudo du -xhd1 <path> | sort -h",
      description: "Rank first-level directory usage without crossing into other filesystems.",
      category: "Storage",
      impact: "read",
      tags: ["disk", "space", "large", "directory", "du"]
    },
    {
      id: "storage-block",
      title: "Map disks and filesystems",
      command: "lsblk -e7 -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS,MODEL,SERIAL",
      description: "Show the block-device tree with filesystem, mount, model, and serial identity.",
      category: "Storage",
      impact: "read",
      tags: ["disk", "device", "mount", "filesystem", "inventory"]
    },
    {
      id: "storage-io",
      title: "Identify busy block devices",
      command: "iostat -xz 1 5",
      description: "Sample device utilization, queueing, latency, and throughput five times.",
      category: "Storage",
      impact: "read",
      tags: ["disk", "io", "latency", "iostat", "performance"]
    },
    {
      id: "storage-smart",
      title: "Read SMART health",
      command: "sudo smartctl -x /dev/<device>",
      description: "Retrieve drive identity, health, error counters, and self-test history.",
      category: "Storage",
      impact: "read",
      tags: ["disk", "smart", "failure", "health", "nvme"]
    },
    {
      id: "storage-mounts",
      title: "Inspect a mount source",
      command: "findmnt -T <path> -o TARGET,SOURCE,FSTYPE,OPTIONS",
      description: "Resolve a path to its backing device or remote filesystem and active mount options.",
      category: "Storage",
      impact: "read",
      tags: ["mount", "filesystem", "device", "options"]
    },
    {
      id: "ceph-status",
      title: "Ceph cluster summary",
      command: "ceph -s",
      description: "Start here: see health, services, data usage, PG state, and current I/O activity.",
      category: "Ceph",
      impact: "read",
      tags: ["health", "status", "cluster", "pg", "osd"]
    },
    {
      id: "ceph-health",
      title: "Explain Ceph health alerts",
      command: "ceph health detail",
      description: "Expand HEALTH_WARN or HEALTH_ERR into affected daemons, PGs, and thresholds.",
      category: "Ceph",
      impact: "read",
      tags: ["health", "warn", "error", "diagnose", "alert"]
    },
    {
      id: "ceph-watch",
      title: "Watch live cluster events",
      command: "ceph -w",
      description: "Stream health changes, daemon reports, and cluster log events in real time.",
      category: "Ceph",
      impact: "read",
      tags: ["watch", "events", "live", "health", "logs"]
    },
    {
      id: "ceph-osd-tree",
      title: "Inspect OSD topology and state",
      command: "ceph osd tree",
      description: "Map OSDs to CRUSH hosts and verify whether each device is up and in.",
      category: "Ceph",
      impact: "read",
      tags: ["osd", "crush", "down", "out", "host", "topology"]
    },
    {
      id: "ceph-osd-df",
      title: "Compare OSD utilization",
      command: "ceph osd df tree",
      description: "Find uneven usage, full OSDs, and variance across the CRUSH hierarchy.",
      category: "Ceph",
      impact: "read",
      tags: ["osd", "full", "capacity", "usage", "variance"]
    },
    {
      id: "ceph-pg",
      title: "Summarize placement group states",
      command: "ceph pg stat && ceph pg dump_stuck inactive",
      description: "Check aggregate PG state and list placement groups that remain inactive.",
      category: "Ceph",
      impact: "read",
      tags: ["pg", "stuck", "inactive", "peering", "degraded"]
    },
    {
      id: "ceph-pg-query",
      title: "Query one placement group",
      command: "ceph tell <pgid> query",
      description: "Inspect peering history, acting set, blockers, and recovery state for one PG.",
      category: "Ceph",
      impact: "read",
      tags: ["pg", "query", "peering", "blocked", "acting"]
    },
    {
      id: "ceph-versions",
      title: "Check daemon version consistency",
      command: "ceph versions",
      description: "Count running daemon versions to detect partial or inconsistent upgrades.",
      category: "Ceph",
      impact: "read",
      tags: ["version", "upgrade", "daemon", "consistency"]
    },
    {
      id: "ceph-quorum",
      title: "Inspect monitor quorum",
      command: "ceph quorum_status --format json-pretty",
      description: "Show participating monitors, quorum age, election epoch, and leader.",
      category: "Ceph",
      impact: "read",
      tags: ["mon", "monitor", "quorum", "election", "leader"]
    },
    {
      id: "ceph-slow-ops",
      title: "Find slow operations",
      command: "ceph daemon osd.<id> dump_historic_ops",
      description: "Inspect recent slow operations on a specific OSD through its admin socket.",
      category: "Ceph",
      impact: "read",
      tags: ["slow", "ops", "latency", "osd", "performance"]
    },
    {
      id: "ceph-ok-to-stop",
      title: "Check whether OSDs are safe to stop",
      command: "ceph osd ok-to-stop <id> [<id>...]",
      description: "Ask Ceph whether stopping selected OSDs would preserve availability and redundancy.",
      category: "Ceph",
      impact: "read",
      tags: ["osd", "maintenance", "safe", "stop", "redundancy"]
    },
    {
      id: "ceph-osd-out",
      title: "Mark an OSD out",
      command: "ceph osd out <id>",
      description: "Remove an OSD from data placement and trigger rebalancing. Confirm capacity first.",
      category: "Ceph",
      impact: "danger",
      tags: ["osd", "out", "recovery", "rebalance", "maintenance"],
      note: "Triggers data movement"
    },
    {
      id: "ceph-repair-pg",
      title: "Request repair of one PG",
      command: "ceph pg repair <pgid>",
      description: "Ask Ceph to repair a known inconsistent placement group after evidence review.",
      category: "Ceph",
      impact: "danger",
      tags: ["pg", "repair", "inconsistent", "scrub"],
      note: "Review scrub evidence first"
    },
    {
      id: "ceph-orch-status",
      title: "Inspect cephadm services",
      command: "ceph orch ps --format yaml",
      description: "List orchestrated daemons with hosts, status, image, version, and last refresh.",
      category: "Ceph services",
      impact: "read",
      tags: ["cephadm", "orchestrator", "daemon", "container", "service"]
    },
    {
      id: "ceph-orch-ls",
      title: "List cephadm service specifications",
      command: "ceph orch ls --export",
      description: "Export service placement and configuration specifications for review or backup.",
      category: "Ceph services",
      impact: "read",
      tags: ["cephadm", "orchestrator", "spec", "placement", "config"]
    },
    {
      id: "ceph-logs",
      title: "Read a cephadm daemon log",
      command: "cephadm logs --name <daemon-name> -- --since '30 minutes ago'",
      description: "Retrieve bounded journald logs for a containerized Ceph daemon.",
      category: "Ceph services",
      impact: "read",
      tags: ["cephadm", "logs", "daemon", "journal"]
    },
    {
      id: "ceph-rbd-list",
      title: "List RBD images with provisioned size",
      command: "rbd du --pool <pool>",
      description: "Show RBD image provisioned and actual usage for a pool.",
      category: "Ceph services",
      impact: "read",
      tags: ["rbd", "image", "pool", "usage", "block"]
    },
    {
      id: "ceph-rbd-status",
      title: "Show clients using an RBD image",
      command: "rbd status <pool>/<image>",
      description: "List watchers and client addresses currently attached to an RBD image.",
      category: "Ceph services",
      impact: "read",
      tags: ["rbd", "watcher", "client", "image", "block"]
    },
    {
      id: "ceph-fs-status",
      title: "Check CephFS status",
      command: "ceph fs status",
      description: "Summarize MDS ranks, client sessions, pools, and CephFS activity.",
      category: "Ceph services",
      impact: "read",
      tags: ["cephfs", "mds", "filesystem", "client"]
    },
    {
      id: "ceph-rgw",
      title: "Check RGW daemon health",
      command: "ceph orch ps --daemon-type rgw && radosgw-admin sync status",
      description: "Inspect gateway daemons and multisite synchronization state.",
      category: "Ceph services",
      impact: "read",
      tags: ["rgw", "s3", "gateway", "multisite", "sync"]
    },
    {
      id: "redis-ping",
      title: "Verify Redis responsiveness",
      command: "redis-cli -h <host> -p <port> PING",
      description: "Confirm TCP, protocol handling, and basic server responsiveness.",
      category: "Redis",
      impact: "read",
      tags: ["ping", "connectivity", "health", "server"]
    },
    {
      id: "redis-info",
      title: "Capture Redis server facts",
      command: "redis-cli -h <host> -p <port> INFO",
      description: "Collect server, client, memory, persistence, replication, CPU, and keyspace metrics.",
      category: "Redis",
      impact: "read",
      tags: ["info", "health", "memory", "replication", "persistence"]
    },
    {
      id: "redis-memory",
      title: "Inspect Redis memory",
      command: "redis-cli INFO memory && redis-cli MEMORY STATS",
      description: "Compare memory usage, fragmentation, allocator behavior, and configured limits.",
      category: "Redis",
      impact: "read",
      tags: ["memory", "maxmemory", "fragmentation", "oom"]
    },
    {
      id: "redis-bigkeys",
      title: "Sample large Redis keys",
      command: "redis-cli --bigkeys",
      description: "Scan the keyspace and report the largest sampled keys by data type.",
      category: "Redis",
      impact: "read",
      tags: ["bigkeys", "memory", "scan", "keyspace"],
      note: "Adds scan load; schedule thoughtfully"
    },
    {
      id: "redis-scan",
      title: "Scan keys without blocking Redis",
      command: "redis-cli --scan --pattern '<pattern>*' --count 100",
      description: "Iterate matching keys using SCAN rather than the blocking KEYS command.",
      category: "Redis",
      impact: "read",
      tags: ["keys", "scan", "pattern", "production"]
    },
    {
      id: "redis-latency",
      title: "Measure Redis latency",
      command: "redis-cli --latency-history -h <host> -p <port>",
      description: "Continuously sample server round-trip latency and reset stats by interval.",
      category: "Redis",
      impact: "read",
      tags: ["latency", "performance", "slow", "network"]
    },
    {
      id: "redis-slowlog",
      title: "Read the Redis slow log",
      command: "redis-cli SLOWLOG GET 20",
      description: "Show recent commands whose server-side execution exceeded the slow-log threshold.",
      category: "Redis",
      impact: "read",
      tags: ["slowlog", "latency", "commands", "performance"]
    },
    {
      id: "redis-replication",
      title: "Inspect Redis replication",
      command: "redis-cli INFO replication",
      description: "Show role, upstream status, replica offsets, backlog, and connected replicas.",
      category: "Redis",
      impact: "read",
      tags: ["replica", "primary", "lag", "offset", "failover"]
    },
    {
      id: "redis-cluster",
      title: "Check Redis Cluster health",
      command: "redis-cli --cluster check <host>:<port>",
      description: "Validate slot coverage, node agreement, and replica distribution.",
      category: "Redis",
      impact: "read",
      tags: ["cluster", "slots", "nodes", "replicas", "health"]
    },
    {
      id: "redis-clients",
      title: "Inspect connected Redis clients",
      command: "redis-cli CLIENT LIST",
      description: "List connections with age, idle time, selected database, flags, and current command.",
      category: "Redis",
      impact: "read",
      tags: ["client", "connection", "blocked", "idle"]
    },
    {
      id: "tidb-list",
      title: "List TiUP-managed clusters",
      command: "tiup cluster list",
      description: "Show cluster names, versions, users, paths, and control metadata.",
      category: "TiDB",
      impact: "read",
      tags: ["tiup", "cluster", "inventory", "version"]
    },
    {
      id: "tidb-display",
      title: "Display TiDB cluster status",
      command: "tiup cluster display <cluster-name> --process --uptime",
      description: "Show every component, role, host, port, state, process usage, and uptime.",
      category: "TiDB",
      impact: "read",
      tags: ["tiup", "status", "tikv", "pd", "tiflash", "tidb"]
    },
    {
      id: "tidb-check",
      title: "Run TiUP environment checks",
      command: "tiup cluster check <cluster-name> --cluster",
      description: "Validate the existing cluster hosts against TiDB operating requirements.",
      category: "TiDB",
      impact: "read",
      tags: ["tiup", "check", "environment", "kernel", "limits"]
    },
    {
      id: "tidb-logs",
      title: "Search TiDB component logs",
      command: "tiup cluster exec <cluster-name> -R <role> --command \"journalctl -u <unit> --since '-30 min' --no-pager\"",
      description: "Run a bounded journal query across all nodes of one TiDB component role.",
      category: "TiDB",
      impact: "read",
      tags: ["tiup", "logs", "tikv", "pd", "journal"]
    },
    {
      id: "tidb-pd-store",
      title: "Inspect PD store state",
      command: "tiup ctl:v<cluster-version> pd -u http://<pd-host>:2379 store",
      description: "List TiKV stores, capacity, state, heartbeat, region, and leader counts.",
      category: "TiDB",
      impact: "read",
      tags: ["pd", "store", "tikv", "region", "leader"]
    },
    {
      id: "tidb-hot-regions",
      title: "Inspect TiKV hot regions",
      command: "tiup ctl:v<cluster-version> pd -u http://<pd-host>:2379 hot read",
      description: "Identify read-hot regions contributing to localized TiKV load.",
      category: "TiDB",
      impact: "read",
      tags: ["pd", "hot", "region", "tikv", "latency", "performance"]
    },
    {
      id: "tidb-diag",
      title: "Collect a TiDB diagnostic bundle",
      command: "tiup diag collect <cluster-name> --include='system,config,log' --since='2h'",
      description: "Collect bounded system, configuration, and log evidence for offline analysis.",
      category: "TiDB",
      impact: "read",
      tags: ["diagnostic", "bundle", "logs", "support", "evidence"],
      note: "Review bundle for secrets before sharing"
    },
    {
      id: "tidb-reload",
      title: "Apply reloadable TiDB configuration",
      command: "tiup cluster reload <cluster-name> -R <role>",
      description: "Regenerate configuration and restart the selected component role where required.",
      category: "TiDB",
      impact: "change",
      tags: ["tiup", "reload", "config", "restart"],
      note: "Review generated diff and topology"
    },
    {
      id: "container-ps",
      title: "List all containers",
      command: "sudo podman ps -a || sudo docker ps -a",
      description: "Show running and stopped containers using the available runtime.",
      category: "Containers",
      impact: "read",
      tags: ["podman", "docker", "container", "status", "cephadm"]
    },
    {
      id: "container-stats",
      title: "Snapshot container resource usage",
      command: "sudo podman stats --no-stream || sudo docker stats --no-stream",
      description: "Capture CPU, memory, network, and block I/O usage for active containers.",
      category: "Containers",
      impact: "read",
      tags: ["podman", "docker", "container", "cpu", "memory"]
    },
    {
      id: "container-logs",
      title: "Read recent container logs",
      command: "sudo podman logs --since 30m <container> || sudo docker logs --since 30m <container>",
      description: "Retrieve a bounded log window from a named container.",
      category: "Containers",
      impact: "read",
      tags: ["podman", "docker", "container", "logs"]
    },
    {
      id: "security-ufw",
      title: "Inspect host firewall rules",
      command: "sudo ufw status verbose && sudo nft list ruleset",
      description: "Review UFW policy and the effective nftables ruleset.",
      category: "Security",
      impact: "read",
      tags: ["firewall", "ufw", "nftables", "ports", "network"]
    },
    {
      id: "security-cert",
      title: "Inspect a remote TLS certificate",
      command: "openssl s_client -connect <host>:<port> -servername <name> </dev/null 2>/dev/null | openssl x509 -noout -subject -issuer -dates -ext subjectAltName",
      description: "Show identity, issuer, validity period, and SANs from the served certificate.",
      category: "Security",
      impact: "read",
      tags: ["tls", "ssl", "certificate", "expiry", "x509"]
    },
    {
      id: "security-logins",
      title: "Review recent login activity",
      command: "last -ai | head -30 && sudo journalctl -u ssh --since '-24 hours' --no-pager",
      description: "Correlate successful sessions with recent SSH service events.",
      category: "Security",
      impact: "read",
      tags: ["ssh", "login", "audit", "security"]
    }
  ],

  runbooks: [
    {
      id: "incident-ceph-osd-down",
      title: "Ceph OSD down or degraded PGs",
      severity: "Potential data risk",
      keywords: ["ceph", "osd", "down", "degraded", "undersized", "pg", "health_err", "health_warn", "host reboot"],
      requireAny: ["ceph", "osd", "pg", "degraded"],
      explanation: "Reduced redundancy usually follows one or more OSDs becoming unavailable. Establish monitor quorum and network health before touching the OSD or forcing recovery.",
      cause: "Common causes include a stopped daemon, failed device, host outage, network loss, or an OSD that is up but out.",
      stages: [
        { name: "Observe", text: "Confirm the alert, quorum, and affected failure domain.", commands: ["ceph -s", "ceph health detail", "ceph quorum_status --format json-pretty", "ceph osd tree down"] },
        { name: "Correlate", text: "Inspect the OSD host, daemon, device, and network evidence.", commands: ["systemctl status ceph-osd@<id> --no-pager -l", "journalctl -u ceph-osd@<id> --since '-30 min' --no-pager", "ceph osd metadata <id>", "smartctl -x /dev/<device>"] },
        { name: "Recover", text: "Restart only if the device and host are healthy; otherwise follow your replacement runbook.", commands: ["ceph osd ok-to-stop <id>", "sudo systemctl restart ceph-osd@<id>"] },
        { name: "Verify", text: "Watch recovery and confirm all OSDs return up/in and PGs active+clean.", commands: ["ceph -w", "ceph osd tree", "ceph pg stat"] }
      ],
      guardrail: "Do not mark an OSD out or purge it until you understand capacity, failure-domain impact, and whether the device is recoverable."
    },
    {
      id: "incident-ceph-full",
      title: "Ceph nearfull, backfillfull, or full",
      severity: "Recovery blocked",
      keywords: ["ceph", "nearfull", "backfillfull", "full", "capacity", "recovery_toofull", "backfill_toofull"],
      requireAny: ["nearfull", "backfillfull", "recovery_toofull", "ceph full", "osd full"],
      explanation: "Ceph progressively blocks backfill and then client writes as OSDs cross fullness ratios. Adding capacity or safely reducing stored data is the durable fix.",
      cause: "Uneven CRUSH placement, a failed OSD, rapid growth, stranded deleted data, or inadequate free capacity can trigger the threshold.",
      stages: [
        { name: "Observe", text: "Measure cluster and per-OSD utilization before changing ratios.", commands: ["ceph -s", "ceph health detail", "ceph df detail", "ceph osd df tree"] },
        { name: "Correlate", text: "Find pool growth and misplaced data.", commands: ["ceph df", "rados df", "ceph pg dump pgs_brief | grep -E 'backfill_toofull|recovery_toofull'"] },
        { name: "Contain", text: "Pause nonessential writers and add capacity or remove verified disposable data.", commands: ["ceph osd tree", "ceph osd pool ls detail"] },
        { name: "Verify", text: "Confirm utilization falls below thresholds and recovery resumes.", commands: ["ceph -w", "ceph osd df tree", "ceph pg stat"] }
      ],
      guardrail: "Do not raise full ratios as a substitute for capacity. Never delete PG directories or unknown RADOS objects."
    },
    {
      id: "incident-ceph-quorum",
      title: "Ceph monitor quorum loss",
      severity: "Control plane outage",
      keywords: ["ceph", "mon", "monitor", "quorum", "probing", "electing", "synchronizing", "clock skew"],
      requireAny: ["quorum", "monitor down", "mon down", "clock skew"],
      explanation: "Without monitor quorum, clients and daemons cannot reliably update the cluster map. Restore time and network agreement before investigating data services.",
      cause: "Monitor service failure, blocked ports, name/address mismatch, storage exhaustion, or clock drift are typical triggers.",
      stages: [
        { name: "Observe", text: "Determine quorum membership and affected monitors.", commands: ["ceph -s", "ceph quorum_status --format json-pretty", "ceph mon dump"] },
        { name: "Correlate", text: "Check service, clock, capacity, and ports on the affected monitor.", commands: ["systemctl status ceph-mon@<name> --no-pager -l", "journalctl -u ceph-mon@<name> --since '-30 min' --no-pager", "timedatectl timesync-status", "ss -lntp | grep -E ':3300|:6789'"] },
        { name: "Recover", text: "Correct the underlying dependency, then restart only the failed monitor.", commands: ["sudo systemctl restart ceph-mon@<name>"] },
        { name: "Verify", text: "Confirm a stable majority and normal election state.", commands: ["ceph quorum_status --format json-pretty", "ceph health detail"] }
      ],
      guardrail: "Do not rebuild monitor stores while a healthy quorum can still be restored from service, network, time, or capacity fixes."
    },
    {
      id: "incident-ceph-slow",
      title: "Ceph slow operations or high latency",
      severity: "Performance degradation",
      keywords: ["ceph", "slow ops", "latency", "blocked", "slow requests", "heartbeat"],
      requireAny: ["ceph slow", "slow ops", "slow requests", "osd latency"],
      explanation: "Slow operations are symptoms, not a diagnosis. Correlate affected OSDs with storage latency, network loss, recovery traffic, and client workload.",
      cause: "Device latency, network drops, overloaded hosts, recovery/backfill, or a single problematic PG can stall operations.",
      stages: [
        { name: "Observe", text: "Identify which services, OSDs, and PGs report delays.", commands: ["ceph -s", "ceph health detail", "ceph osd perf", "ceph pg stat"] },
        { name: "Correlate", text: "Compare Ceph delay with host I/O and network counters.", commands: ["iostat -xz 1 5", "vmstat 1 5", "ip -s link", "ceph daemon osd.<id> dump_historic_ops"] },
        { name: "Contain", text: "Reduce nonessential load; avoid broad restarts that increase recovery pressure.", commands: ["ceph progress", "ceph osd df tree"] },
        { name: "Verify", text: "Track latency and health until slow operations drain.", commands: ["ceph -w", "ceph osd perf"] }
      ],
      guardrail: "Restarting many OSDs at once can reduce availability and amplify recovery traffic."
    },
    {
      id: "incident-disk-full",
      title: "Ubuntu filesystem or inode exhaustion",
      severity: "Service interruption",
      keywords: ["disk full", "no space left", "filesystem full", "inode", "100%", "service restarting", "read only filesystem"],
      requireAny: ["disk", "space", "filesystem", "inode", "no space"],
      explanation: "Applications may fail even when byte capacity remains if inodes are exhausted. Locate the exact filesystem and growth source before deleting anything.",
      cause: "Unbounded logs, container layers, deleted-but-open files, temporary data, or inode-heavy workloads are common causes.",
      stages: [
        { name: "Observe", text: "Find the full filesystem and whether bytes or inodes are exhausted.", commands: ["df -hT", "df -ih", "findmnt -T <affected-path>"] },
        { name: "Correlate", text: "Locate large directories and deleted files still held open.", commands: ["sudo du -xhd1 <mountpoint> | sort -h", "sudo lsof +L1"] },
        { name: "Contain", text: "Stop the growth source and use application-aware retention or cleanup.", commands: ["systemctl status <unit> --no-pager -l", "journalctl --disk-usage"] },
        { name: "Verify", text: "Confirm headroom and service stability.", commands: ["df -hT && df -ih", "systemctl status <unit> --no-pager -l", "journalctl -u <unit> --since '-10 min' --no-pager"] }
      ],
      guardrail: "Do not recursively delete unknown files. For databases and Ceph, use the product’s supported retention or data-removal workflow."
    },
    {
      id: "incident-service-loop",
      title: "systemd service crash loop",
      severity: "Service unavailable",
      keywords: ["service down", "failed", "restarting", "start-limit-hit", "exit code", "crash loop", "systemd"],
      requireAny: ["service", "systemd", "start-limit", "crash", "restarting"],
      explanation: "A repeated restart usually preserves the original failure in the unit state and journal. Capture it before resetting counters or changing configuration.",
      cause: "Invalid configuration, missing dependencies, permissions, exhausted resources, or a conflicting port commonly cause immediate exits.",
      stages: [
        { name: "Observe", text: "Capture unit result, exit status, and recent logs.", commands: ["systemctl status <unit> --no-pager -l", "systemctl show <unit> -p Result -p ExecMainStatus -p NRestarts", "journalctl -u <unit> -b --no-pager -n 200"] },
        { name: "Correlate", text: "Check configuration, dependencies, ports, and resources.", commands: ["systemctl cat <unit>", "systemctl list-dependencies <unit>", "ss -lntup", "df -hT && free -h"] },
        { name: "Recover", text: "Fix the root cause, validate configuration, then reset the failure and start.", commands: ["<application> --test-config", "sudo systemctl reset-failed <unit>", "sudo systemctl start <unit>"] },
        { name: "Verify", text: "Confirm active state and stable logs.", commands: ["systemctl is-active <unit>", "journalctl -u <unit> --since '-5 min' --no-pager"] }
      ],
      guardrail: "Do not keep restarting a stateful service without understanding its exit reason and recovery semantics."
    },
    {
      id: "incident-memory-pressure",
      title: "High memory usage or OOM kills",
      severity: "Host instability",
      keywords: ["memory", "ram", "oom", "out of memory", "killed process", "swap", "thrashing"],
      requireAny: ["memory", "oom", "swap", "killed"],
      explanation: "Linux uses free memory for cache, so low ‘free’ alone is not a fault. Look for sustained pressure, swapping, allocation failures, and OOM decisions.",
      cause: "A workload leak, undersized limits, excessive cache pressure, or too many colocated services may exhaust reclaimable memory.",
      stages: [
        { name: "Observe", text: "Measure available memory, swap, and pressure.", commands: ["free -h", "vmstat 1 10", "cat /proc/pressure/memory"] },
        { name: "Correlate", text: "Identify consumers and OOM victims.", commands: ["ps -eo pid,user,%mem,rss,cmd --sort=-rss | head -25", "journalctl -k -g 'Out of memory|Killed process' --since '-24 hours'"] },
        { name: "Contain", text: "Reduce workload or gracefully restart only the confirmed leaking service.", commands: ["systemctl status <unit> --no-pager -l", "sudo systemctl restart <unit>"] },
        { name: "Verify", text: "Ensure pressure and swap activity fall while the service remains healthy.", commands: ["vmstat 1 10", "systemctl status <unit> --no-pager -l"] }
      ],
      guardrail: "Avoid dropping caches or disabling the OOM killer as a routine fix; both can hide or worsen the underlying problem."
    },
    {
      id: "incident-network-path",
      title: "Network timeout, packet loss, or refused connection",
      severity: "Connectivity degradation",
      keywords: ["timeout", "connection refused", "network", "packet loss", "unreachable", "port", "firewall", "dns"],
      requireAny: ["network", "timeout", "refused", "unreachable", "packet", "dns", "port"],
      explanation: "Separate name resolution, routing, packet loss, firewall policy, listener state, and application behavior to localize the fault.",
      cause: "DNS errors, missing routes, host or network ACLs, a non-listening service, MTU mismatch, or packet loss can appear similar to callers.",
      stages: [
        { name: "Observe", text: "Resolve the name and test the exact TCP endpoint.", commands: ["resolvectl query <name>", "nc -vz -w 3 <host> <port>", "curl -vk --connect-timeout 3 <url>"] },
        { name: "Correlate", text: "Inspect path, local route, listener, and counters.", commands: ["ip route get <host>", "mtr -rwzc 20 <host>", "ss -lntup", "ip -s link"] },
        { name: "Contain", text: "Correct the narrowest confirmed layer; record any temporary firewall change.", commands: ["sudo ufw status verbose", "sudo nft list ruleset"] },
        { name: "Verify", text: "Retest from the original caller and check service logs.", commands: ["nc -vz -w 3 <host> <port>", "journalctl -u <unit> --since '-10 min' --no-pager"] }
      ],
      guardrail: "Do not disable the firewall globally. Prove which source, destination, protocol, and port require a rule."
    },
    {
      id: "incident-redis-memory",
      title: "Redis memory growth or OOM errors",
      severity: "Write availability risk",
      keywords: ["redis", "memory", "oom", "maxmemory", "eviction", "fragmentation", "big key"],
      requireAny: ["redis memory", "redis oom", "maxmemory", "eviction", "redis full"],
      explanation: "Redis can reject writes or evict keys at maxmemory. Distinguish dataset growth from allocator fragmentation and verify the configured eviction policy.",
      cause: "Unexpected key growth, missing TTLs, large values, client buffers, replication backlog, or fragmentation can consume memory.",
      stages: [
        { name: "Observe", text: "Capture memory, keyspace, and eviction state.", commands: ["redis-cli INFO memory", "redis-cli MEMORY STATS", "redis-cli INFO stats", "redis-cli INFO keyspace"] },
        { name: "Correlate", text: "Sample large keys and inspect configuration without blocking the server.", commands: ["redis-cli --bigkeys", "redis-cli CONFIG GET maxmemory", "redis-cli CONFIG GET maxmemory-policy"] },
        { name: "Contain", text: "Stop growth at the writer; delete or expire keys only with application ownership.", commands: ["redis-cli --scan --pattern '<namespace>:*' --count 100"] },
        { name: "Verify", text: "Confirm memory stabilizes and writes succeed.", commands: ["redis-cli INFO memory", "redis-cli INFO stats", "redis-cli PING"] }
      ],
      guardrail: "Never run KEYS * in production. Do not FLUSHDB, change eviction policy, or delete namespaces without application-owner approval."
    },
    {
      id: "incident-redis-latency",
      title: "Redis latency or blocked clients",
      severity: "Application slowdown",
      keywords: ["redis", "latency", "slow", "blocked clients", "timeout", "slowlog", "cpu"],
      requireAny: ["redis latency", "redis slow", "blocked clients", "slowlog"],
      explanation: "Measure client round-trip latency separately from server command execution and intrinsic host latency.",
      cause: "Slow commands, large keys, fork/persistence stalls, network jitter, CPU contention, or blocked clients may be responsible.",
      stages: [
        { name: "Observe", text: "Measure latency and server counters.", commands: ["redis-cli --latency-history", "redis-cli INFO stats", "redis-cli INFO clients"] },
        { name: "Correlate", text: "Find slow commands and host-level stalls.", commands: ["redis-cli SLOWLOG GET 20", "redis-cli LATENCY DOCTOR", "redis-cli --intrinsic-latency 10"] },
        { name: "Contain", text: "Stop the offending client or workload only after identifying it.", commands: ["redis-cli CLIENT LIST", "redis-cli INFO persistence"] },
        { name: "Verify", text: "Retest latency and confirm blocked clients drain.", commands: ["redis-cli --latency -c", "redis-cli INFO clients"] }
      ],
      guardrail: "Avoid KEYS and other unbounded commands. CLIENT KILL and configuration changes require precise target confirmation."
    },
    {
      id: "incident-redis-replication",
      title: "Redis replica lag or link down",
      severity: "Reduced resilience",
      keywords: ["redis", "replica", "replication", "master_link_status", "lag", "failover", "sentinel"],
      requireAny: ["redis replica", "redis replication", "master_link", "redis lag", "redis failover"],
      explanation: "Replication health depends on connectivity, offsets, backlog, persistence pressure, and the primary’s ability to serve a resynchronization.",
      cause: "Network interruption, authentication mismatch, overloaded primary, insufficient backlog, or disk/fork stalls can break or delay replication.",
      stages: [
        { name: "Observe", text: "Compare role, link status, offsets, and lag on both sides.", commands: ["redis-cli INFO replication", "redis-cli ROLE"] },
        { name: "Correlate", text: "Test network and review persistence and logs.", commands: ["nc -vz -w 3 <primary-host> 6379", "redis-cli INFO persistence", "journalctl -u redis-server --since '-30 min' --no-pager"] },
        { name: "Contain", text: "Stabilize network and load before initiating topology changes.", commands: ["redis-cli CONFIG GET repl-backlog-size", "redis-cli INFO memory"] },
        { name: "Verify", text: "Confirm link up, falling lag, and converging offsets.", commands: ["redis-cli INFO replication", "redis-cli --cluster check <host>:<port>"] }
      ],
      guardrail: "Do not promote or reconfigure replication until you know which node has the authoritative dataset and current offset."
    },
    {
      id: "incident-tidb-node-down",
      title: "TiDB, PD, TiKV, or TiFlash node down",
      severity: "Database availability risk",
      keywords: ["tidb", "tikv", "pd", "tiflash", "down", "unreachable", "pending offline", "tombstone"],
      requireAny: ["tidb", "tikv", "pd", "tiflash", "tiup"],
      explanation: "Component impact differs by role. First establish the cluster-wide view and PD store state, then diagnose the specific host and service.",
      cause: "Process failure, network partition, disk pressure, clock drift, or an intentional scale-in state can make a node unavailable.",
      stages: [
        { name: "Observe", text: "Capture component status, role, uptime, and PD store view.", commands: ["tiup cluster display <cluster-name> --process --uptime", "tiup ctl:v<cluster-version> pd -u http://<pd-host>:2379 store"] },
        { name: "Correlate", text: "Collect bounded service and host evidence.", commands: ["tiup cluster exec <cluster-name> -N <node> --command 'df -hT; free -h; uptime'", "journalctl -u <component-unit> --since '-30 min' --no-pager"] },
        { name: "Recover", text: "Fix the dependency, then restart only the affected component through TiUP.", commands: ["tiup cluster restart <cluster-name> -N <node>"] },
        { name: "Verify", text: "Confirm Up state and check for remaining hot or unavailable regions.", commands: ["tiup cluster display <cluster-name> --process --uptime", "tiup ctl:v<cluster-version> pd -u http://<pd-host>:2379 store"] }
      ],
      guardrail: "Do not destroy, tombstone, or force-remove a TiKV store while its data and region replicas are not fully understood."
    },
    {
      id: "incident-tidb-latency",
      title: "TiDB query latency or hot TiKV regions",
      severity: "Database slowdown",
      keywords: ["tidb", "slow query", "latency", "hot region", "tikv", "leader", "coprocessor"],
      requireAny: ["tidb latency", "slow query", "hot region", "tikv latency", "tidb slow"],
      explanation: "Separate SQL planning/execution from TiKV storage latency and regional hotspots. Preserve the slow-query evidence before tuning.",
      cause: "Missing indexes, changed plans, hot keys, uneven leaders, disk latency, or overloaded TiKV nodes commonly drive tail latency.",
      stages: [
        { name: "Observe", text: "Capture component health and current process load.", commands: ["tiup cluster display <cluster-name> --process --uptime", "tiup ctl:v<cluster-version> pd -u http://<pd-host>:2379 hot read"] },
        { name: "Correlate", text: "Review slow queries and host storage latency.", commands: ["grep -h 'SLOW_QUERY' <tidb-log-path> | tail -100", "iostat -xz 1 5", "tiup ctl:v<cluster-version> pd -u http://<pd-host>:2379 store"] },
        { name: "Contain", text: "Reduce the offending workload and validate a query or schema change outside production.", commands: ["mysql -h <tidb-host> -P 4000 -u <user> -e 'SHOW PROCESSLIST;'"] },
        { name: "Verify", text: "Compare latency, hot regions, and resource distribution after mitigation.", commands: ["tiup cluster display <cluster-name> --process", "tiup ctl:v<cluster-version> pd -u http://<pd-host>:2379 hot read"] }
      ],
      guardrail: "Do not add indexes, cancel transactions, or change PD scheduling without confirming workload and replication impact."
    }
  ]
};