From d8197272c7f5bf4e15929c9525df48e9b09639c8 Mon Sep 17 00:00:00 2001 From: df Date: Sat, 18 Jul 2026 03:40:29 +0800 Subject: [PATCH] feat: add TUI diagnostics and operations docs --- .dockerignore | 1 + .env.example | 8 + .gitignore | 4 + Makefile | 16 +- README.md | 543 +++++++++++++++------------------ USAGE_GUIDE.md | 152 ++++++--- docker-compose.yml | 2 + docs/TROUBLESHOOTING.md | 176 +++++++++++ src/FanControlWithEsxiSmart.sh | 387 ++++++++++++++++++++--- src/fan-control-tui.sh | 541 ++++++++++++++++++++++++++++++++ tests/fan-control.test.sh | 176 ++++++++++- tests/tui.test.sh | 99 ++++++ 12 files changed, 1715 insertions(+), 390 deletions(-) create mode 100644 docs/TROUBLESHOOTING.md create mode 100755 src/fan-control-tui.sh create mode 100644 tests/tui.test.sh diff --git a/.dockerignore b/.dockerignore index 73dc1dd..6f931b4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,7 @@ !.env.example logs +secrets tests README.md USAGE_GUIDE.md diff --git a/.env.example b/.env.example index 2dcb2df..7c030f7 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,10 @@ CHECK_INTERVAL=60 COMMAND_TIMEOUT=20 DRY_RUN=false +# DEBUG prints command context and source-selection details without passwords. +# INFO is suitable for normal unattended operation. +LOG_LEVEL=INFO + # restore iDRAC automatic fan control when auto mode exits RESTORE_AUTO_ON_EXIT=true @@ -93,3 +97,7 @@ MANUAL_FAN_SPEED=35 # ----------------------------------------------------------------------------- LOG_DIR=/var/log/fan-control LOG_FILE=fan_control.log + +# 0 derives a limit from CHECK_INTERVAL*3 + COMMAND_TIMEOUT. Set a positive +# value only when your monitoring cadence needs a different freshness window. +HEALTHCHECK_MAX_AGE=0 diff --git a/.gitignore b/.gitignore index 8435fdd..d84af9d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,11 @@ !.env.example logs/ +secrets/ *.log +*.env.backup* +*-after-incident.txt +diagnose-*.txt .DS_Store Thumbs.db diff --git a/Makefile b/Makefile index 1b64e6f..2befcef 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,26 @@ -.PHONY: test validate docker-build +.PHONY: test validate validate-example tui docker-build test: bash -n src/FanControlWithEsxiSmart.sh bash -n src/setIdracFanSpeed.sh + bash -n src/fan-control-tui.sh bash tests/fan-control.test.sh + bash tests/tui.test.sh validate: + @test -f .env || { echo "Missing .env; run 'make tui' first" >&2; exit 1; } + docker compose run --rm --no-deps \ + --volume "$(CURDIR)/src/FanControlWithEsxiSmart.sh:/usr/local/bin/fan-control-dev.sh:ro" \ + --entrypoint /usr/local/bin/fan-control-dev.sh \ + idrac-fan-control validate + +validate-example: DRY_RUN=true OPERATION_MODE=auto TEMPERATURE_SOURCES=idrac \ - IDRAC_IP=192.0.2.10 IDRAC_ID=root IDRAC_PASSWORD=test \ + IDRAC_IP=10.0.0.10 IDRAC_ID=root IDRAC_PASSWORD=test \ src/FanControlWithEsxiSmart.sh validate +tui: + src/fan-control-tui.sh + docker-build: docker build -t idrac-fan-control:local . diff --git a/README.md b/README.md index 116f521..e9ad9fc 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,86 @@ # iDRAC Fan Speed Control -[![Docker image](https://img.shields.io/badge/GHCR-idrac--fan--control-blue)](https://github.com/DF-wu/iDRACFanSpeedControl/pkgs/container/idrac-fan-control) -[![Tests](https://img.shields.io/badge/tests-make%20test-green)](#testing) +[![Tests](https://img.shields.io/badge/tests-make%20test-green)](#測試與品質門檻) +[![Container](https://img.shields.io/badge/container-GHCR-blue)](https://github.com/DF-wu/iDRACFanSpeedControl/pkgs/container/idrac-fan-control) [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) -Dell PowerEdge 伺服器風扇控制工具。它透過 iDRAC/IPMI raw command 設定風扇轉速,並可依 ESXi NVMe SMART、iDRAC temperature sensors、NVIDIA GPU 溫度自動調整。 +這是一個給 Dell PowerEdge 使用的風扇控制器。它透過 iDRAC 的 IPMI OEM raw command 設定 fan duty cycle,再用 ESXi NVMe SMART、iDRAC temperature sensor、NVIDIA GPU 中任一個或多個來源決定風扇曲線。 -> Safety first: 這個專案會覆寫 Dell 原廠風扇控制。設定錯誤可能讓硬體過熱。第一次使用請在旁監看溫度、保留 iDRAC Web UI 存取權,並確認 `restore` 指令可用。 +> [!CAUTION] +> 這個程式會暫時覆寫 Dell 原廠風扇控制。第一次設定請保留 iDRAC Web UI 或實體主控台,先用 `manual`、`restore`、`diagnose` 驗證,再讓 `auto` 長時間執行。任何過熱、讀值不可信或行為異常時,立即執行 `restore` 並停止容器。 -![iDRAC 8 IPMI over LAN setting](images/image.png) +## 先看懂它如何工作 -## What It Does +```mermaid +flowchart LR + TUI[make tui\n互動式設定] --> ENV[.env\n600 permissions] + ENV --> COMPOSE[Docker Compose] + COMPOSE --> CTRL[fan-control.sh] + CTRL --> IPMI[iDRAC / IPMI\nfan raw command] + CTRL --> ESXI[ESXi\nesxcli SMART] + CTRL --> SDR[iDRAC\nTemperature SDR] + CTRL --> GPU[NVIDIA\nnvidia-smi] + CTRL --> LOG[logs/fan_control.log\nhealthcheck] +``` -- `auto`: 持續讀取溫度,依 fan curve 設定風扇。 -- `once`: 執行一次溫度讀取與風扇調整後離開。 -- `manual`: 手動設定 1-100% 風扇 duty cycle。 -- `restore`: 將 iDRAC 還原為 Dell 自動風扇控制。 -- `status`: 印出 iDRAC chassis status 與 temperature sensors。 -- `validate`: 檢查本機設定是否足以啟動目前模式。 +控制器只會對 iDRAC 發送風扇命令;溫度來源全部是讀取。多來源時採用最高的有效決策溫度,GPU 會先減去 `GPU_TEMP_OFFSET`,避免 GPU 的封裝溫度直接把機箱風扇推到不必要的高轉速。 -## Improvements In This Version +![iDRAC IPMI over LAN 設定畫面](images/image.png) -- 溫度來源可選 `esxi`、`idrac`、`gpu`,也可混用。 -- 溫度讀取失敗時套用 `FAILSAFE_FAN_SPEED`,不再掉到最低風扇轉速。 -- 支援 hysteresis,避免溫度卡在閾值附近時風扇反覆跳速。 -- 支援 ESXi SSH password 或 key authentication。 -- IPMI 密碼改用 `IPMI_PASSWORD` environment 傳給 `ipmitool -E`,避免出現在 command line。 -- 舊手動腳本已改成 wrapper,共用同一份核心邏輯。 -- 新增 `make test`、Docker healthcheck、`.dockerignore`、`.gitignore`、MIT `LICENSE`。 +*圖 1:在 iDRAC Web UI 開啟 IPMI over LAN;實際選項名稱會依 iDRAC 韌體版本略有不同。* -## Requirements +## 快速開始:用 TUI 完成安全的第一次設定 -| Requirement | Why | -| --- | --- | -| Dell PowerEdge with iDRAC | 送出 IPMI fan raw commands | -| iDRAC IPMI over LAN enabled | `ipmitool -I lanplus` 需要 LAN access | -| Docker / Docker Compose | 建議部署方式 | -| ESXi SSH access | 只有 `TEMPERATURE_SOURCES` 包含 `esxi` 時需要 | -| NVIDIA Container Toolkit | 只有 `gpu` 溫度來源需要 | +建議在 Docker 主機上操作。TUI 不需要 `dialog`、Python 或其他額外套件;它使用純 Bash,會保留 `.env.example` 的註解、以原子方式更新檔案,並將檔案權限設為 `600`。 -Tested target: Dell PowerEdge R730xd/R730XD class servers with iDRAC 8. Other Dell models may support the same OEM raw commands, but you should test with `manual` and `restore` before running unattended. +```bash +git clone https://github.com/DF-wu/iDRACFanSpeedControl.git +cd iDRACFanSpeedControl +make tui +``` + +主畫面如下(每次進入子頁都會寫回 `.env`,離開時會再次顯示檔案位置): -## Quick Start +```text +╭────────────────────────────────────────────────────────────╮ +│ iDRAC Fan Control · Setup TUI │ +╰────────────────────────────────────────────────────────────╯ + + 1) Quick setup wizard 6) Safety, timing, and logging + 2) iDRAC / IPMI settings 7) Review redacted configuration + 3) Temperature source 8) Validate configuration + 4) ESXi NVMe source 9) Run read-only diagnostics + 5) Fan curve 0) Save and exit +``` -1. Clone the repository. +第一次建議選 `iDRAC sensors only`,先不要把 ESXi 或 GPU 加入決策。TUI 完成後,依序執行: ```bash -git clone https://github.com/DF-wu/iDRACFanSpeedControl.git -cd iDRACFanSpeedControl +make validate +docker compose run --rm idrac-fan-control diagnose +docker compose up -d +docker logs -f idrac-fan-control +``` + +`diagnose` 是唯讀檢查,不會送出風扇 raw command。它會逐項顯示 iDRAC、每個溫度來源、可寫 log 路徑與「如果現在控制會選哪一檔」的 preview。若某個來源不用,從 TUI 的 source preset 移除它,而不是忽略錯誤後繼續運行。 + +若要編輯另一份設定檔: + +```bash +src/fan-control-tui.sh --config /path/to/staging.env ``` -2. Create a private environment file. +### 手動建立 `.env`(不用 TUI 也可以) ```bash cp .env.example .env chmod 600 .env +$EDITOR .env ``` -3. Edit `.env`. +最小的 iDRAC-only 範例: -Minimum iDRAC-only auto mode: - -```env +```dotenv IDRAC_IP=192.0.2.10 IDRAC_ID=root IDRAC_PASSWORD=change-me @@ -69,343 +88,283 @@ OPERATION_MODE=auto TEMPERATURE_SOURCES=idrac ``` -ESXi NVMe SMART mode: +先將 `IDRAC_PASSWORD`、IP 與管理網路換成真實值。`.env.example` 的 `192.0.2.*`、`change-me`、`replace_with_*` 都是故意放的 placeholder,`validate` 會拒絕它們。 -```env -TEMPERATURE_SOURCES=esxi -ESXI_HOST=192.0.2.20 -ESXI_USERNAME=root -ESXI_PASSWORD=change-me -DRIVE_DEVICE=t10.NVMe____replace_with_your_device_identifier -``` +## 命令速查 -4. Validate and start. +| 命令 | 作用 | 是否改變風扇 | +| --- | --- | --- | +| `auto` | 持續依曲線控制,離開時依設定還原自動模式 | 是 | +| `once` | 讀取一次並套用一個決策後離開 | 是 | +| `manual 35` | 設定 35% duty cycle;未帶數字時讀 `MANUAL_FAN_SPEED` | 是 | +| `restore` | 發送 Dell 自動風扇控制命令 | 是(還原) | +| `status` | 顯示 chassis 與原始 temperature SDR | 否 | +| `config` | 顯示有效設定摘要,密碼只顯示「set/長度」 | 否 | +| `validate` | 檢查數值、placeholder、依賴命令與來源條件 | 否 | +| `diagnose` | 逐項連線與溫度讀值探測,含決策 preview | 否 | +| `healthcheck` | 檢查設定與 auto 控制 log 是否在合理時間內更新 | 否 | + +Docker Compose 用法: ```bash +docker compose run --rm idrac-fan-control config docker compose run --rm idrac-fan-control validate -docker compose up -d -docker logs -f idrac-fan-control -``` - -5. Restore Dell automatic control when needed. - -```bash -docker compose run --rm idrac-fan-control restore -``` - -## Commands - -Docker Compose passes extra arguments to the container entrypoint: - -```bash -# Run continuously according to OPERATION_MODE=auto -docker compose up -d - -# Run one automatic cycle +docker compose run --rm idrac-fan-control diagnose +docker compose run --rm idrac-fan-control status docker compose run --rm idrac-fan-control once - -# Set manual fan speed to 35% docker compose run --rm idrac-fan-control manual 35 - -# Restore Dell automatic fan control docker compose run --rm idrac-fan-control restore - -# Print chassis and temperature sensor status -docker compose run --rm idrac-fan-control status - -# Validate config without touching fan speed -docker compose run --rm idrac-fan-control validate ``` -Local script usage is the same: +本機腳本可直接使用相同命令: ```bash -src/FanControlWithEsxiSmart.sh manual 35 -src/FanControlWithEsxiSmart.sh restore +src/FanControlWithEsxiSmart.sh diagnose +src/FanControlWithEsxiSmart.sh config ``` -## Temperature Sources - -`TEMPERATURE_SOURCES` accepts comma-separated values. - -| Source | Reads | Requires | -| --- | --- | --- | -| `esxi` | One NVMe drive's SMART temperature through `esxcli` | ESXi SSH, `DRIVE_DEVICE` | -| `idrac` | iDRAC temperature sensors from `ipmitool sdr type Temperature` | iDRAC credentials | -| `gpu` | NVIDIA GPU temperature from `nvidia-smi` | NVIDIA Container Toolkit | - -Hybrid mode uses the highest decision temperature: - -```text -decision_temperature = max(esxi_temp, idrac_sensor_temps..., gpu_temp - GPU_TEMP_OFFSET) +本機執行需要 `bash`、`coreutils`、`ipmitool`、`timeout`,ESXi password 模式另需 `sshpass`;Docker image 已包含這些依賴。 + +## 安全啟動順序 + +```mermaid +sequenceDiagram + participant U as 操作者 + participant C as Controller + participant D as iDRAC + U->>C: validate + C-->>U: 靜態設定與依賴結果 + U->>C: diagnose + C->>D: mc info / SDR (read-only) + D-->>C: 連線與溫度 + C-->>U: source checks + decision preview + U->>C: manual 30 + C->>D: set fan duty (受控測試) + U->>C: restore + C->>D: Dell automatic fan control + U->>C: auto ``` -Examples: - -```env -# No ESXi dependency. Use iDRAC sensors only. -TEMPERATURE_SOURCES=idrac - -# NVMe SMART plus iDRAC ambient/CPU sensors. -TEMPERATURE_SOURCES=esxi,idrac - -# NVMe SMART plus GPU compensation. -TEMPERATURE_SOURCES=esxi,gpu -GPU_TEMP_OFFSET=15 - -# Backward-compatible switch. This appends gpu to TEMPERATURE_SOURCES. -WITH_GPU_TEMP=true +正式開始前逐項確認: + +1. iDRAC Web UI 的 IPMI over LAN 已開啟,管理網路可達。 +2. iDRAC 帳號具備必要權限,密碼不再是 placeholder。 +3. `manual 30` 能設定低速,`manual 70` 能設定保守速度,`restore` 能回到 Dell 自動模式。 +4. `diagnose` 的每一個已啟用來源都是 `[PASS]`;不用的來源請停用。 +5. 先以較保守的 `FAILSAFE_FAN_SPEED` 和 `RESTORE_AUTO_ON_EXIT=true` 運行。 + +## 溫度來源與控制決策 + +`TEMPERATURE_SOURCES` 使用逗號分隔,可填 `esxi`、`idrac`、`gpu`。來源可以混合;控制器會保留每個來源的 label 供 log 與診斷追蹤。 + +| 來源 | 讀取內容 | 必要條件 | 失敗時的行為 | +| --- | --- | --- | --- | +| `idrac` | `ipmitool sdr type Temperature` 中可讀的 sensor | iDRAC IPMI | 該來源標記失敗 | +| `esxi` | 指定 NVMe 的 `esxcli storage core device smart get` | SSH、`DRIVE_DEVICE` | 該來源標記失敗 | +| `gpu` | `nvidia-smi` 的每張 GPU 溫度 | NVIDIA Container Toolkit/驅動 | 該來源標記失敗 | + +```mermaid +flowchart TD + R[所有有效 readings] --> M{取最高 adjusted temperature} + M --> L[temperature_level] + L --> H{比上一檔更低?} + H -- 否 --> S[立即升檔或維持] + H -- 是 --> X{低於前一檔 threshold - HYSTERESIS?} + X -- 否 --> S2[維持上一檔] + X -- 是 --> S3[降到新檔] + S --> F[套用 fan speed] + S2 --> F + S3 --> F + R -. 全部失敗 .-> FS[FAILSAFE_FAN_SPEED] ``` -## Fan Curve - -The controller maps the decision temperature to a fan level. +預設曲線如下;`validate` 會確保 threshold 嚴格遞增、fan speed 不遞減,且 fail-safe 不低於 critical speed。 -| Level | Temperature range | Default speed | +| Level | 決策溫度 | 預設風扇 | | --- | --- | --- | -| `idle` | `< TEMP_LOW` | `FAN_SPEED_IDLE=25` | -| `low` | `TEMP_LOW` to `< TEMP_MEDIUM` | `FAN_SPEED_LOW=30` | -| `medium` | `TEMP_MEDIUM` to `< TEMP_HIGH` | `FAN_SPEED_MEDIUM=40` | -| `high` | `TEMP_HIGH` to `< TEMP_CRITICAL` | `FAN_SPEED_HIGH=50` | -| `critical` | `>= TEMP_CRITICAL` | `FAN_SPEED_CRITICAL=60` | -| `failsafe` | all temperature sources failed | `FAILSAFE_FAN_SPEED=70` | - -Default thresholds: - -```env -TEMP_LOW=65 -TEMP_MEDIUM=70 -TEMP_HIGH=75 -TEMP_CRITICAL=80 -HYSTERESIS=2 -``` +| `idle` | `<65°C` | 25% | +| `low` | `65–69°C` | 30% | +| `medium` | `70–74°C` | 40% | +| `high` | `75–79°C` | 50% | +| `critical` | `>=80°C` | 60% | +| `failsafe` | 所有來源失敗 | 70% | -Hysteresis only delays downshifts. For example, if the current level is `high` and `TEMP_HIGH=75`, the controller keeps `high` until the temperature falls to `73C` or lower when `HYSTERESIS=2`. +降檔會等待 `HYSTERESIS` 度,避免溫度在閾值附近來回跳速;升檔不延遲。GPU 的 adjusted 值為 `GPU_TEMP - GPU_TEMP_OFFSET`,最低不會小於 0°C。 -## Configuration Reference +## 設定參考 -| Variable | Default | Notes | +### 連線與模式 + +| 變數 | 預設 | 說明 | +| --- | --- | --- | +| `IDRAC_IP` | 空 | iDRAC hostname 或 IP;所有 fan 命令必要 | +| `IDRAC_ID` | `root` | iDRAC 使用者 | +| `IDRAC_PASSWORD` | 空 | 由 `IPMI_PASSWORD` environment 傳給 `ipmitool -E`,不出現在 argv | +| `IPMI_INTERFACE` | `lanplus` | 通常保持不變 | +| `IPMI_TIMEOUT` / `IPMI_RETRIES` | `5` / `2` | 單次 timeout 與 retry | +| `OPERATION_MODE` | `manual` | `auto`、`once`、`manual` | +| `DRY_RUN` | `false` | 只在測試命令組合時使用;不會產生真實讀值 | +| `COMMAND_TIMEOUT` | `20` | SSH、IPMI、GPU 外部命令上限(秒) | +| `CHECK_INTERVAL` | `60` | auto 每個 cycle 間隔(秒) | +| `RESTORE_AUTO_ON_EXIT` | `true` | auto 收到 SIGTERM/離開時還原 | + +### 溫度來源 + +| 變數 | 預設 | 說明 | | --- | --- | --- | -| `IDRAC_IP` | empty | Required for all fan commands | -| `IDRAC_ID` | `root` | iDRAC username | -| `IDRAC_PASSWORD` | empty | iDRAC password | -| `IPMI_INTERFACE` | `lanplus` | Usually keep this value | -| `IPMI_TIMEOUT` | `5` | Seconds per IPMI attempt | -| `IPMI_RETRIES` | `2` | IPMI retry count | -| `OPERATION_MODE` | `manual` | `auto`, `once`, or `manual` | -| `TEMPERATURE_SOURCES` | `esxi` | Comma-separated: `esxi,idrac,gpu` | -| `WITH_GPU_TEMP` | `false` | Backward-compatible GPU switch | -| `GPU_TEMP_OFFSET` | `15` | Subtracted from GPU temperature | -| `CHECK_INTERVAL` | `60` | Seconds between auto cycles | -| `COMMAND_TIMEOUT` | `20` | Wrapper timeout for SSH, IPMI, and GPU reads | -| `ESXI_HOST` | empty | Required when source includes `esxi` | -| `ESXI_USERNAME` | `root` | ESXi SSH username | -| `ESXI_PASSWORD` | empty | Use password auth when `ESXI_SSH_KEY` is empty | -| `ESXI_SSH_KEY` | empty | Optional SSH private key path | -| `ESXI_SSH_PORT` | `22` | ESXi SSH port | -| `SSH_CONNECT_TIMEOUT` | `10` | SSH connection timeout | -| `DRIVE_DEVICE` | empty | ESXi NVMe device identifier | -| `TEMP_LOW` | `65` | Idle to low threshold | -| `TEMP_MEDIUM` | `70` | Low to medium threshold | -| `TEMP_HIGH` | `75` | Medium to high threshold | -| `TEMP_CRITICAL` | `80` | High to critical threshold | -| `FAN_SPEED_IDLE` | `25` | Fan speed below `TEMP_LOW` | -| `FAN_SPEED_LOW` | `30` | Fan speed at low level | -| `FAN_SPEED_MEDIUM` | `40` | Fan speed at medium level | -| `FAN_SPEED_HIGH` | `50` | Fan speed at high level | -| `FAN_SPEED_CRITICAL` | `60` | Fan speed at critical level | -| `HYSTERESIS` | `2` | Degrees C before downshift | -| `FAILSAFE_ON_ERROR` | `true` | Apply fail-safe when all sources fail | -| `FAILSAFE_FAN_SPEED` | `70` | Conservative speed for sensor failure | -| `RESTORE_AUTO_ON_EXIT` | `true` | Restore Dell auto control when auto mode exits | -| `MANUAL_FAN_SPEED` | empty | Used by `manual` mode without an argument | -| `LOG_DIR` | `/var/log/fan-control` | Set empty only for tests | -| `LOG_FILE` | `fan_control.log` | Status log file name | -| `DRY_RUN` | `false` | Log commands without running `ipmitool` | - -## Getting The ESXi Drive Identifier - -SSH to the ESXi host and list storage devices: +| `TEMPERATURE_SOURCES` | `esxi` | `esxi,idrac,gpu` 的逗號清單 | +| `WITH_GPU_TEMP` | `false` | 舊版相容開關;true 會附加 `gpu` | +| `GPU_TEMP_OFFSET` | `15` | GPU 溫度的補償度數 | +| `ESXI_HOST` / `ESXI_USERNAME` | 空 / `root` | ESXi SSH 目標 | +| `ESXI_PASSWORD` | 空 | 沒有 key 時使用;TUI 不會回顯 | +| `ESXI_SSH_KEY` | 空 | 設定後優先使用 key authentication | +| `ESXI_SSH_PORT` | `22` | SSH port(1–65535) | +| `SSH_CONNECT_TIMEOUT` | `10` | SSH 建立連線上限(秒) | +| `SSH_STRICT_HOST_KEY_CHECKING` | `accept-new` | `yes`、`no`、`ask` 或 `accept-new` | +| `DRIVE_DEVICE` | 空 | `esxcli storage core device list` 找到的完整 ID | +| `IDRAC_SENSOR_INCLUDE_REGEX` | 空 | 只保留符合 sensor 名稱的 awk regex | +| `IDRAC_SENSOR_EXCLUDE_REGEX` | `no reading\|disabled\|not readable` | 排除無效 SDR | + +### 風扇曲線、安全與診斷 + +| 變數 | 預設 | 說明 | +| --- | --- | --- | +| `TEMP_LOW/MEDIUM/HIGH/CRITICAL` | `65/70/75/80` | 嚴格遞增的°C threshold | +| `FAN_SPEED_IDLE/LOW/MEDIUM/HIGH/CRITICAL` | `25/30/40/50/60` | 1–100%,不可遞減 | +| `HYSTERESIS` | `2` | 降檔所需低於前 threshold 的°C | +| `FAILSAFE_ON_ERROR` | `true` | 所有來源失敗時使用保守速度 | +| `FAILSAFE_FAN_SPEED` | `70` | 必須 >= critical speed | +| `MANUAL_FAN_SPEED` | `35` | `manual` 未帶參數時使用 | +| `LOG_DIR` / `LOG_FILE` | `/var/log/fan-control` / `fan_control.log` | 持久化狀態 log | +| `LOG_LEVEL` | `INFO` | `DEBUG` 會增加命令與來源細節,絕不列密碼 | +| `HEALTHCHECK_MAX_AGE` | `0` | 0 代表 `CHECK_INTERVAL*3 + COMMAND_TIMEOUT` | -```bash -ssh root@192.0.2.20 -esxcli storage core device list -``` +## Docker 部署與 GPU -Find the NVMe device ID and test SMART temperature output: +Compose 預設使用 GHCR image、host network 與 `./logs` volume: ```bash -esxcli storage core device smart get -d t10.NVMe____replace_with_your_device_identifier +docker compose pull +docker compose up -d +docker compose ps +docker inspect --format '{{.State.Health.Status}}' idrac-fan-control ``` -Use that full ID as `DRIVE_DEVICE`. +`healthcheck` 不只驗證 env;在 `OPERATION_MODE=auto` 時也會確認 `fan_control.log` 在 `HEALTHCHECK_MAX_AGE` 內更新。若資料來源全部失敗且 `FAILSAFE_ON_ERROR=false`,控制 cycle 不會寫成功紀錄,容器會變成 unhealthy,這是刻意的安全訊號。 -## GPU Mode +GPU 模式需要 NVIDIA Container Toolkit。編輯 `docker-compose.yml`,取消 `gpus: all` 註解,再設定: -Install NVIDIA Container Toolkit on the Docker host, then enable GPU access in `docker-compose.yml`: - -```yaml -gpus: all -``` - -Set one of these in `.env`: - -```env -TEMPERATURE_SOURCES=esxi,gpu -# or -WITH_GPU_TEMP=true +```dotenv +TEMPERATURE_SOURCES=idrac,gpu +GPU_TEMP_OFFSET=15 ``` -Verify GPU access: +驗證: ```bash docker run --rm --gpus all nvidia/cuda:12.9.0-runtime-ubuntu24.04 nvidia-smi -docker compose run --rm idrac-fan-control status -``` - -## Logs - -Container logs show control decisions: - -```bash -docker logs -f idrac-fan-control -``` - -Persistent fan-control logs are written to `./logs/fan_control.log` by the Compose volume: - -```bash -tail -f logs/fan_control.log +docker compose run --rm idrac-fan-control diagnose ``` -Each line includes `status`, `temp`, `level`, `fan`, and source details. - -## Testing - -Run local tests: +若不需要 GPU,使用不含 CUDA 的本機 build 可降低 image 大小: ```bash -make test +docker build --build-arg BASE_IMAGE=ubuntu:24.04 -t idrac-fan-control:local . ``` -The tests cover: - -- Bash syntax checks. -- Temperature source normalization. -- iDRAC sensor parsing. -- GPU offset decision logic. -- Fan curve and hysteresis behavior. -- Fail-safe behavior when every source fails. -- Validation for iDRAC-only auto mode. +## Debug、log 與故障排除 -Run a dry validation target: +正常 log 範例: -```bash -make validate +```text +2026-07-18 03:12:10 [INFO] Control temp 68C -> low (30%). Sources: idrac:Inlet Temp=68C ``` -Build a local image: +需要更多上下文時暫時設定: -```bash -make docker-build +```dotenv +LOG_LEVEL=DEBUG ``` -If you do not need GPU support, build with a smaller Ubuntu base: +再觀察: ```bash -docker build --build-arg BASE_IMAGE=ubuntu:24.04 -t idrac-fan-control:local . +docker logs -f idrac-fan-control +tail -f logs/fan_control.log +docker compose run --rm idrac-fan-control diagnose ``` -## Troubleshooting - -### `validate` says iDRAC config is missing +`DEBUG` 只會印目標、命令種類、來源選擇與失敗階段;密碼不會放在 argv,也不會寫入 config summary。完整的症狀→檢查→修復表請看 [docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md)。日常與緊急操作請看 [USAGE_GUIDE.md](USAGE_GUIDE.md)。 -Set `IDRAC_IP`, `IDRAC_ID`, and `IDRAC_PASSWORD` in `.env`. Also confirm IPMI over LAN is enabled in iDRAC. +常見最短修復路徑: ```bash -ipmitool -I lanplus -H "$IDRAC_IP" -U "$IDRAC_ID" -P "$IDRAC_PASSWORD" chassis status -``` - -### Auto mode fails because ESXi is missing - -If you do not want ESXi SSH, use iDRAC sensors: +# 任何不確定的風險先還原 Dell 自動控制 +docker compose run --rm idrac-fan-control restore -```env -TEMPERATURE_SOURCES=idrac +# 看目前有效設定與依賴狀態 +docker compose run --rm idrac-fan-control config +docker compose run --rm idrac-fan-control diagnose ``` -If you want ESXi SMART temperature, set `ESXI_HOST`, `ESXI_USERNAME`, `ESXI_PASSWORD` or `ESXI_SSH_KEY`, and `DRIVE_DEVICE`. +## 安全與備份 -### Temperature reads fail and fans jump to fail-safe +- `.env`、`logs/`、私鑰與 incident dump 都不應提交到 Git;TUI 會將設定檔權限設為 `600`。 +- Compose 會以 read-only 掛載 gitignored 的 `./secrets` 到 `/run/secrets`;ESXi key 請使用容器內路徑(例如 `/run/secrets/esxi_ed25519`)。 +- 優先使用 `ESXI_SSH_KEY`;若必須用 password,控制器會以 `SSHPASS` environment 搭配 `sshpass -e`,不把密碼放進 argv。 +- iDRAC 密碼透過 `IPMI_PASSWORD` environment 搭配 `ipmitool -E`;`config`/TUI review 只顯示 set 與字元數。 +- iDRAC/ESXi 應位於隔離管理網路,避免把 IPMI over LAN 暴露到公網。 +- 每次變更 `.env` 前保留一份 `chmod 600` 的離線備份,並測試 `restore`。 -That is intentional. The controller uses `FAILSAFE_FAN_SPEED` when no temperature source succeeds. Check the source-specific logs, then test each dependency: +## 從哪裡取得 ESXi drive identifier ```bash -docker compose run --rm idrac-fan-control status -ssh root@your-esxi-host -nvidia-smi -``` - -### Manual mode works, but auto mode is too noisy - -Raise the fan curve or add hysteresis: - -```env -TEMP_LOW=70 -TEMP_MEDIUM=75 -TEMP_HIGH=80 -TEMP_CRITICAL=85 -HYSTERESIS=3 +ssh root@ESXI_HOST +esxcli storage core device list +esxcli storage core device smart get -d 't10.NVMe____完整識別字串' ``` -### Restore automatic control +把完整 identifier 放到 `DRIVE_DEVICE`,不要自行縮短。若 SMART 輸出沒有 `Drive Temperature` 數值,`diagnose` 會將 ESXi source 標成 FAIL,而不會偷偷把錯誤當成 0°C。 -Run: +## 測試與品質門檻 ```bash -docker compose run --rm idrac-fan-control restore -``` - -This sends: - -```text -raw 0x30 0x30 0x01 0x01 +make test # bash -n + 37 個核心 assertions + 10 個 TUI assertions +make validate # 以 Docker 依賴檢查目前的 .env(不改變風扇) +make validate-example # 不需要 .env 的 DRY_RUN smoke test +make docker-build ``` -## Security Notes - -- Keep `.env` private. This repository now ignores `.env`; if your clone still tracks it, run `git rm --cached .env`. -- Prefer a dedicated iDRAC account if your iDRAC version supports suitable privileges. -- Keep iDRAC and ESXi on a management network, not a public network. -- Prefer `ESXI_SSH_KEY` over password auth when possible. -- Review logs after every fan curve change. +測試涵蓋 source normalization、SDR parsing、GPU offset、hysteresis、fail-safe、曲線/port/regex 驗證、credential redaction、health freshness、診斷 preview,以及 TUI 設定檔的安全 round-trip。真實硬體、ESXi SSH 和 GPU 仍須在你的管理網路上以 `diagnose` 驗證。 -## Repository Layout +## 專案結構 ```text . ├── src/ -│ ├── FanControlWithEsxiSmart.sh # main entrypoint -│ └── setIdracFanSpeed.sh # backward-compatible manual wrapper +│ ├── FanControlWithEsxiSmart.sh # 控制器、validate、diagnose、healthcheck +│ ├── fan-control-tui.sh # 純 Bash 設定 TUI +│ └── setIdracFanSpeed.sh # 舊腳本相容 wrapper ├── tests/ -│ └── fan-control.test.sh # shell tests with mocked behavior -├── images/ -│ └── image.png # iDRAC IPMI over LAN screenshot -├── .env.example # documented configuration template +│ ├── fan-control.test.sh # 核心邏輯與安全測試 +│ └── tui.test.sh # .env parser / writer 測試 +├── docs/ +│ └── TROUBLESHOOTING.md # 症狀導向除錯手冊 +├── images/image.png # iDRAC IPMI 設定畫面 +├── .env.example # 帶完整註解的設定模板 ├── docker-compose.yml ├── Dockerfile ├── Makefile -└── README.md +├── README.md +└── USAGE_GUIDE.md ``` -## Roadmap +## 相容性與限制 -- Add structured metrics output for Prometheus or node exporters. -- Add optional fan ramping for smoother transitions. -- Add log rotation examples for long-running hosts. -- Add model-specific notes for more Dell PowerEdge generations. +專案以 Dell PowerEdge R730/R730xd、iDRAC 8 類機型的 OEM fan raw command 為主要驗證目標。其他世代可能相容,但不能假設相同;請在無人值守前完成 `manual`/`restore`/`diagnose`。iDRAC、ESXi、GPU 的實際 sensor 名稱與權限會依韌體和驅動改變,應以你自己的 diagnostic output 為準。 -## License +## 授權 -MIT. See [LICENSE](LICENSE). +MIT,詳見 [LICENSE](LICENSE)。 -Last reviewed: 2026-06-08. +文件最後檢視:2026-07-18。 diff --git a/USAGE_GUIDE.md b/USAGE_GUIDE.md index a6e09c5..1f8d5e0 100644 --- a/USAGE_GUIDE.md +++ b/USAGE_GUIDE.md @@ -1,127 +1,181 @@ -# iDRAC Fan Control Runbook +# iDRAC Fan Control 日常 Runbook -This runbook is for day-to-day operation. Use [README.md](README.md) for setup, full configuration, and troubleshooting details. +這份文件給已完成初始設定的操作者。第一次安裝、參數定義與架構請先看 [README.md](README.md);遇到錯誤則直接跳到 [docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md)。 -## Daily Commands +## 每次變更前的安全習慣 -Start the controller: - -```bash -docker compose up -d +```text +編輯 .env → validate → diagnose → manual 小幅測試 → restore → 啟動 auto ``` -Watch decisions: +永遠保留一個能登入 iDRAC Web UI 的工作階段。`diagnose` 只讀取,不等於 `manual` 已在你的機型上驗證;第一次仍應完成一次低速、一次保守速度與一次 restore。 + +## 每日操作 + +啟動、查看狀態與持續 log: ```bash +docker compose up -d +docker compose ps docker logs -f idrac-fan-control tail -f logs/fan_control.log ``` -Run one automatic cycle: +`fan_control.log` 的每一筆成功 cycle 會包含 status、決策溫度、level、fan 與來源細節。若 container 顯示 `unhealthy`,不要只重啟;先執行: ```bash -docker compose run --rm idrac-fan-control once +docker compose run --rm idrac-fan-control healthcheck +docker compose run --rm idrac-fan-control diagnose ``` -Set a manual fan speed: +執行一次控制(適合維護視窗): ```bash -docker compose run --rm idrac-fan-control manual 35 +docker compose run --rm idrac-fan-control once ``` -Restore Dell automatic fan control: +查看不改變風扇的摘要: ```bash -docker compose run --rm idrac-fan-control restore +docker compose run --rm idrac-fan-control config +docker compose run --rm idrac-fan-control status ``` -Validate configuration: +## TUI 設定與變更流程 ```bash -docker compose run --rm idrac-fan-control validate +make tui ``` -Print iDRAC status and temperature sensors: +TUI 的每個子頁會直接更新 `.env`,退出時不會自動啟動服務。建議順序: -```bash -docker compose run --rm idrac-fan-control status -``` +1. `Review redacted configuration` 確認來源、曲線與 fail-safe;密碼只會顯示為 set/長度。 +2. `Validate configuration` 修正所有 ERROR;不要把 WARN 當成成功的替代品。 +3. `Run read-only diagnostics` 確認每個選取的 source 都是 PASS。 +4. 回到 shell,執行 `docker compose up -d`。 + +如果只要改一個值,手動編輯 `.env` 也可以;不要直接改容器裡的檔案,因為 container 會被重建。 -## Emergency Procedure +## 手動風扇與還原 -1. Restore Dell automatic fan control. +先在低風險維護時段測試: ```bash +docker compose run --rm idrac-fan-control manual 30 +docker compose run --rm idrac-fan-control status +docker compose run --rm idrac-fan-control manual 70 docker compose run --rm idrac-fan-control restore ``` -2. Stop the container. +`manual` 的百分比範圍是 1–100;`restore` 送出 `raw 0x30 0x30 0x01 0x01`,將控制權交還 Dell。不要用 `DRY_RUN=true` 來判斷真實硬體反應,它只會顯示命令而不觸碰 iDRAC。 -```bash -docker compose down -``` +## 緊急程序 -3. Check temperatures from iDRAC Web UI or `status`. +遇到溫度異常、噪音失控、來源讀值不可信或不確定目前狀態時: -```bash -docker compose run --rm idrac-fan-control status -``` +1. 立即還原 Dell 自動控制。 + + ```bash + docker compose run --rm idrac-fan-control restore + ``` + +2. 停止 controller,避免下一個 cycle 再次覆寫設定。 + + ```bash + docker compose down + ``` -4. Raise `FAILSAFE_FAN_SPEED` or the fan curve before starting auto mode again. +3. 從 iDRAC Web UI 或實體主控台確認溫度與風扇;必要時保持 Dell 自動模式。 -## Common Changes +4. 保存當下輸出供除錯: -Use iDRAC sensors without ESXi: + ```bash + docker compose run --rm idrac-fan-control config > config-after-incident.txt + docker compose run --rm idrac-fan-control diagnose 2>&1 | tee diagnose-after-incident.txt + cp .env .env.after-incident + chmod 600 .env.after-incident + ``` -```env +5. 修正來源、曲線或管理網路後,重新走一次 `validate → diagnose → manual → restore`,才恢復 `auto`。 + +## 常見變更 + +### 改用 iDRAC sensors,不依賴 ESXi + +```dotenv TEMPERATURE_SOURCES=idrac ``` -Use ESXi NVMe SMART plus iDRAC sensors: +### 混合 ESXi 與 iDRAC -```env +```dotenv TEMPERATURE_SOURCES=esxi,idrac +ESXI_HOST=192.0.2.20 +ESXI_USERNAME=root +ESXI_SSH_KEY=/run/secrets/esxi_ed25519 +DRIVE_DEVICE=t10.NVMe____full_identifier +``` + +準備 key volume(不要提交 `secrets/`): + +```bash +mkdir -p secrets +cp ~/.ssh/esxi_ed25519 secrets/esxi_ed25519 +chmod 600 secrets/esxi_ed25519 ``` -Enable GPU temperature: +### 啟用 GPU -```env -TEMPERATURE_SOURCES=esxi,gpu +```dotenv +TEMPERATURE_SOURCES=idrac,gpu GPU_TEMP_OFFSET=15 ``` -Reduce speed changes near thresholds: +同時確認 Compose 的 `gpus: all` 已啟用、主機有 NVIDIA Container Toolkit,再用 `diagnose` 驗證;GPU 失敗不應被悄悄當作 0°C。 -```env +### 降低閾值附近的跳速 + +```dotenv HYSTERESIS=3 ``` -Use a conservative fail-safe: +Hysteresis 只延遲降檔,不延遲升檔。若溫度持續上升,critical 仍會立即生效。 + +### 提高感測器失敗時的保護 -```env +```dotenv FAILSAFE_ON_ERROR=true FAILSAFE_FAN_SPEED=80 ``` -## Maintenance +`FAILSAFE_FAN_SPEED` 必須不低於 `FAN_SPEED_CRITICAL`;`validate` 會拒絕不安全的遞減曲線。 -Update image and restart: +## 更新與回復 ```bash +git pull --ff-only docker compose pull docker compose up -d +docker compose ps ``` -Run tests after local changes: +更新前先備份設定(不要提交到 Git): ```bash -make test +cp .env ".env.backup.$(date +%Y%m%d-%H%M%S)" +chmod 600 .env.backup.* ``` -Build a local image: +若新 image 行為不符預期,先 `restore`、`down`,再切回已知版本 image 或回復程式碼;不要在風扇已異常時反覆 `up`。 + +## 維護與驗證 ```bash +make test +make validate make docker-build ``` -Last reviewed: 2026-06-08. +長時間運行請確認 `logs/` 有輪替策略;專案只負責寫入單一 log,不會替主機設定 logrotate。建議以主機的 logrotate 或 journald retention 管理容量。 + +文件最後檢視:2026-07-18。 diff --git a/docker-compose.yml b/docker-compose.yml index afbd71e..374b065 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ services: - .env volumes: - ./logs:/var/log/fan-control + # Put an ESXi private key in ./secrets and use /run/secrets/ in .env. + - ./secrets:/run/secrets:ro network_mode: host restart: unless-stopped init: true diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md new file mode 100644 index 0000000..6e9bc94 --- /dev/null +++ b/docs/TROUBLESHOOTING.md @@ -0,0 +1,176 @@ +# iDRAC Fan Control 故障排除手冊 + +這份手冊以症狀為入口。任何可能涉及過熱或未知 fan state 的問題,第一步都不是重啟,而是把控制交還 Dell: + +```bash +docker compose run --rm idrac-fan-control restore +docker compose down +``` + +## 標準診斷流程 + +```mermaid +flowchart TD + A[發現錯誤或 unhealthy] --> B{溫度或風扇有立即風險?} + B -- 是/不確定 --> C[restore + docker compose down] + B -- 否 --> D[config + validate] + C --> D + D --> E[LOG_LEVEL=DEBUG] + E --> F[diagnose] + F --> G{iDRAC/IPMI PASS?} + G -- 否 --> H[修復管理網路、權限、IPMI over LAN] + G -- 是 --> I{每個 selected source PASS?} + I -- 否 --> J[修復或從 TEMPERATURE_SOURCES 移除] + I -- 是 --> K[once / manual 維護測試] + K --> L[restore 後再啟動 auto] +``` + +先收集不含明文密碼的證據: + +```bash +docker compose run --rm idrac-fan-control config +docker compose run --rm idrac-fan-control validate +docker compose run --rm idrac-fan-control diagnose +docker compose ps +docker logs --tail 200 idrac-fan-control +tail -n 200 logs/fan_control.log +``` + +需要命令階段與來源選擇細節時,在 `.env` 設 `LOG_LEVEL=DEBUG` 後重跑 `diagnose`。Debug log 不會列出 iDRAC/ESXi 密碼;提交 issue 前仍應檢查主機名稱、IP、device identifier 是否需要遮罩。 + +## Diagnostic output 怎麼讀 + +成功範例: + +```text +iDRAC Fan Control diagnostics (read-only) +[PASS] configuration static validation passed +[PASS] log path /var/log/fan-control +[PASS] iDRAC/IPMI 2.60, PowerEdge R730xd +[PASS] source:idrac Inlet Temp=24C, Exhaust Temp=39C +[PASS] source:esxi t10.NVMe...=46C +[PASS] decision preview 46C -> idle (25%), no command sent + +Result: PASS - controller dependencies are ready. +``` + +`source:*` 只要有一個選取來源失敗,diagnose 就回傳非零,即使另一來源仍可讀。這是為了讓「降級運行」變得可觀測;auto 實際決策仍會使用剩餘的有效 readings,只有全部失敗才進 fail-safe。 + +## 症狀對照表 + +| 症狀/訊息 | 最可能原因 | 確認方式 | 修復 | +| --- | --- | --- | --- | +| `Missing required configuration: IDRAC_IP` | `.env` 未填或 TUI 未完成 | `... config` | `make tui` 填入真實 target | +| `Replace placeholder value` | 還在用 `change-me`、`192.0.2.*` 類模板值 | `... config` | 換成環境實值 | +| `Required command not found: ipmitool` | 直接在 host 跑但未安裝依賴 | `command -v ipmitool` | 用 Docker,或安裝 ipmitool/coreutils | +| `iDRAC/IPMI ... failed` | IPMI over LAN 關閉、網路 ACL、帳密/權限錯 | `ipmitool ... mc info` | 修復 iDRAC 設定與管理網路 | +| `source:idrac ... no valid` | SDR 無 readable temperature 或 regex 過濾全部 | `status`、清空 include regex | 修正 filter;保留預設 exclude | +| `source:esxi ... no valid` | SSH、device ID 或 SMART 格式錯 | ESXi 上直接跑 `esxcli ... smart get` | 修復 key/password、port、完整 ID | +| `source:gpu ... no valid` | 容器無 GPU、驅動/Toolkit 未配置 | `docker ... nvidia-smi` | 啟用 `gpus: all` 與 Toolkit,或移除 gpu source | +| `No valid temperature readings` | 所有 selected sources 都失敗 | `diagnose` | 修復至少一個 source;先保留 fail-safe | +| `Fail-safe fan speed applied` | 上述全部失敗且 fail-safe 開啟 | 同時看前面的 source WARN | 修復來源;這不是曲線 level | +| `Fan speeds must not decrease` | 高溫檔位比低溫檔位低 | TUI Review | 讓 idle ≤ low ≤ medium ≤ high ≤ critical | +| `FAILSAFE_FAN_SPEED must be at least...` | fail-safe 比 critical 還低 | `validate` | 提高 fail-safe 或調整曲線 | +| `Healthcheck control loop is stale` | auto cycle 不再寫 log、volume 或來源失敗 | `healthcheck`、`docker logs` | 修正 cycle 錯誤/volume,不要只提高 max age | +| 容器反覆 restart | 主程序啟動 validation 失敗 | `docker compose logs` | 修正 `.env` 後再 up | +| 關閉容器後仍是 manual fan | restore 指令失敗或容器被強制終止 | 直接執行 `restore` | 確認 iDRAC 可達並從 Web UI 檢查 | + +## iDRAC / IPMI 深入檢查 + +控制器使用 `IPMI_PASSWORD` environment 搭配 `ipmitool -E`,密碼不會出現在 process argv。手動測試時也建議使用相同方式: + +```bash +read -rs IPMI_PASSWORD +export IPMI_PASSWORD +ipmitool -I lanplus -H "$IDRAC_IP" -U "$IDRAC_ID" -E mc info +ipmitool -I lanplus -H "$IDRAC_IP" -U "$IDRAC_ID" -E sdr type Temperature +unset IPMI_PASSWORD +``` + +檢查項目: + +1. iDRAC Web UI 的 IPMI over LAN 已開啟。 +2. host 到 iDRAC 的 UDP 623 未被 ACL/firewall 擋住。 +3. 帳號不是只有 read-only 權限。 +4. `IPMI_INTERFACE=lanplus` 適合該 iDRAC 世代。 +5. timeout 才發生時,可小幅增加 `IPMI_TIMEOUT` 與 `IPMI_RETRIES`,但先排除網路品質。 + +如果 `mc info` 成功但 OEM raw fan command 失敗,機型或 firmware 可能不支援這組 Dell command。不要假設其他 PowerEdge 世代一定相容。 + +## iDRAC sensor filtering + +先看原始輸出: + +```bash +docker compose run --rm idrac-fan-control status +``` + +預設排除 `no reading|disabled|not readable`。若只想採用特定 sensor,可設定 awk regular expression: + +```dotenv +IDRAC_SENSOR_INCLUDE_REGEX=^(Inlet Temp|Exhaust Temp|CPU[12] Temp)$ +``` + +注意 sensor 名稱大小寫由 iDRAC 決定;include regex 套用在 sensor name,exclude regex 套用在轉成小寫的整行。`validate` 會拒絕語法不合法的 regex,但無法預先知道它是否會過濾掉所有實際 sensor,因此仍須跑 `diagnose`。 + +## ESXi NVMe 深入檢查 + +```bash +ssh -p 22 root@ESXI_HOST +esxcli storage core device list +esxcli storage core device smart get -d '完整 DRIVE_DEVICE' +``` + +控制器預期輸出含 `Drive Temperature` 與整數。若裝置只提供其他欄位、輸出語系/格式不同,parser 會安全地回報失敗,不會猜測溫度。SSH key 模式下確認: + +```bash +chmod 600 /path/to/key +ssh -i /path/to/key -o BatchMode=yes root@ESXI_HOST true +``` + +容器要使用 host key file 時,還必須把 key 以 read-only volume 掛進 container,且 `.env` 的 `ESXI_SSH_KEY` 要填 container 內路徑;只填 host path 不會自動掛載。 + +## NVIDIA GPU 深入檢查 + +```bash +nvidia-smi +docker run --rm --gpus all nvidia/cuda:12.9.0-runtime-ubuntu24.04 nvidia-smi +docker compose run --rm idrac-fan-control diagnose +``` + +第一個失敗代表主機驅動問題;第一個成功、第二個失敗通常是 Container Toolkit;兩者成功但 Compose 失敗則檢查 `docker-compose.yml` 的 `gpus: all`。不用 GPU 做控制時,最安全的修復是從 `TEMPERATURE_SOURCES` 移除 `gpu`,而不是把錯誤隱藏。 + +## Healthcheck 與 log + +auto mode 每次成功決策或套用 fail-safe 都會更新 `${LOG_DIR}/${LOG_FILE}`。預設允許的最大年齡為: + +```text +CHECK_INTERVAL * 3 + COMMAND_TIMEOUT +``` + +`HEALTHCHECK_MAX_AGE=0` 代表使用上述公式。只有在外部監控有明確 cadence 需求時才固定正整數;把值設得很大會掩蓋停止更新的控制迴圈。 + +檢查 host volume: + +```bash +ls -ld logs +ls -l logs/fan_control.log +docker inspect idrac-fan-control --format '{{json .Mounts}}' +``` + +專案不自動 rotate log。可用 host logrotate,例如針對 repository 的 `logs/fan_control.log` 設定 weekly rotation;不要在 container 裡安裝 cron,因為 container 重建後不會保留。 + +## 收集 issue 資訊 + +請附上: + +- Server model 與 iDRAC major version(不要附 service tag)。 +- `config` 輸出。 +- `diagnose` 與最近 100 行 container log。 +- 使用的 `TEMPERATURE_SOURCES`。 +- 問題發生前的變更。 +- `make test` 結果。 + +請勿附上 `.env`、密碼、SSH private key、公開可達的管理 IP 或完整 service tag。 + +文件最後檢視:2026-07-18。 diff --git a/src/FanControlWithEsxiSmart.sh b/src/FanControlWithEsxiSmart.sh index 5c6e608..e726b39 100755 --- a/src/FanControlWithEsxiSmart.sh +++ b/src/FanControlWithEsxiSmart.sh @@ -50,6 +50,8 @@ PROGRAM_NAME="$(basename "$0")" : "${LOG_DIR:=/var/log/fan-control}" : "${LOG_FILE:=fan_control.log}" +: "${LOG_LEVEL:=INFO}" +: "${HEALTHCHECK_MAX_AGE:=0}" : "${DRY_RUN:=false}" LAST_LEVEL="" @@ -66,7 +68,9 @@ Usage: ${PROGRAM_NAME} restore Return iDRAC to automatic fan control ${PROGRAM_NAME} status Print iDRAC chassis and temperature status ${PROGRAM_NAME} validate Validate local configuration - ${PROGRAM_NAME} healthcheck Validate configuration for Docker health checks + ${PROGRAM_NAME} config Print the effective configuration (secrets redacted) + ${PROGRAM_NAME} diagnose Probe every configured dependency without changing fans + ${PROGRAM_NAME} healthcheck Check config and automatic control loop freshness The command defaults to OPERATION_MODE when no argument is supplied. EOF @@ -76,12 +80,44 @@ timestamp() { date '+%Y-%m-%d %H:%M:%S' } +log_level_rank() { + case "$(printf '%s' "${1:-}" | tr '[:lower:]' '[:upper:]')" in + DEBUG) printf '10' ;; + INFO) printf '20' ;; + WARN) printf '30' ;; + ERROR) printf '40' ;; + *) printf '20' ;; + esac +} + log() { local level="$1" shift + + if (( $(log_level_rank "$level") < $(log_level_rank "$LOG_LEVEL") )); then + return 0 + fi + printf '%s [%s] %s\n' "$(timestamp)" "$level" "$*" >&2 } +compact_error() { + local value="${1:-}" + value="${value//$'\r'/ }" + value="${value//$'\n'/ }" + value="${value//$'\t'/ }" + value="$(trim "$value")" + printf '%.400s' "$value" +} + +sanitize_log_field() { + local value="${1:-}" + value="${value//$'\r'/ }" + value="${value//$'\n'/ }" + value="${value//\"/\'}" + printf '%s' "$value" +} + trim() { local value="$*" value="${value#"${value%%[![:space:]]*}"}" @@ -122,6 +158,10 @@ is_placeholder_value() { REPLACE_TO_YOUR_*|replace_with_*|your_*|change-me|changeme|t10.NVMe____replace*) return 0 ;; + 192.0.2.*|198.51.100.*|203.0.113.*) + # RFC 5737 documentation networks must never be used for a live target. + return 0 + ;; *) return 1 ;; @@ -196,18 +236,29 @@ source_enabled_in_list() { command_needs_ipmi() { case "$1" in - auto|once|manual|restore|status) return 0 ;; + auto|once|manual|restore|status|diagnose) return 0 ;; *) return 1 ;; esac } command_needs_temperature() { case "$1" in - auto|once) return 0 ;; + auto|once|diagnose) return 0 ;; *) return 1 ;; esac } +validate_awk_regex() { + local name="$1" + local regex="$2" + + [[ -z "$regex" ]] && return 0 + if ! awk -v regex="$regex" 'BEGIN { exit !("probe" ~ regex || "probe" !~ regex) }' /dev/null; then + log "ERROR" "${name} is not a valid awk regular expression" + return 1 + fi +} + validate_sources() { local sources local source @@ -240,7 +291,7 @@ validate_config() { local sources case "$command" in - auto|once|manual|restore|status|validate|healthcheck) ;; + auto|once|manual|restore|status|validate|config|diagnose|healthcheck) ;; help|-h|--help) return 0 ;; *) log "ERROR" "Invalid command or OPERATION_MODE: ${command}" @@ -248,35 +299,80 @@ validate_config() { ;; esac - for bool_name in WITH_GPU_TEMP FAILSAFE_ON_ERROR RESTORE_AUTO_ON_EXIT DRY_RUN; do - if ! is_bool "${!bool_name}"; then - log "ERROR" "${bool_name} must be a boolean value" - error=1 - fi - done - - validate_integer_range "TEMP_LOW" "$TEMP_LOW" 0 120 || error=1 - validate_integer_range "TEMP_MEDIUM" "$TEMP_MEDIUM" 0 120 || error=1 - validate_integer_range "TEMP_HIGH" "$TEMP_HIGH" 0 120 || error=1 - validate_integer_range "TEMP_CRITICAL" "$TEMP_CRITICAL" 0 120 || error=1 - validate_integer_range "FAN_SPEED_IDLE" "$FAN_SPEED_IDLE" 1 100 || error=1 - validate_integer_range "FAN_SPEED_LOW" "$FAN_SPEED_LOW" 1 100 || error=1 - validate_integer_range "FAN_SPEED_MEDIUM" "$FAN_SPEED_MEDIUM" 1 100 || error=1 - validate_integer_range "FAN_SPEED_HIGH" "$FAN_SPEED_HIGH" 1 100 || error=1 - validate_integer_range "FAN_SPEED_CRITICAL" "$FAN_SPEED_CRITICAL" 1 100 || error=1 - validate_integer_range "FAILSAFE_FAN_SPEED" "$FAILSAFE_FAN_SPEED" 1 100 || error=1 - validate_integer_range "CHECK_INTERVAL" "$CHECK_INTERVAL" 1 86400 || error=1 + if ! is_bool "$DRY_RUN"; then + log "ERROR" "DRY_RUN must be a boolean value" + error=1 + fi validate_integer_range "COMMAND_TIMEOUT" "$COMMAND_TIMEOUT" 1 300 || error=1 - validate_integer_range "SSH_CONNECT_TIMEOUT" "$SSH_CONNECT_TIMEOUT" 1 300 || error=1 validate_integer_range "IPMI_TIMEOUT" "$IPMI_TIMEOUT" 1 60 || error=1 validate_integer_range "IPMI_RETRIES" "$IPMI_RETRIES" 0 20 || error=1 - validate_integer_range "GPU_TEMP_OFFSET" "$GPU_TEMP_OFFSET" 0 120 || error=1 - validate_integer_range "HYSTERESIS" "$HYSTERESIS" 0 30 || error=1 - if is_integer "$TEMP_LOW" && is_integer "$TEMP_MEDIUM" && is_integer "$TEMP_HIGH" && is_integer "$TEMP_CRITICAL"; then - if (( TEMP_LOW >= TEMP_MEDIUM || TEMP_MEDIUM >= TEMP_HIGH || TEMP_HIGH >= TEMP_CRITICAL )); then - log "ERROR" "Temperature thresholds must increase: TEMP_LOW < TEMP_MEDIUM < TEMP_HIGH < TEMP_CRITICAL" - error=1 + if [[ "$command" == "manual" && -n "$MANUAL_FAN_SPEED" ]]; then + validate_integer_range "MANUAL_FAN_SPEED" "$MANUAL_FAN_SPEED" 1 100 || error=1 + fi + + if command_needs_temperature "$command"; then + for bool_name in WITH_GPU_TEMP FAILSAFE_ON_ERROR RESTORE_AUTO_ON_EXIT; do + if ! is_bool "${!bool_name}"; then + log "ERROR" "${bool_name} must be a boolean value" + error=1 + fi + done + + validate_integer_range "TEMP_LOW" "$TEMP_LOW" 0 120 || error=1 + validate_integer_range "TEMP_MEDIUM" "$TEMP_MEDIUM" 0 120 || error=1 + validate_integer_range "TEMP_HIGH" "$TEMP_HIGH" 0 120 || error=1 + validate_integer_range "TEMP_CRITICAL" "$TEMP_CRITICAL" 0 120 || error=1 + validate_integer_range "FAN_SPEED_IDLE" "$FAN_SPEED_IDLE" 1 100 || error=1 + validate_integer_range "FAN_SPEED_LOW" "$FAN_SPEED_LOW" 1 100 || error=1 + validate_integer_range "FAN_SPEED_MEDIUM" "$FAN_SPEED_MEDIUM" 1 100 || error=1 + validate_integer_range "FAN_SPEED_HIGH" "$FAN_SPEED_HIGH" 1 100 || error=1 + validate_integer_range "FAN_SPEED_CRITICAL" "$FAN_SPEED_CRITICAL" 1 100 || error=1 + validate_integer_range "FAILSAFE_FAN_SPEED" "$FAILSAFE_FAN_SPEED" 1 100 || error=1 + validate_integer_range "CHECK_INTERVAL" "$CHECK_INTERVAL" 1 86400 || error=1 + validate_integer_range "SSH_CONNECT_TIMEOUT" "$SSH_CONNECT_TIMEOUT" 1 300 || error=1 + validate_integer_range "ESXI_SSH_PORT" "$ESXI_SSH_PORT" 1 65535 || error=1 + validate_integer_range "GPU_TEMP_OFFSET" "$GPU_TEMP_OFFSET" 0 120 || error=1 + validate_integer_range "HYSTERESIS" "$HYSTERESIS" 0 30 || error=1 + validate_integer_range "HEALTHCHECK_MAX_AGE" "$HEALTHCHECK_MAX_AGE" 0 86400 || error=1 + + case "$(printf '%s' "$LOG_LEVEL" | tr '[:lower:]' '[:upper:]')" in + DEBUG|INFO|WARN|ERROR) ;; + *) + log "ERROR" "LOG_LEVEL must be DEBUG, INFO, WARN, or ERROR" + error=1 + ;; + esac + + case "$(printf '%s' "$SSH_STRICT_HOST_KEY_CHECKING" | tr '[:upper:]' '[:lower:]')" in + yes|no|ask|accept-new) ;; + *) + log "ERROR" "SSH_STRICT_HOST_KEY_CHECKING must be yes, no, ask, or accept-new" + error=1 + ;; + esac + + validate_awk_regex "IDRAC_SENSOR_INCLUDE_REGEX" "$IDRAC_SENSOR_INCLUDE_REGEX" || error=1 + validate_awk_regex "IDRAC_SENSOR_EXCLUDE_REGEX" "$IDRAC_SENSOR_EXCLUDE_REGEX" || error=1 + + if is_integer "$TEMP_LOW" && is_integer "$TEMP_MEDIUM" && is_integer "$TEMP_HIGH" && is_integer "$TEMP_CRITICAL"; then + if (( TEMP_LOW >= TEMP_MEDIUM || TEMP_MEDIUM >= TEMP_HIGH || TEMP_HIGH >= TEMP_CRITICAL )); then + log "ERROR" "Temperature thresholds must increase: TEMP_LOW < TEMP_MEDIUM < TEMP_HIGH < TEMP_CRITICAL" + error=1 + fi + fi + + if is_integer "$FAN_SPEED_IDLE" && is_integer "$FAN_SPEED_LOW" && is_integer "$FAN_SPEED_MEDIUM" \ + && is_integer "$FAN_SPEED_HIGH" && is_integer "$FAN_SPEED_CRITICAL"; then + if (( FAN_SPEED_IDLE > FAN_SPEED_LOW || FAN_SPEED_LOW > FAN_SPEED_MEDIUM \ + || FAN_SPEED_MEDIUM > FAN_SPEED_HIGH || FAN_SPEED_HIGH > FAN_SPEED_CRITICAL )); then + log "ERROR" "Fan speeds must not decrease from idle through critical" + error=1 + fi + if is_integer "$FAILSAFE_FAN_SPEED" && (( FAILSAFE_FAN_SPEED < FAN_SPEED_CRITICAL )); then + log "ERROR" "FAILSAFE_FAN_SPEED must be at least FAN_SPEED_CRITICAL" + error=1 + fi fi fi @@ -344,6 +440,7 @@ run_ipmitool() { return 0 fi + log "DEBUG" "Running IPMI command against ${IDRAC_IP}: ${args[*]}" timeout "$COMMAND_TIMEOUT" env IPMI_PASSWORD="$IDRAC_PASSWORD" \ ipmitool -I "$IPMI_INTERFACE" -H "$IDRAC_IP" -U "$IDRAC_ID" -E \ -N "$IPMI_TIMEOUT" -R "$IPMI_RETRIES" "${args[@]}" @@ -380,9 +477,10 @@ run_esxi_command() { fi ssh_command+=("${ESXI_USERNAME}@${ESXI_HOST}" "$remote_command") + log "DEBUG" "Running ESXi command against ${ESXI_HOST}:${ESXI_SSH_PORT} as ${ESXI_USERNAME}" if [[ -n "$ESXI_PASSWORD" && -z "$ESXI_SSH_KEY" ]]; then - timeout "$COMMAND_TIMEOUT" sshpass -p "$ESXI_PASSWORD" "${ssh_command[@]}" + SSHPASS="$ESXI_PASSWORD" timeout "$COMMAND_TIMEOUT" sshpass -e "${ssh_command[@]}" else timeout "$COMMAND_TIMEOUT" "${ssh_command[@]}" fi @@ -394,7 +492,8 @@ get_esxi_drive_temperature() { local temp remote_device="$(remote_quote "$DRIVE_DEVICE")" - if ! output="$(run_esxi_command "esxcli storage core device smart get -d ${remote_device}" 2>/dev/null)"; then + if ! output="$(run_esxi_command "esxcli storage core device smart get -d ${remote_device}")"; then + log "DEBUG" "ESXi SMART query failed for device ${DRIVE_DEVICE}" return 1 fi @@ -410,6 +509,7 @@ get_esxi_drive_temperature() { ' <<< "$output")" if ! is_integer "$temp"; then + log "DEBUG" "ESXi SMART output did not contain a numeric Drive Temperature" return 1 fi @@ -452,12 +552,16 @@ get_idrac_temperatures() { local output local parsed - if ! output="$(run_ipmitool sdr type Temperature 2>/dev/null)"; then + if ! output="$(run_ipmitool sdr type Temperature)"; then + log "DEBUG" "iDRAC sensor query failed" return 1 fi parsed="$(parse_idrac_sdr_temperatures <<< "$output")" - [[ -n "$parsed" ]] || return 1 + if [[ -z "$parsed" ]]; then + log "DEBUG" "iDRAC returned no readable temperature sensors after filtering" + return 1 + fi printf '%s\n' "$parsed" } @@ -469,7 +573,8 @@ get_gpu_temperatures() { command -v nvidia-smi >/dev/null 2>&1 || return 1 - if ! output="$(timeout "$COMMAND_TIMEOUT" nvidia-smi --query-gpu=index,temperature.gpu --format=csv,noheader,nounits 2>/dev/null)"; then + if ! output="$(timeout "$COMMAND_TIMEOUT" nvidia-smi --query-gpu=index,temperature.gpu --format=csv,noheader,nounits)"; then + log "DEBUG" "nvidia-smi temperature query failed" return 1 fi @@ -496,6 +601,7 @@ collect_temperature_readings() { IFS=',' read -r -a source_list <<< "$sources" for source in "${source_list[@]}"; do + log "DEBUG" "Collecting temperature source: ${source}" case "$source" in esxi) if output="$(get_esxi_drive_temperature)"; then @@ -527,8 +633,8 @@ collect_temperature_readings() { return "$found" } -get_decision_temperature() { - local readings +calculate_decision_temperature() { + local readings="$1" local source local label local temp @@ -536,10 +642,6 @@ get_decision_temperature() { local max_temp="" local detail_text="" - if ! readings="$(collect_temperature_readings)"; then - return 1 - fi - while IFS=$'\t' read -r source label temp; do [[ -z "${source:-}" || -z "${temp:-}" ]] && continue is_integer "$temp" || continue @@ -562,6 +664,16 @@ get_decision_temperature() { printf '%s\t%s\n' "$max_temp" "$detail_text" } +get_decision_temperature() { + local readings + + if ! readings="$(collect_temperature_readings)"; then + return 1 + fi + + calculate_decision_temperature "$readings" +} + temperature_level() { local temp="$1" @@ -669,6 +781,7 @@ write_status_log() { local status="$4" local details="${5:-}" + details="$(sanitize_log_field "$details")" [[ -n "$LOG_DIR" ]] || return 0 prepare_log_dir || return 1 printf '%s status=%s temp=%s level=%s fan=%s%% details="%s"\n' \ @@ -764,6 +877,181 @@ once_mode() { control_cycle } +credential_state() { + local value="${1:-}" + + if [[ -z "$value" ]]; then + printf 'not set' + elif is_placeholder_value "$value"; then + printf 'placeholder (replace it)' + else + printf 'set (%s characters)' "${#value}" + fi +} + +config_row() { + printf ' %-30s %s\n' "$1" "$2" +} + +print_effective_config() { + local esxi_auth="password" + + [[ -n "$ESXI_SSH_KEY" ]] && esxi_auth="ssh-key" + + printf 'Effective configuration (credentials are never printed)\n' + printf '%s\n' '------------------------------------------------------------' + config_row "Operation mode" "$OPERATION_MODE" + config_row "Temperature sources" "$(normalize_sources)" + config_row "iDRAC target" "${IDRAC_ID}@${IDRAC_IP:-not set} (${IPMI_INTERFACE})" + config_row "iDRAC password" "$(credential_state "$IDRAC_PASSWORD")" + config_row "ESXi target" "${ESXI_USERNAME}@${ESXI_HOST:-not set}:${ESXI_SSH_PORT}" + config_row "ESXi authentication" "$esxi_auth" + config_row "ESXi password" "$(credential_state "$ESXI_PASSWORD")" + config_row "ESXi drive" "${DRIVE_DEVICE:-not set}" + config_row "Fan thresholds" "${TEMP_LOW}/${TEMP_MEDIUM}/${TEMP_HIGH}/${TEMP_CRITICAL} C" + config_row "Fan speeds" "${FAN_SPEED_IDLE}/${FAN_SPEED_LOW}/${FAN_SPEED_MEDIUM}/${FAN_SPEED_HIGH}/${FAN_SPEED_CRITICAL}%" + config_row "Hysteresis" "${HYSTERESIS} C" + config_row "Fail-safe" "enabled=${FAILSAFE_ON_ERROR}, speed=${FAILSAFE_FAN_SPEED}%" + config_row "Restore on exit" "$RESTORE_AUTO_ON_EXIT" + config_row "Timing" "interval=${CHECK_INTERVAL}s, command-timeout=${COMMAND_TIMEOUT}s" + config_row "Logging" "level=${LOG_LEVEL}, path=${LOG_DIR:-disabled}/${LOG_FILE}" + config_row "Dry run" "$DRY_RUN" +} + +diagnostic_row() { + local status="$1" + local component="$2" + local detail="$3" + printf '[%-4s] %-20s %s\n' "$status" "$component" "$detail" +} + +reading_summary() { + local readings="$1" + local source + local label + local temp + local summary="" + + while IFS=$'\t' read -r source label temp; do + [[ -z "${source:-}" || -z "${temp:-}" ]] && continue + summary="${summary:+${summary}, }${label}=${temp}C" + done <<< "$readings" + + printf '%s' "$summary" +} + +diagnose_mode() { + local failures=0 + local output + local summary + local sources + local source + local readings="" + local decision + local temp + local details + local selected + local level + local speed + local -a source_list + + printf 'iDRAC Fan Control diagnostics (read-only)\n' + printf 'Generated: %s\n\n' "$(timestamp)" + diagnostic_row "PASS" "configuration" "static validation passed" + + if [[ -z "$LOG_DIR" ]] || prepare_log_dir; then + diagnostic_row "PASS" "log path" "${LOG_DIR:-disabled}" + else + diagnostic_row "FAIL" "log path" "cannot write ${LOG_DIR}/${LOG_FILE}" + failures=$((failures + 1)) + fi + + if output="$(run_ipmitool mc info)"; then + summary="$(awk -F: '/Firmware Revision|Product Name/ { gsub(/^[ \t]+|[ \t]+$/, "", $2); printf "%s%s", separator, $2; separator=", " }' <<< "$output")" + diagnostic_row "PASS" "iDRAC/IPMI" "${summary:-connection and authentication succeeded}" + else + diagnostic_row "FAIL" "iDRAC/IPMI" "connection, authentication, or IPMI-over-LAN failed" + failures=$((failures + 1)) + fi + + sources="$(normalize_sources)" + IFS=',' read -r -a source_list <<< "$sources" + for source in "${source_list[@]}"; do + output="" + case "$source" in + esxi) output="$(get_esxi_drive_temperature)" || true ;; + idrac) output="$(get_idrac_temperatures)" || true ;; + gpu) output="$(get_gpu_temperatures)" || true ;; + esac + + if [[ -n "$output" ]]; then + diagnostic_row "PASS" "source:${source}" "$(reading_summary "$output")" + readings="${readings}${readings:+$'\n'}${output}" + else + diagnostic_row "FAIL" "source:${source}" "no valid temperature reading" + failures=$((failures + 1)) + fi + done + + if [[ -n "$readings" ]] && decision="$(calculate_decision_temperature "$readings")"; then + IFS=$'\t' read -r temp details <<< "$decision" + selected="$(choose_fan_speed "$temp")" + IFS=$'\t' read -r level speed <<< "$selected" + diagnostic_row "PASS" "decision preview" "${temp}C -> ${level} (${speed}%), no command sent" + log "DEBUG" "Diagnostic source details: ${details}" + else + diagnostic_row "FAIL" "decision preview" "no usable temperature remains" + failures=$((failures + 1)) + fi + + printf '\nResult: ' + if (( failures == 0 )); then + printf 'PASS - controller dependencies are ready.\n' + return 0 + fi + + printf 'FAIL - %s check(s) need attention. Set LOG_LEVEL=DEBUG for command context.\n' "$failures" + return 1 +} + +healthcheck_mode() { + local log_path + local modified + local now + local age + local max_age + + [[ "$OPERATION_MODE" == "auto" ]] || return 0 + if [[ -z "$LOG_DIR" ]]; then + log "ERROR" "Healthcheck cannot verify auto mode when LOG_DIR is disabled" + return 1 + fi + + log_path="${LOG_DIR}/${LOG_FILE}" + if [[ ! -s "$log_path" ]]; then + log "ERROR" "Healthcheck has no control-cycle record at ${log_path}" + return 1 + fi + + if ! modified="$(stat -c '%Y' "$log_path" 2>/dev/null)"; then + log "ERROR" "Healthcheck cannot read the modification time for ${log_path}" + return 1 + fi + + now="$(date '+%s')" + age=$((now - modified)) + (( age < 0 )) && age=0 + max_age="$HEALTHCHECK_MAX_AGE" + (( max_age == 0 )) && max_age=$((CHECK_INTERVAL * 3 + COMMAND_TIMEOUT)) + + if (( age > max_age )); then + log "ERROR" "Healthcheck control loop is stale: last record ${age}s ago (limit ${max_age}s)" + return 1 + fi + + log "DEBUG" "Healthcheck passed: last control record ${age}s ago" +} + status_mode() { log "INFO" "iDRAC chassis status" run_ipmitool chassis status @@ -781,8 +1069,23 @@ main() { usage return 0 ;; - validate|healthcheck) - validate_config "$OPERATION_MODE" + config) + print_effective_config + return 0 + ;; + validate) + validate_config "$OPERATION_MODE" || return 1 + log "INFO" "Configuration is valid for OPERATION_MODE=${OPERATION_MODE}" + return 0 + ;; + healthcheck) + validate_config "$OPERATION_MODE" || return 1 + healthcheck_mode + return $? + ;; + diagnose) + validate_config "diagnose" || return 1 + diagnose_mode return $? ;; auto|once|manual|restore|status) diff --git a/src/fan-control-tui.sh b/src/fan-control-tui.sh new file mode 100755 index 0000000..05e46d9 --- /dev/null +++ b/src/fan-control-tui.sh @@ -0,0 +1,541 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +TEMPLATE_FILE="${ROOT_DIR}/.env.example" +CONFIG_FILE="${FAN_CONTROL_CONFIG:-${ROOT_DIR}/.env}" +CONTROLLER_SCRIPT="${SCRIPT_DIR}/FanControlWithEsxiSmart.sh" + +CONFIG_KEYS=( + IDRAC_IP IDRAC_ID IDRAC_PASSWORD IPMI_INTERFACE IPMI_TIMEOUT IPMI_RETRIES + OPERATION_MODE CHECK_INTERVAL COMMAND_TIMEOUT DRY_RUN RESTORE_AUTO_ON_EXIT + TEMPERATURE_SOURCES WITH_GPU_TEMP GPU_TEMP_OFFSET + IDRAC_SENSOR_INCLUDE_REGEX IDRAC_SENSOR_EXCLUDE_REGEX + ESXI_HOST ESXI_USERNAME ESXI_PASSWORD ESXI_SSH_KEY ESXI_SSH_PORT + SSH_CONNECT_TIMEOUT SSH_STRICT_HOST_KEY_CHECKING DRIVE_DEVICE + TEMP_LOW TEMP_MEDIUM TEMP_HIGH TEMP_CRITICAL + FAN_SPEED_IDLE FAN_SPEED_LOW FAN_SPEED_MEDIUM FAN_SPEED_HIGH FAN_SPEED_CRITICAL + HYSTERESIS FAILSAFE_ON_ERROR FAILSAFE_FAN_SPEED MANUAL_FAN_SPEED + LOG_DIR LOG_FILE LOG_LEVEL HEALTHCHECK_MAX_AGE +) + +if [[ -t 1 && "${NO_COLOR:-}" != "1" ]]; then + COLOR_BLUE=$'\033[1;34m' + COLOR_GREEN=$'\033[1;32m' + COLOR_YELLOW=$'\033[1;33m' + COLOR_RED=$'\033[1;31m' + COLOR_DIM=$'\033[2m' + COLOR_RESET=$'\033[0m' +else + COLOR_BLUE="" + COLOR_GREEN="" + COLOR_YELLOW="" + COLOR_RED="" + COLOR_DIM="" + COLOR_RESET="" +fi + +usage() { + cat <= 2 )) && [[ "${value:0:1}" == "'" && "${value: -1}" == "'" ]]; then + value="${value:1:${#value}-2}" + value="${value//\\\'/\'}" + elif (( ${#value} >= 2 )) && [[ "${value:0:1}" == '"' && "${value: -1}" == '"' ]]; then + value="${value:1:${#value}-2}" + value="${value//\\\"/\"}" + fi + + printf '%s' "$value" +} + +encode_env_value() { + local value="${1:-}" + + if [[ -z "$value" ]]; then + printf '' + elif [[ "$value" =~ ^[[:alnum:]_.,:/@+=%-]+$ ]]; then + printf '%s' "$value" + else + value="${value//\'/\\\'}" + printf "'%s'" "$value" + fi +} + +config_get() { + local key="$1" + local line + local raw + + is_supported_key "$key" || return 2 + [[ -r "$CONFIG_FILE" ]] || return 1 + + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" == "${key}="* ]]; then + raw="${line#*=}" + decode_env_value "$raw" + return 0 + fi + done < "$CONFIG_FILE" + return 1 +} + +config_set() { + local key="$1" + local value="$2" + local encoded + local tmp_file + local line + local found=false + + is_supported_key "$key" || { + printf 'Unsupported configuration key: %s\n' "$key" >&2 + return 2 + } + [[ "$value" != *$'\n'* && "$value" != *$'\r'* ]] || { + printf 'Configuration values cannot contain newlines.\n' >&2 + return 2 + } + + encoded="$(encode_env_value "$value")" + tmp_file="$(mktemp "${CONFIG_FILE}.tmp.XXXXXX")" + + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" == "${key}="* ]]; then + printf '%s=%s\n' "$key" "$encoded" >> "$tmp_file" + found=true + else + printf '%s\n' "$line" >> "$tmp_file" + fi + done < "$CONFIG_FILE" + + if [[ "$found" != "true" ]]; then + printf '\n%s=%s\n' "$key" "$encoded" >> "$tmp_file" + fi + + chmod 600 "$tmp_file" + mv -f "$tmp_file" "$CONFIG_FILE" +} + +initialize_config() { + local config_dir + + config_dir="$(dirname "$CONFIG_FILE")" + mkdir -p "$config_dir" + config_dir="$(cd "$config_dir" && pwd)" + CONFIG_FILE="${config_dir}/$(basename "$CONFIG_FILE")" + + if [[ ! -e "$CONFIG_FILE" ]]; then + cp "$TEMPLATE_FILE" "$CONFIG_FILE" + chmod 600 "$CONFIG_FILE" + printf '%sCreated %s from .env.example.%s\n' "$COLOR_GREEN" "$CONFIG_FILE" "$COLOR_RESET" + elif [[ ! -f "$CONFIG_FILE" || ! -r "$CONFIG_FILE" || ! -w "$CONFIG_FILE" ]]; then + printf '%sConfig must be a readable and writable file: %s%s\n' "$COLOR_RED" "$CONFIG_FILE" "$COLOR_RESET" >&2 + return 1 + else + chmod 600 "$CONFIG_FILE" + fi +} + +screen_clear() { + if [[ -t 1 && -n "${TERM:-}" && "$TERM" != "dumb" ]]; then + printf '\033[2J\033[H' + fi +} + +header() { + screen_clear + printf '%s╭────────────────────────────────────────────────────────────╮%s\n' "$COLOR_BLUE" "$COLOR_RESET" + printf '%s│ iDRAC Fan Control · Setup TUI │%s\n' "$COLOR_BLUE" "$COLOR_RESET" + printf '%s╰────────────────────────────────────────────────────────────╯%s\n' "$COLOR_BLUE" "$COLOR_RESET" + printf '%sConfig: %s%s\n\n' "$COLOR_DIM" "$CONFIG_FILE" "$COLOR_RESET" +} + +pause_screen() { + printf '\nPress Enter to continue... ' + IFS= read -r || true +} + +notice() { + printf '%s%s%s\n' "$COLOR_GREEN" "$*" "$COLOR_RESET" +} + +warning() { + printf '%s%s%s\n' "$COLOR_YELLOW" "$*" "$COLOR_RESET" +} + +prompt_value() { + local key="$1" + local label="$2" + local required="${3:-false}" + local current + local value + + current="$(config_get "$key" 2>/dev/null || true)" + while true; do + printf '%s [%s]: ' "$label" "${current:-empty}" + IFS= read -r value || return 1 + if [[ "$value" == "-" ]]; then + value="" + elif [[ -z "$value" ]]; then + value="$current" + fi + if [[ "$required" == "true" && -z "$value" ]]; then + warning "${label} is required." + continue + fi + config_set "$key" "$value" + return 0 + done +} + +prompt_secret() { + local key="$1" + local label="$2" + local required="${3:-false}" + local current + local value + local state="not set" + + current="$(config_get "$key" 2>/dev/null || true)" + [[ -n "$current" ]] && state="already set" + while true; do + printf '%s [%s; Enter keeps current]: ' "$label" "$state" + IFS= read -rs value || return 1 + printf '\n' + if [[ "$value" == "-" ]]; then + value="" + elif [[ -z "$value" ]]; then + value="$current" + fi + if [[ "$required" == "true" && -z "$value" ]]; then + warning "${label} is required." + continue + fi + config_set "$key" "$value" + return 0 + done +} + +prompt_integer() { + local key="$1" + local label="$2" + local minimum="$3" + local maximum="$4" + local current + local value + + current="$(config_get "$key" 2>/dev/null || true)" + while true; do + printf '%s (%s-%s) [%s]: ' "$label" "$minimum" "$maximum" "${current:-empty}" + IFS= read -r value || return 1 + [[ -z "$value" ]] && value="$current" + if [[ "$value" =~ ^[0-9]+$ ]] && (( value >= minimum && value <= maximum )); then + config_set "$key" "$value" + return 0 + fi + warning "Enter an integer from ${minimum} to ${maximum}." + done +} + +prompt_boolean() { + local key="$1" + local label="$2" + local current + local value + + current="$(config_get "$key" 2>/dev/null || true)" + while true; do + printf '%s (yes/no) [%s]: ' "$label" "${current:-empty}" + IFS= read -r value || return 1 + [[ -z "$value" ]] && value="$current" + case "$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]')" in + y|yes|true|1|on) config_set "$key" "true"; return 0 ;; + n|no|false|0|off) config_set "$key" "false"; return 0 ;; + *) warning "Answer yes or no." ;; + esac + done +} + +configure_idrac() { + header + printf '%siDRAC / IPMI%s\n\n' "$COLOR_BLUE" "$COLOR_RESET" + prompt_value IDRAC_IP "iDRAC hostname or IP" true + prompt_value IDRAC_ID "iDRAC username" true + prompt_secret IDRAC_PASSWORD "iDRAC password" true + prompt_integer IPMI_TIMEOUT "IPMI per-attempt timeout (seconds)" 1 60 + prompt_integer IPMI_RETRIES "IPMI retry count" 0 20 + notice "iDRAC settings saved." +} + +select_sources() { + local choice + local sources + + header + printf '%sTemperature source preset%s\n\n' "$COLOR_BLUE" "$COLOR_RESET" + printf ' 1) iDRAC sensors only recommended first setup\n' + printf ' 2) ESXi NVMe SMART only\n' + printf ' 3) iDRAC + ESXi NVMe SMART\n' + printf ' 4) iDRAC + NVIDIA GPU\n' + printf ' 5) ESXi NVMe SMART + NVIDIA GPU\n' + printf ' 6) iDRAC + ESXi + NVIDIA GPU\n\n' + while true; do + printf 'Choose [1-6]: ' + IFS= read -r choice || return 1 + case "$choice" in + 1) sources="idrac" ;; + 2) sources="esxi" ;; + 3) sources="idrac,esxi" ;; + 4) sources="idrac,gpu" ;; + 5) sources="esxi,gpu" ;; + 6) sources="idrac,esxi,gpu" ;; + *) warning "Choose a number from 1 to 6."; continue ;; + esac + break + done + + config_set TEMPERATURE_SOURCES "$sources" + config_set WITH_GPU_TEMP "false" + if [[ "$sources" == *gpu* ]]; then + prompt_integer GPU_TEMP_OFFSET "GPU temperature offset (C)" 0 120 + fi + notice "Temperature sources saved: ${sources}" +} + +configure_esxi() { + header + printf '%sESXi NVMe SMART source%s\n\n' "$COLOR_BLUE" "$COLOR_RESET" + prompt_value ESXI_HOST "ESXi hostname or IP" true + prompt_value ESXI_USERNAME "ESXi SSH username" true + prompt_integer ESXI_SSH_PORT "ESXi SSH port" 1 65535 + prompt_value ESXI_SSH_KEY "SSH private key path (- clears it; blank keeps current)" false + if [[ -z "$(config_get ESXI_SSH_KEY 2>/dev/null || true)" ]]; then + prompt_secret ESXI_PASSWORD "ESXi SSH password" true + else + warning "SSH key authentication selected; the password will be ignored." + fi + prompt_value DRIVE_DEVICE "Full ESXi storage device identifier" true + notice "ESXi settings saved." +} + +configure_curve() { + header + printf '%sFan curve%s\n' "$COLOR_BLUE" "$COLOR_RESET" + printf '%sThresholds and speeds must rise from idle to critical.%s\n\n' "$COLOR_DIM" "$COLOR_RESET" + prompt_integer TEMP_LOW "Low threshold (C)" 0 120 + prompt_integer TEMP_MEDIUM "Medium threshold (C)" 0 120 + prompt_integer TEMP_HIGH "High threshold (C)" 0 120 + prompt_integer TEMP_CRITICAL "Critical threshold (C)" 0 120 + prompt_integer FAN_SPEED_IDLE "Idle fan speed (%)" 1 100 + prompt_integer FAN_SPEED_LOW "Low fan speed (%)" 1 100 + prompt_integer FAN_SPEED_MEDIUM "Medium fan speed (%)" 1 100 + prompt_integer FAN_SPEED_HIGH "High fan speed (%)" 1 100 + prompt_integer FAN_SPEED_CRITICAL "Critical fan speed (%)" 1 100 + notice "Fan curve saved. Use Validate to catch ordering mistakes." +} + +configure_safety() { + local log_level + + header + printf '%sSafety, timing, and logging%s\n\n' "$COLOR_BLUE" "$COLOR_RESET" + prompt_integer HYSTERESIS "Downshift hysteresis (C)" 0 30 + prompt_integer CHECK_INTERVAL "Automatic check interval (seconds)" 1 86400 + prompt_integer COMMAND_TIMEOUT "External command timeout (seconds)" 1 300 + prompt_boolean FAILSAFE_ON_ERROR "Apply fail-safe if every source fails" + prompt_integer FAILSAFE_FAN_SPEED "Fail-safe fan speed (%)" 1 100 + prompt_boolean RESTORE_AUTO_ON_EXIT "Restore Dell automatic control on exit" + + while true; do + printf 'Log level DEBUG/INFO/WARN/ERROR [%s]: ' "$(config_get LOG_LEVEL 2>/dev/null || printf INFO)" + IFS= read -r log_level || return 1 + [[ -z "$log_level" ]] && log_level="$(config_get LOG_LEVEL 2>/dev/null || printf INFO)" + log_level="$(printf '%s' "$log_level" | tr '[:lower:]' '[:upper:]')" + case "$log_level" in + DEBUG|INFO|WARN|ERROR) config_set LOG_LEVEL "$log_level"; break ;; + *) warning "Choose DEBUG, INFO, WARN, or ERROR." ;; + esac + done + notice "Safety and logging settings saved." +} + +quick_setup() { + config_set OPERATION_MODE "auto" + configure_idrac + select_sources + if [[ "$(config_get TEMPERATURE_SOURCES)" == *esxi* ]]; then + configure_esxi + fi + header + notice "Quick setup is complete. Run Validate, then Diagnostics before starting auto mode." + pause_screen +} + +masked_state() { + local value="$1" + if [[ -n "$value" && "$value" != "change-me" && "$value" != REPLACE_TO_YOUR_* ]]; then + printf 'set (%s characters)' "${#value}" + else + printf 'not set / placeholder' + fi +} + +show_config() { + local idrac_password + local esxi_password + + header + idrac_password="$(config_get IDRAC_PASSWORD 2>/dev/null || true)" + esxi_password="$(config_get ESXI_PASSWORD 2>/dev/null || true)" + printf '%sEffective setup (secrets redacted)%s\n\n' "$COLOR_BLUE" "$COLOR_RESET" + printf ' %-28s %s\n' "Mode" "$(config_get OPERATION_MODE 2>/dev/null || true)" + printf ' %-28s %s\n' "Sources" "$(config_get TEMPERATURE_SOURCES 2>/dev/null || true)" + printf ' %-28s %s@%s\n' "iDRAC" "$(config_get IDRAC_ID 2>/dev/null || true)" "$(config_get IDRAC_IP 2>/dev/null || true)" + printf ' %-28s %s\n' "iDRAC password" "$(masked_state "$idrac_password")" + printf ' %-28s %s@%s:%s\n' "ESXi" "$(config_get ESXI_USERNAME 2>/dev/null || true)" "$(config_get ESXI_HOST 2>/dev/null || true)" "$(config_get ESXI_SSH_PORT 2>/dev/null || true)" + printf ' %-28s %s\n' "ESXi password" "$(masked_state "$esxi_password")" + printf ' %-28s %s\n' "Drive device" "$(config_get DRIVE_DEVICE 2>/dev/null || true)" + printf ' %-28s %s/%s/%s/%s C\n' "Thresholds" "$(config_get TEMP_LOW)" "$(config_get TEMP_MEDIUM)" "$(config_get TEMP_HIGH)" "$(config_get TEMP_CRITICAL)" + printf ' %-28s %s/%s/%s/%s/%s %%\n' "Fan speeds" "$(config_get FAN_SPEED_IDLE)" "$(config_get FAN_SPEED_LOW)" "$(config_get FAN_SPEED_MEDIUM)" "$(config_get FAN_SPEED_HIGH)" "$(config_get FAN_SPEED_CRITICAL)" + printf ' %-28s enabled=%s, %s%%\n' "Fail-safe" "$(config_get FAILSAFE_ON_ERROR)" "$(config_get FAILSAFE_FAN_SPEED)" + printf ' %-28s %s\n' "Log level" "$(config_get LOG_LEVEL 2>/dev/null || printf INFO)" + printf '\n%sNo fan command was sent.%s\n' "$COLOR_DIM" "$COLOR_RESET" + pause_screen +} + +load_config_environment() { + local key + local value + + for key in "${CONFIG_KEYS[@]}"; do + if value="$(config_get "$key" 2>/dev/null)"; then + export "${key}=${value}" + fi + done +} + +run_controller() { + local command="$1" + + header + printf '%sRunning %s...%s\n\n' "$COLOR_BLUE" "$command" "$COLOR_RESET" + + if command -v docker >/dev/null 2>&1 \ + && docker compose version >/dev/null 2>&1 \ + && docker info >/dev/null 2>&1; then + docker compose -f "${ROOT_DIR}/docker-compose.yml" run --rm --no-deps \ + --env-from-file "$CONFIG_FILE" \ + --volume "${CONTROLLER_SCRIPT}:/usr/local/bin/fan-control-dev.sh:ro" \ + --entrypoint /usr/local/bin/fan-control-dev.sh \ + idrac-fan-control "$command" + else + warning "Docker is unavailable; using local binaries for this check." + ( + load_config_environment + "$CONTROLLER_SCRIPT" "$command" + ) + fi +} + +run_and_pause() { + local command="$1" + + if run_controller "$command"; then + notice "${command} completed successfully." + else + warning "${command} reported a problem. Review the output above." + fi + pause_screen +} + +main_menu() { + local choice + + while true; do + header + printf ' 1) Quick setup wizard\n' + printf ' 2) iDRAC / IPMI settings\n' + printf ' 3) Temperature source preset\n' + printf ' 4) ESXi NVMe source settings\n' + printf ' 5) Fan curve\n' + printf ' 6) Safety, timing, and logging\n' + printf ' 7) Review redacted configuration\n' + printf ' 8) Validate configuration\n' + printf ' 9) Run read-only diagnostics\n' + printf ' 0) Save and exit\n\n' + printf 'Choose [0-9]: ' + IFS= read -r choice || return 0 + case "$choice" in + 1) quick_setup ;; + 2) configure_idrac; pause_screen ;; + 3) select_sources; pause_screen ;; + 4) configure_esxi; pause_screen ;; + 5) configure_curve; pause_screen ;; + 6) configure_safety; pause_screen ;; + 7) show_config ;; + 8) run_and_pause validate ;; + 9) run_and_pause diagnose ;; + 0) + header + notice "Saved ${CONFIG_FILE} with mode 600." + printf 'Next: make validate && docker compose up -d\n' + return 0 + ;; + *) warning "Choose a number from 0 to 9."; pause_screen ;; + esac + done +} + +main() { + while (( $# > 0 )); do + case "$1" in + --config) + [[ $# -ge 2 ]] || { printf '%s\n' '--config requires a path' >&2; return 2; } + CONFIG_FILE="$2" + shift 2 + ;; + -h|--help) + usage + return 0 + ;; + *) + printf 'Unknown option: %s\n' "$1" >&2 + usage >&2 + return 2 + ;; + esac + done + + initialize_config + main_menu +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/tests/fan-control.test.sh b/tests/fan-control.test.sh index 281205e..b581074 100755 --- a/tests/fan-control.test.sh +++ b/tests/fan-control.test.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2034 # Test globals are consumed by sourced controller functions. set -Eeuo pipefail @@ -6,10 +7,10 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" export LOG_DIR="" export DRY_RUN=true -export IDRAC_IP=192.0.2.10 +export IDRAC_IP=10.0.0.10 export IDRAC_ID=root export IDRAC_PASSWORD=test-password -export ESXI_HOST=192.0.2.20 +export ESXI_HOST=10.0.0.20 export ESXI_USERNAME=root export ESXI_PASSWORD=test-password export DRIVE_DEVICE=test-drive @@ -71,6 +72,27 @@ test_remote_quote_handles_single_quotes() { assert_eq "'abc'\\''def'" "$(remote_quote "abc'def")" "quotes ESXi device IDs for remote shell" } +test_esxi_password_is_not_in_process_arguments() { + local output + + ESXI_HOST=10.0.0.20 + ESXI_USERNAME=root + ESXI_PASSWORD='secret with spaces' + ESXI_SSH_KEY="" + ESXI_SSH_PORT=22 + timeout() { + [[ "${SSHPASS:-}" == 'secret with spaces' ]] || return 2 + printf '%s' "$*" + } + + output="$(run_esxi_command true)" + assert_contains "sshpass -e" "$output" "passes the ESXi password through SSHPASS" + if [[ "$output" == *'secret with spaces'* ]]; then + fail "ESXi password must not be present in the sshpass argument list" + fi + pass +} + test_temperature_levels() { TEMP_LOW=65 TEMP_MEDIUM=70 @@ -152,7 +174,7 @@ test_validate_accepts_idrac_only_auto_mode() { TEMPERATURE_SOURCES=idrac WITH_GPU_TEMP=false DRY_RUN=true - IDRAC_IP=192.0.2.10 + IDRAC_IP=10.0.0.10 IDRAC_ID=root IDRAC_PASSWORD=test-password ESXI_HOST="" @@ -177,16 +199,23 @@ test_validate_rejects_placeholder_values() { pass } +test_documentation_networks_are_placeholders() { + is_placeholder_value 192.0.2.10 || fail "RFC 5737 TEST-NET-1 should be treated as a placeholder" + is_placeholder_value 198.51.100.20 || fail "RFC 5737 TEST-NET-2 should be treated as a placeholder" + is_placeholder_value 203.0.113.30 || fail "RFC 5737 TEST-NET-3 should be treated as a placeholder" + pass +} + test_validate_rejects_esxi_password_placeholder() { ( OPERATION_MODE=auto TEMPERATURE_SOURCES=esxi WITH_GPU_TEMP=false DRY_RUN=true - IDRAC_IP=192.0.2.10 + IDRAC_IP=10.0.0.10 IDRAC_ID=root IDRAC_PASSWORD=test-password - ESXI_HOST=192.0.2.20 + ESXI_HOST=10.0.0.20 ESXI_USERNAME=root ESXI_PASSWORD=change-me ESXI_SSH_KEY="" @@ -212,9 +241,138 @@ test_fail_safe_can_be_disabled() { assert_eq "" "$LAST_SET_SPEED" "does not set a fan speed when fail-safe is disabled" } +test_validate_rejects_unsafe_curve_and_transport_values() { + ( + OPERATION_MODE=auto + TEMPERATURE_SOURCES=idrac + WITH_GPU_TEMP=false + DRY_RUN=true + IDRAC_IP=10.0.0.10 + IDRAC_ID=root + IDRAC_PASSWORD=test-password + ESXI_SSH_PORT=0 + FAN_SPEED_IDLE=40 + FAN_SPEED_LOW=30 + FAN_SPEED_MEDIUM=40 + FAN_SPEED_HIGH=50 + FAN_SPEED_CRITICAL=60 + FAILSAFE_FAN_SPEED=60 + LOG_LEVEL=INFO + validate_config auto >/dev/null 2>&1 + ) && fail "validate_config should reject a falling fan curve and invalid SSH port" + pass +} + +test_validate_rejects_invalid_log_level_and_regex() { + ( + OPERATION_MODE=auto + TEMPERATURE_SOURCES=idrac + WITH_GPU_TEMP=false + DRY_RUN=true + IDRAC_IP=10.0.0.10 + IDRAC_ID=root + IDRAC_PASSWORD=test-password + LOG_LEVEL=verbose + IDRAC_SENSOR_INCLUDE_REGEX='[' + validate_config auto >/dev/null 2>&1 + ) && fail "validate_config should reject invalid debug and sensor filters" + pass +} + +test_restore_only_requires_ipmi_configuration() { + ( + OPERATION_MODE=auto + DRY_RUN=true + IDRAC_IP=10.0.0.10 + IDRAC_ID=root + IDRAC_PASSWORD=test-password + FAN_SPEED_LOW=not-a-speed + TEMP_CRITICAL=not-a-threshold + validate_config restore >/dev/null 2>&1 + ) || fail "restore should not be blocked by unrelated fan curve values" + pass +} + +test_calculate_decision_temperature_accepts_precollected_readings() { + local decision + + GPU_TEMP_OFFSET=10 + decision="$(calculate_decision_temperature $'idrac\tInlet\t42\ngpu\tgpu0\t90')" + assert_contains $'80\t' "$decision" "calculates a decision from diagnostic readings" + assert_contains "gpu:gpu0=90C(adjusted=80C)" "$decision" "keeps source details in diagnostic preview" +} + +test_config_output_redacts_credentials() { + local output + + IDRAC_IP=10.0.0.10 + IDRAC_PASSWORD=top-secret-password + ESXI_PASSWORD=another-secret + output="$(print_effective_config)" + assert_contains "set (19 characters)" "$output" "shows credential state without values" + if [[ "$output" == *top-secret-password* || "$output" == *another-secret* ]]; then + fail "print_effective_config must never print credentials" + fi + pass +} + +test_healthcheck_checks_control_loop_freshness() { + local temp_dir + + temp_dir="$(mktemp -d)" + ( + OPERATION_MODE=auto + LOG_DIR="$temp_dir" + LOG_FILE=fan_control.log + CHECK_INTERVAL=60 + COMMAND_TIMEOUT=20 + HEALTHCHECK_MAX_AGE=10 + printf 'recent control cycle\n' > "${temp_dir}/fan_control.log" + healthcheck_mode + ) || fail "healthcheck should accept a recent control cycle" + + ( + OPERATION_MODE=auto + LOG_DIR="$temp_dir" + LOG_FILE=fan_control.log + CHECK_INTERVAL=1 + COMMAND_TIMEOUT=1 + HEALTHCHECK_MAX_AGE=1 + touch -d @1 "${temp_dir}/fan_control.log" + healthcheck_mode >/dev/null 2>&1 + ) && fail "healthcheck should reject a stale control cycle" + rm -rf "$temp_dir" + pass +} + +test_diagnose_reports_read_only_source_and_preview() { + local output + + output="$( + OPERATION_MODE=auto + TEMPERATURE_SOURCES=idrac + DRY_RUN=false + IDRAC_IP=10.0.0.10 + IDRAC_ID=root + IDRAC_PASSWORD=test-password + LOG_DIR="" + run_ipmitool() { + [[ "$*" == "mc info" ]] && printf 'Firmware Revision : 2.60\n' + } + get_idrac_temperatures() { + printf 'idrac\tInlet Temp\t42\n' + } + diagnose_mode + )" + assert_contains "[PASS] iDRAC/IPMI" "$output" "diagnose checks iDRAC connectivity" + assert_contains "[PASS] source:idrac" "$output" "diagnose checks each selected source" + assert_contains "decision preview" "$output" "diagnose previews the fan decision" +} + test_normalize_sources test_hex_formatting test_remote_quote_handles_single_quotes +test_esxi_password_is_not_in_process_arguments test_temperature_levels test_hysteresis_only_delays_downshift test_idrac_sensor_parsing @@ -222,7 +380,15 @@ test_decision_temperature_uses_max_adjusted_source test_fail_safe_when_all_sources_fail test_validate_accepts_idrac_only_auto_mode test_validate_rejects_placeholder_values +test_documentation_networks_are_placeholders test_validate_rejects_esxi_password_placeholder test_fail_safe_can_be_disabled +test_validate_rejects_unsafe_curve_and_transport_values +test_validate_rejects_invalid_log_level_and_regex +test_restore_only_requires_ipmi_configuration +test_calculate_decision_temperature_accepts_precollected_readings +test_config_output_redacts_credentials +test_healthcheck_checks_control_loop_freshness +test_diagnose_reports_read_only_source_and_preview printf 'ok - %s assertions passed\n' "$PASS_COUNT" diff --git a/tests/tui.test.sh b/tests/tui.test.sh new file mode 100644 index 0000000..af816b9 --- /dev/null +++ b/tests/tui.test.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +export NO_COLOR=1 +export FAN_CONTROL_CONFIG="${TMP_DIR}/.env" + +# shellcheck source=../src/fan-control-tui.sh +source "${ROOT_DIR}/src/fan-control-tui.sh" + +PASS_COUNT=0 + +pass() { + PASS_COUNT=$((PASS_COUNT + 1)) +} + +fail() { + printf 'not ok - %s\n' "$*" >&2 + exit 1 +} + +assert_eq() { + local expected="$1" + local actual="$2" + local message="$3" + + [[ "$expected" == "$actual" ]] || fail "${message}: expected '${expected}', got '${actual}'" + pass +} + +assert_contains() { + local needle="$1" + local haystack="$2" + local message="$3" + + [[ "$haystack" == *"$needle"* ]] || fail "${message}: missing '${needle}'" + pass +} + +initialize_config + +test_config_round_trip_and_preserves_comments() { + local value + local raw + + value="p@ss word's \$literal" + config_set IDRAC_PASSWORD "$value" + assert_eq "$value" "$(config_get IDRAC_PASSWORD)" "round-trips a quoted password without eval" + raw="$(sed -n 's/^IDRAC_PASSWORD=//p' "$CONFIG_FILE")" + assert_contains "'p@ss word\\'s \$literal'" "$raw" "quotes values that would be unsafe in env files" + assert_contains '# Copy this file to .env' "$(sed -n '1p' "$CONFIG_FILE")" "preserves template comments" +} + +test_config_set_does_not_duplicate_keys() { + local count + + config_set IDRAC_IP 192.0.2.55 + config_set IDRAC_IP 192.0.2.56 + count="$(grep -c '^IDRAC_IP=' "$CONFIG_FILE")" + assert_eq "1" "$count" "updates an existing key in place" + assert_eq "192.0.2.56" "$(config_get IDRAC_IP)" "returns the newest value" +} + +test_config_rejects_unknown_key_and_newline() { + if config_set UNKNOWN_KEY value >/dev/null 2>&1; then + fail "rejects an unknown key" + fi + if config_set IDRAC_ID $'root\nunsafe' >/dev/null 2>&1; then + fail "rejects a multiline value" + fi + pass +} + +test_sensitive_values_are_only_summarized() { + assert_eq "set (6 characters)" "$(masked_state secret)" "masks configured secrets" + assert_eq "not set / placeholder" "$(masked_state change-me)" "flags template placeholders" +} + +test_load_config_environment_is_allowlisted() { + local before + + config_set IDRAC_PASSWORD secret-value + before="${PATH}" + load_config_environment + assert_eq "secret-value" "$IDRAC_PASSWORD" "loads known env keys" + assert_eq "$before" "$PATH" "does not overwrite unrelated process environment" +} + +test_config_round_trip_and_preserves_comments +test_config_set_does_not_duplicate_keys +test_config_rejects_unknown_key_and_newline +test_sensitive_values_are_only_summarized +test_load_config_environment_is_allowlisted + +printf 'ok - %s assertions passed\n' "$PASS_COUNT"