diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/playbook.yaml b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/playbook.yaml index e99d5c9..b5c4f88 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/playbook.yaml +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/playbook.yaml @@ -7,6 +7,11 @@ # 보고 mc-o11y-manager가 "linux" / "windows" 둘 중 하나로 채움. # default "linux" 로 두는 이유: 옛 호출자(직접 Semaphore template 호출 등)에 대한 호환 + Telegraf/FluentBit/Beyla 기존 케이스. os_type: "{{ os_type | default('linux') }}" + # enable_gpu: NVIDIA GPU 노드에서 DCGM Exporter를 함께 설치/제거할지 여부. + # telegraf 에이전트 설치 시에만 의미가 있으며, telegraf의 prometheus input이 + # localhost:9400(DCGM Exporter)을 스크랩해 GPU 메트릭을 수집한다. + enable_gpu: "{{ enable_gpu | default(false) }}" + NVIDIA_CONTAINER_TOOLKIT_VERSION: "1.19.0-1" tasks: - name: Check variables include_role: @@ -36,6 +41,13 @@ no_log: true when: os_type == 'windows' + # DCGM Exporter (GPU 메트릭): Linux + telegraf + enable_gpu 인 경우에만 수행. + # (o11y-infra dcgm role 포팅 — podman으로 dcgm-exporter 컨테이너 구동, 포트 9400) + - name: DCGM Exporter Operation + include_role: + name: dcgm + when: os_type == 'linux' and agent == 'telegraf' and (enable_gpu | bool) + - name: Agent Operation (Linux) include_role: name: agent diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf index 3d46633..eefbddb 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf @@ -80,6 +80,6 @@ name loki host ${LOKI_HOST} match winevtlog.* - labels host=${HOSTNAME}, NS_ID=${NS_ID}, MCI_ID=${MCI_ID}, VM_ID=${VM_ID} + labels host=${HOSTNAME}, NS_ID=${NS_ID}, INFRA_ID=${INFRA_ID}, NODE_ID=${NODE_ID} label_keys $service, $level, $source line_format key_value diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/variables b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/variables index ff2125a..bb72749 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/variables +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/variables @@ -1,4 +1,4 @@ NS_ID= -MCI_ID= -VM_ID= +INFRA_ID= +NODE_ID= LOKI_HOST=127.0.0.1 diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/telegraf/etc/telegraf.conf b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/telegraf/etc/telegraf.conf index 3ee9c4e..fe0dc2d 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/telegraf/etc/telegraf.conf +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/telegraf/etc/telegraf.conf @@ -1,7 +1,7 @@ [global_tags] ns_id="@NS_ID" - mci_id="@MCI_ID" - vm_id="@VM_ID" + infra_id="@INFRA_ID" + node_id="@NODE_ID" [agent] interval = "30s" diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf index 2392faf..b0736e9 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf @@ -54,5 +54,5 @@ name loki host ${LOKI_HOST} match syslog - labels host=${HOSTNAME}, NS_ID=${NS_ID}, MCI_ID=${MCI_ID}, VM_ID=${VM_ID} + labels host=${HOSTNAME}, NS_ID=${NS_ID}, INFRA_ID=${INFRA_ID}, NODE_ID=${NODE_ID} label_keys $service, $level, $source diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/variables b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/variables index ff2125a..bb72749 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/variables +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/variables @@ -1,4 +1,4 @@ NS_ID= -MCI_ID= -VM_ID= +INFRA_ID= +NODE_ID= LOKI_HOST=127.0.0.1 diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/telegraf/etc/telegraf.conf b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/telegraf/etc/telegraf.conf index 0b13663..7e4426c 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/telegraf/etc/telegraf.conf +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/telegraf/etc/telegraf.conf @@ -1,7 +1,7 @@ [global_tags] ns_id="@NS_ID" - mci_id="@MCI_ID" - vm_id="@VM_ID" + infra_id="@INFRA_ID" + node_id="@NODE_ID" [agent] interval = "30s" diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/fluent-bit.conf b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/fluent-bit.conf index 2392faf..b0736e9 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/fluent-bit.conf +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/fluent-bit.conf @@ -54,5 +54,5 @@ name loki host ${LOKI_HOST} match syslog - labels host=${HOSTNAME}, NS_ID=${NS_ID}, MCI_ID=${MCI_ID}, VM_ID=${VM_ID} + labels host=${HOSTNAME}, NS_ID=${NS_ID}, INFRA_ID=${INFRA_ID}, NODE_ID=${NODE_ID} label_keys $service, $level, $source diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/variables b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/variables index ff2125a..bb72749 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/variables +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/variables @@ -1,4 +1,4 @@ NS_ID= -MCI_ID= -VM_ID= +INFRA_ID= +NODE_ID= LOKI_HOST=127.0.0.1 diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/telegraf/etc/telegraf.conf b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/telegraf/etc/telegraf.conf index 0b13663..7e4426c 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/telegraf/etc/telegraf.conf +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/telegraf/etc/telegraf.conf @@ -1,7 +1,7 @@ [global_tags] ns_id="@NS_ID" - mci_id="@MCI_ID" - vm_id="@VM_ID" + infra_id="@INFRA_ID" + node_id="@NODE_ID" [agent] interval = "30s" diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/cmp-dcgm-exporter.service b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/cmp-dcgm-exporter.service new file mode 100644 index 0000000..0a632fa --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/cmp-dcgm-exporter.service @@ -0,0 +1,41 @@ +[Unit] +Description=CMP DCGM Exporter Service +Wants=network-online.target +After=network-online.target +RequiresMountsFor=%t/containers + +[Service] +Type=notify +NotifyAccess=all +User=root + + +Environment=TIME_DIFF_SECONDS=0 +Environment=PODMAN_SYSTEMD_UNIT=%n + +ExecStart=/cmp-agent/common/podman/bin/run-podman run \ + --pid=host \ + --cgroups=no-conmon \ + --env TIME_DIFF_SECONDS \ + --env HOSTNAME=%H \ + --sdnotify=conmon \ + --replace \ + -d \ + --restart=always \ + --tz=Asia/Seoul \ + --network=host \ + --device=nvidia.com/gpu=all \ + --name cmp-dcgm \ + -p 9400:9400 \ + --mount type=bind,src=/cmp-agent/dcgm-exporter-custom.csv,dst=/cmp-agent/dcgm-exporter-custom.csv,ro=true \ + nvcr.io/nvidia/k8s/dcgm-exporter:4.4.2-4.7.0-ubuntu22.04 \ + -f /cmp-agent/dcgm-exporter-custom.csv + +ExecStop=/cmp-agent/common/podman/bin/run-podman stop cmp-dcgm + +Restart=on-failure +RestartPreventExitStatus=100 +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter-custom.csv b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter-custom.csv new file mode 100644 index 0000000..0a0bb67 --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter-custom.csv @@ -0,0 +1,92 @@ +# Format +# If line starts with a '#' it is considered a comment +# DCGM FIELD, Prometheus metric type, help message + +# Clocks +DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). +DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). +# DCGM_EXP_CLOCK_EVENTS_COUNT, gauge, Count of clock events within the user-specified time window (see clock-events-count-window-size param). + +# Temperature +DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). +DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). +DCGM_FI_DEV_FAN_SPEED, gauge, Fan speed for the device in percent(in %). + +# Power +DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). +DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). +DCGM_FI_DEV_PSTATE, gauge, Performance state. + +# PCIE +DCGM_FI_DEV_PCIE_TX_THROUGHPUT, counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML. +DCGM_FI_DEV_PCIE_RX_THROUGHPUT, counter, Total number of bytes received through PCIe RX (in KB) via NVML. +DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. + +# Utilization (the sample period varies depending on the product) +DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). +DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). +DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). +DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %). + +# Errors and violations +DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. +DCGM_FI_DEV_CLOCKS_EVENT_REASONS, gauge, Clock throttle reasons +DCGM_EXP_XID_ERRORS_COUNT, gauge, Count of XID Errors within user-specified time window (see xid-count-window-size param). + +# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). +# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). +# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). +# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). +# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). +# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). + +# Memory usage +DCGM_FI_DEV_FB_FREE, gauge, Frame buffer memory free (in MB). +DCGM_FI_DEV_FB_USED, gauge, Frame buffer memory used (in MB). +DCGM_FI_DEV_FB_TOTAL, gauge, Frame buffer memory (in MB). + +# ECC (Datacenter GPU only, A100, H100, V100, T4) +DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. +DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. +DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. +DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. + +# Retired pages +# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. +# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. +# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. + +# NVLink +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors. +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors. +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries. +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors. +DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes + +# MIG (Multi-Instance GPU) +DCGM_FI_DEV_MIG_MODE, gauge, MIG mode status (0=disabled / 1=enabled) +DCGM_FI_DEV_MIG_MAX_SLICES, gauge, Maximum number of MIG slices supported + +# VGPU License status +DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status + +# Remapped rows +DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors +DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors +DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed + +# Static configuration information. These appear as labels on the other metrics +DCGM_FI_DRIVER_VERSION, label, Driver Version +DCGM_FI_DEV_PCI_BUSID, label, PCI attributes for the device +DCGM_FI_CUDA_DRIVER_VERSION, label, CUDA Driver Version +DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device +DCGM_FI_NVML_VERSION, label, NVML Version +# DCGM_FI_DEV_BRAND, label, Device Brand +# DCGM_FI_DEV_SERIAL, label, Device Serial Number +# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version +# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version +# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version +# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version + +# Process identifier +DCGM_FI_PROCESS_NAME, label, Process name diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_00 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_00 new file mode 100644 index 0000000..8efd1af Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_00 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_01 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_01 new file mode 100644 index 0000000..506193c Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_01 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_02 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_02 new file mode 100644 index 0000000..ba64b9a Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_02 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_03 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_03 new file mode 100644 index 0000000..7f8e05b Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_03 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_04 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_04 new file mode 100644 index 0000000..e7dd57e Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_04 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_05 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_05 new file mode 100644 index 0000000..7f9fccd Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_05 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/libnvidia-container-tools_1.19.0-1_amd64.deb b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/libnvidia-container-tools_1.19.0-1_amd64.deb new file mode 100644 index 0000000..d9f0f10 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/libnvidia-container-tools_1.19.0-1_amd64.deb differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/libnvidia-container1_1.19.0-1_amd64.deb b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/libnvidia-container1_1.19.0-1_amd64.deb new file mode 100644 index 0000000..8b0983c Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/libnvidia-container1_1.19.0-1_amd64.deb differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/nvidia-container-toolkit-base_1.19.0-1_amd64.deb b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/nvidia-container-toolkit-base_1.19.0-1_amd64.deb new file mode 100644 index 0000000..498e483 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/nvidia-container-toolkit-base_1.19.0-1_amd64.deb differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/nvidia-container-toolkit_1.19.0-1_amd64.deb b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/nvidia-container-toolkit_1.19.0-1_amd64.deb new file mode 100644 index 0000000..bc0b4ce Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/deb/nvidia-container-toolkit_1.19.0-1_amd64.deb differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/libnvidia-container-tools-1.19.0-1.x86_64.rpm b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/libnvidia-container-tools-1.19.0-1.x86_64.rpm new file mode 100644 index 0000000..5c9bd30 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/libnvidia-container-tools-1.19.0-1.x86_64.rpm differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/libnvidia-container1-1.19.0-1.x86_64.rpm b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/libnvidia-container1-1.19.0-1.x86_64.rpm new file mode 100644 index 0000000..e4ab169 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/libnvidia-container1-1.19.0-1.x86_64.rpm differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/nvidia-container-toolkit-1.19.0-1.x86_64.rpm b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/nvidia-container-toolkit-1.19.0-1.x86_64.rpm new file mode 100644 index 0000000..16aba76 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/nvidia-container-toolkit-1.19.0-1.x86_64.rpm differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/nvidia-container-toolkit-base-1.19.0-1.x86_64.rpm b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/nvidia-container-toolkit-base-1.19.0-1.x86_64.rpm new file mode 100644 index 0000000..a9c1914 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/rpm/nvidia-container-toolkit-base-1.19.0-1.x86_64.rpm differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/cmp-dcgm-exporter.service b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/cmp-dcgm-exporter.service new file mode 100644 index 0000000..0a632fa --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/cmp-dcgm-exporter.service @@ -0,0 +1,41 @@ +[Unit] +Description=CMP DCGM Exporter Service +Wants=network-online.target +After=network-online.target +RequiresMountsFor=%t/containers + +[Service] +Type=notify +NotifyAccess=all +User=root + + +Environment=TIME_DIFF_SECONDS=0 +Environment=PODMAN_SYSTEMD_UNIT=%n + +ExecStart=/cmp-agent/common/podman/bin/run-podman run \ + --pid=host \ + --cgroups=no-conmon \ + --env TIME_DIFF_SECONDS \ + --env HOSTNAME=%H \ + --sdnotify=conmon \ + --replace \ + -d \ + --restart=always \ + --tz=Asia/Seoul \ + --network=host \ + --device=nvidia.com/gpu=all \ + --name cmp-dcgm \ + -p 9400:9400 \ + --mount type=bind,src=/cmp-agent/dcgm-exporter-custom.csv,dst=/cmp-agent/dcgm-exporter-custom.csv,ro=true \ + nvcr.io/nvidia/k8s/dcgm-exporter:4.4.2-4.7.0-ubuntu22.04 \ + -f /cmp-agent/dcgm-exporter-custom.csv + +ExecStop=/cmp-agent/common/podman/bin/run-podman stop cmp-dcgm + +Restart=on-failure +RestartPreventExitStatus=100 +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter-custom.csv b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter-custom.csv new file mode 100644 index 0000000..1529b6a --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter-custom.csv @@ -0,0 +1,92 @@ +# Format +# If line starts with a '#' it is considered a comment +# DCGM FIELD, Prometheus metric type, help message + +# Clocks +DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). +DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). +# DCGM_EXP_CLOCK_EVENTS_COUNT, gauge, Count of clock events within the user-specified time window (see clock-events-count-window-size param). + +# Temperature +DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). +DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). +DCGM_FI_DEV_FAN_SPEED, gauge, Fan speed for the device in percent(in %). + +# Power +DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). +DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). +DCGM_FI_DEV_PSTATE, gauge, Performance state. + +# PCIE +DCGM_FI_DEV_PCIE_TX_THROUGHPUT, counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML. +DCGM_FI_DEV_PCIE_RX_THROUGHPUT, counter, Total number of bytes received through PCIe RX (in KB) via NVML. +DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. + +# Utilization (the sample period varies depending on the product) +DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). +DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). +DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). +DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %). + +# Errors and violations +DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. +DCGM_FI_DEV_CLOCKS_EVENT_REASONS, gauge, Clock throttle reasons +DCGM_EXP_XID_ERRORS_COUNT, gauge, Count of XID Errors within user-specified time window (see xid-count-window-size param). + +# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). +# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). +# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). +# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). +# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). +# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). + +# Memory usage +DCGM_FI_DEV_FB_FREE, gauge, Frame buffer memory free (in MB). +DCGM_FI_DEV_FB_USED, gauge, Frame buffer memory used (in MB). +DCGM_FI_DEV_FB_TOTAL, gauge, Frame buffer memory (in MB). + +# ECC (Datacenter GPU only, A100, H100, V100, T4) +DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. +DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. +DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. +DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. + +# Retired pages +# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. +# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. +# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. + +# NVLink +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors. +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors. +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries. +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors. +DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes + +# MIG (Multi-Instance GPU) +DCGM_FI_DEV_MIG_MODE, gauge, MIG mode status (0=disabled / 1=enabled) +DCGM_FI_DEV_MIG_MAX_SLICES, gauge, Maximum number of MIG slices supported + +# VGPU License status +DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status + +# Remapped rows +DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors +DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors +DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed + +# Static configuration information. These appear as labels on the other metrics +DCGM_FI_DRIVER_VERSION, label, Driver Version +DCGM_FI_DEV_PCI_BUSID, label, PCI attributes for the device +DCGM_FI_CUDA_DRIVER_VERSION, label, CUDA Driver Version +DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device +DCGM_FI_NVML_VERSION, label, NVML Version +# DCGM_FI_DEV_BRAND, label, Device Brand +# DCGM_FI_DEV_SERIAL, label, Device Serial Number +# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version +# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version +# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version +# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version + +# Process identifier +DCGM_FI_PROCESS_NAME, label, Process name diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_00 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_00 new file mode 100644 index 0000000..8efd1af Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_00 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_01 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_01 new file mode 100644 index 0000000..506193c Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_01 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_02 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_02 new file mode 100644 index 0000000..ba64b9a Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_02 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_03 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_03 new file mode 100644 index 0000000..7f8e05b Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_03 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_04 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_04 new file mode 100644 index 0000000..e7dd57e Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_04 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_05 b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_05 new file mode 100644 index 0000000..7f9fccd Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_05 differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/libnvidia-container-tools_1.19.0-1_arm64.deb b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/libnvidia-container-tools_1.19.0-1_arm64.deb new file mode 100644 index 0000000..9beaf54 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/libnvidia-container-tools_1.19.0-1_arm64.deb differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/libnvidia-container1_1.19.0-1_arm64.deb b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/libnvidia-container1_1.19.0-1_arm64.deb new file mode 100644 index 0000000..2bc1a35 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/libnvidia-container1_1.19.0-1_arm64.deb differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/nvidia-container-toolkit-base_1.19.0-1_arm64.deb b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/nvidia-container-toolkit-base_1.19.0-1_arm64.deb new file mode 100644 index 0000000..3e31602 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/nvidia-container-toolkit-base_1.19.0-1_arm64.deb differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/nvidia-container-toolkit_1.19.0-1_arm64.deb b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/nvidia-container-toolkit_1.19.0-1_arm64.deb new file mode 100644 index 0000000..69b41f3 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/deb/nvidia-container-toolkit_1.19.0-1_arm64.deb differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/libnvidia-container-tools-1.19.0-1.aarch64.rpm b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/libnvidia-container-tools-1.19.0-1.aarch64.rpm new file mode 100644 index 0000000..fa24b58 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/libnvidia-container-tools-1.19.0-1.aarch64.rpm differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/libnvidia-container1-1.19.0-1.aarch64.rpm b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/libnvidia-container1-1.19.0-1.aarch64.rpm new file mode 100644 index 0000000..60fa9c0 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/libnvidia-container1-1.19.0-1.aarch64.rpm differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/nvidia-container-toolkit-1.19.0-1.aarch64.rpm b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/nvidia-container-toolkit-1.19.0-1.aarch64.rpm new file mode 100644 index 0000000..26abd58 Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/nvidia-container-toolkit-1.19.0-1.aarch64.rpm differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/nvidia-container-toolkit-base-1.19.0-1.aarch64.rpm b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/nvidia-container-toolkit-base-1.19.0-1.aarch64.rpm new file mode 100644 index 0000000..6f6a68a Binary files /dev/null and b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/rpm/nvidia-container-toolkit-base-1.19.0-1.aarch64.rpm differ diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_common_begin.yaml b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_common_begin.yaml new file mode 100644 index 0000000..3bf06de --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_common_begin.yaml @@ -0,0 +1,40 @@ +# 공통 부트스트랩: busybox + cmp-agent-common(podman 정적 바이너리 포함) 설치. +# agent role의 agent_common_begin.yaml과 동일한 공통 설치 흐름을 dcgm role 단독 실행 시에도 보장한다. +- name: Copy temporary busybox + include_tasks: send_file.yaml + vars: + input_file_path: "{{ playbook_dir }}/roles/agent/agent-install-files/{{ arch }}/busybox" + output_file_path: "/tmp/busybox_{{ request_id }}" + output_file_owner: "root" + output_file_group: "root" + output_file_chmod: "755" + +- name: Copy Common Agent Install File + include_tasks: send_file.yaml + vars: + input_file_path: "{{ playbook_dir }}/roles/agent/agent-install-files/{{ arch }}/cmp-agent-common.tar.gz" + output_file_path: "/tmp/cmp-agent-common_{{ request_id }}.tar.gz" + output_file_owner: "root" + output_file_group: "root" + output_file_chmod: "644" + +- name: Copy Common Agent Install Script File + include_tasks: send_file.yaml + vars: + input_file_path: "{{ playbook_dir }}/roles/agent/agent-install-files/{{ arch }}/cmp-agent_install_common" + output_file_path: "/tmp/cmp-agent_install_common_{{ request_id }}" + output_file_owner: "root" + output_file_group: "root" + output_file_chmod: "755" + +- name: Run Common Agent Install Script File + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "flock -x /tmp/cmp-agent_install_common.lock /tmp/cmp-agent_install_common_{{ request_id }} {{ request_id }}" diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_common_end.yaml b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_common_end.yaml new file mode 100644 index 0000000..223c719 --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_common_end.yaml @@ -0,0 +1,35 @@ +- name: Remove temporary busybox + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "rm -f /tmp/busybox_{{ request_id }}" + +- name: Remove Remote Common Agent Install File + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "rm -f /tmp/cmp-agent-common_{{ request_id }}.tar.gz" + +- name: Remove Remote Common Agent Script File + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "rm -f /tmp/cmp-agent_install_common_{{ request_id }}" diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_install.yaml b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_install.yaml new file mode 100644 index 0000000..a42e522 --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_install.yaml @@ -0,0 +1,308 @@ +# 1. Common bootstrap (podman static 설치) +- name: Begin Agent Common Tasks + include_tasks: "{{ role_path }}/tasks/dcgm_common_begin.yaml" + +- name: Detect OS family + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: grep '^ID=' /etc/os-release | head -n1 | cut -d'=' -f2 | tr -d '"' + register: os_id_raw + changed_when: false + failed_when: false + +- name: Debug OS ID + debug: + msg: "OS ID is {{ os_id_raw.stdout | trim }}" + +- name: Set OS family for RedHat-based + set_fact: + os_family: "redhat" + when: os_id_raw.stdout | trim in ['rhel', 'centos', 'rocky', 'almalinux', 'fedora'] + +- name: Set OS family for Debian-based + set_fact: + os_family: "debian" + when: os_id_raw.stdout | trim in ['ubuntu', 'debian'] + +- name: Set OS family as unknown + set_fact: + os_family: "unknown" + when: os_family is not defined + +- name: Display OS family + debug: + msg: "Detected os family is {{ os_family }}" + +# 2. Install NVIDIA Container Toolkit +- name: Create NVIDIA Container Toolkit Package Temporary Folder + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "mkdir -p /tmp/dcgm-install-files_{{ request_id }}/" + +- name: Copy NVIDIA Container Toolkit DEB Packages + include_tasks: send_file.yaml + vars: + input_file_path: "{{ role_path }}/dcgm-install-files/{{ arch }}/deb/{{ item }}" + output_file_path: "/tmp/dcgm-install-files_{{ request_id }}/{{ item }}" + output_file_owner: "root" + output_file_group: "root" + output_file_chmod: "644" + loop: + - "libnvidia-container1_{{ NVIDIA_CONTAINER_TOOLKIT_VERSION }}_{{ arch }}.deb" + - "libnvidia-container-tools_{{ NVIDIA_CONTAINER_TOOLKIT_VERSION }}_{{ arch }}.deb" + - "nvidia-container-toolkit-base_{{ NVIDIA_CONTAINER_TOOLKIT_VERSION }}_{{ arch }}.deb" + - "nvidia-container-toolkit_{{ NVIDIA_CONTAINER_TOOLKIT_VERSION }}_{{ arch }}.deb" + when: os_family == "debian" + +- name: Copy NVIDIA Container Toolkit RPM Packages + include_tasks: send_file.yaml + vars: + input_file_path: "{{ role_path }}/dcgm-install-files/{{ arch }}/rpm/{{ item }}" + output_file_path: "/tmp/dcgm-install-files_{{ request_id }}/{{ item }}" + output_file_owner: "root" + output_file_group: "root" + output_file_chmod: "644" + loop: + - "libnvidia-container1-{{ NVIDIA_CONTAINER_TOOLKIT_VERSION }}.{{ arch_raw.stdout | trim }}.rpm" + - "libnvidia-container-tools-{{ NVIDIA_CONTAINER_TOOLKIT_VERSION }}.{{ arch_raw.stdout | trim }}.rpm" + - "nvidia-container-toolkit-base-{{ NVIDIA_CONTAINER_TOOLKIT_VERSION }}.{{ arch_raw.stdout | trim }}.rpm" + - "nvidia-container-toolkit-{{ NVIDIA_CONTAINER_TOOLKIT_VERSION }}.{{ arch_raw.stdout | trim }}.rpm" + when: os_family == "redhat" + +- name: Install NVIDIA Container Toolkit Package (Debian/Ubuntu) + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "dpkg -i /tmp/dcgm-install-files_{{ request_id }}/*.deb" + when: os_family == "debian" + +- name: Install NVIDIA Container Toolkit Package (RHEL/CentOS) + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "rpm -ivh --force /tmp/dcgm-install-files_{{ request_id }}/*.rpm" + when: os_family == "redhat" + +- name: Fix nvidia-cdi-refresh.service for compressed kernel modules (.ko.zst) + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: | + sed -i 's/nvidia|nvidia-current)\\\.ko\[:\]/nvidia|nvidia-current)\\.ko[^:]*:/g' /etc/systemd/system/nvidia-cdi-refresh.service && systemctl daemon-reload && systemctl restart nvidia-cdi-refresh + ignore_errors: true + +- name: Remove Temporary NVIDIA Container Toolkit Package Files + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "rm -rf /tmp/dcgm-install-files_{{ request_id }}" + +# 3. DCGM Exporter 이미지 로드 +# 이미지(273MB)는 GitHub 100MB 제한 때문에 50MB 단위(_00.._05)로 분할 저장되어 있다. +# (telegraf의 cmp-telegraf.tar.gz_* 분할 방식과 동일) +- name: Merge split DCGM Exporter Image (local) + raw: "cat {{ role_path }}/dcgm-install-files/{{ arch }}/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04.tar.gz_* > /tmp/dcgm-exporter_{{ request_id }}.tar.gz" + delegate_to: 127.0.0.1 + +- name: Copy DCGM Exporter Image + include_tasks: send_file.yaml + vars: + input_file_path: "/tmp/dcgm-exporter_{{ request_id }}.tar.gz" + output_file_path: "/tmp/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04-{{ site_code }}.tar.gz" + output_file_owner: "root" + output_file_group: "root" + output_file_chmod: "644" + +- name: Remove Local Merged DCGM Exporter Image + raw: "rm -f /tmp/dcgm-exporter_{{ request_id }}.tar.gz" + delegate_to: 127.0.0.1 + +- name: Load DCGM Exporter Image + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "/cmp-agent/common/podman/bin/run-podman load -i /tmp/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04-{{ site_code }}.tar.gz" + +- name: Remove Temporary DCGM Exporter Image File + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "rm -rf /tmp/dcgm-exporter_4.4.2-4.7.0-ubuntu22.04-{{ site_code }}.tar.gz" + changed_when: true + +# 4. systemd 적용 및 시작 +- name: Install DCGM Exporter Service File + include_tasks: send_file.yaml + vars: + input_file_path: "{{ role_path }}/dcgm-install-files/{{ arch }}/cmp-dcgm-exporter.service" + output_file_path: "/lib/systemd/system/cmp-dcgm-exporter.service" + output_file_owner: "root" + output_file_group: "root" + output_file_chmod: "644" + +- name: Copy DCGM Exporter Custom Config File + include_tasks: send_file.yaml + vars: + input_file_path: "{{ role_path }}/dcgm-install-files/{{ arch }}/dcgm-exporter-custom.csv" + output_file_path: "/cmp-agent/dcgm-exporter-custom.csv" + output_file_owner: "root" + output_file_group: "root" + output_file_chmod: "644" + +- name: Check if nvidia-persistenced service exists + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "systemctl list-unit-files nvidia-persistenced.service --no-legend | grep -c nvidia-persistenced" + register: nvidia_persistenced_exists + changed_when: false + failed_when: false + +- name: Remove --no-persistence-mode from nvidia-persistenced service file + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "sed -i 's/--no-persistence-mode//g' /lib/systemd/system/nvidia-persistenced.service" + when: nvidia_persistenced_exists.stdout | trim | int > 0 + ignore_errors: true + +- name: Fix DCGM Exporter Service File SELinux Permission + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "chcon system_u:object_r:systemd_unit_file_t:s0 /lib/systemd/system/cmp-dcgm-exporter.service" + ignore_errors: true + +- name: Fix DCGM Exporter Custom Config File SELinux Permission + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "chcon system_u:object_r:systemd_unit_file_t:s0 /cmp-agent/dcgm-exporter-custom.csv" + ignore_errors: true + +- name: Fix NVIDIA Persistenced Service File SELinux Permission + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "chcon system_u:object_r:systemd_unit_file_t:s0 /lib/systemd/system/nvidia-persistenced.service" + when: nvidia_persistenced_exists.stdout | trim | int > 0 + ignore_errors: true + +- name: Reload systemd daemon + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "systemctl daemon-reload" + +- name: Start and Enable DCGM Exporter service + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "systemctl enable --now cmp-dcgm-exporter.service" + +- name: Restart NVIDIA Persistenced Service + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "systemctl restart nvidia-persistenced.service" + when: nvidia_persistenced_exists.stdout | trim | int > 0 + ignore_errors: true + +# 5. 공통 작업 종료 +- name: End Agent Common Tasks + include_tasks: "{{ role_path }}/tasks/dcgm_common_end.yaml" diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_uninstall.yaml b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_uninstall.yaml new file mode 100644 index 0000000..fe40c6e --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/dcgm_uninstall.yaml @@ -0,0 +1,125 @@ +- name: Stop and Disable DCGM Exporter service + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "systemctl disable --now cmp-dcgm-exporter.service 2>&1 | true && /cmp-agent/common/podman/bin/run-podman rm -f cmp-dcgm 2>&1 | true" + +- name: Detect OS family + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: grep '^ID=' /etc/os-release | head -n1 | cut -d'=' -f2 | tr -d '"' + register: os_id_raw + changed_when: false + failed_when: false + +- name: Debug OS ID + debug: + msg: "OS ID is {{ os_id_raw.stdout | trim }}" + +- name: Set OS family for RedHat-based + set_fact: + os_family: "redhat" + when: os_id_raw.stdout | trim in ['rhel', 'centos', 'rocky', 'almalinux', 'fedora'] + +- name: Set OS family for Debian-based + set_fact: + os_family: "debian" + when: os_id_raw.stdout | trim in ['ubuntu', 'debian'] + +- name: Set OS family as unknown + set_fact: + os_family: "unknown" + when: os_family is not defined + +- name: Display OS family + debug: + msg: "Detected os family is {{ os_family }}" + +# 2. Remove NVIDIA Container Toolkit +- name: Remove NVIDIA Container Toolkit DEB Packages + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "apt purge -y libnvidia-container1 libnvidia-container-tools nvidia-container-toolkit-base nvidia-container-toolkit" + when: os_family == "debian" + +- name: Remove NVIDIA Container Toolkit RPM Packages + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "yum remove -y libnvidia-container1 libnvidia-container-tools nvidia-container-toolkit-base nvidia-container-toolkit" + when: os_family == "redhat" + +# 3. DCGM Exporter 이미지 삭제 +- name: Remove DCGM Exporter Image + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "/cmp-agent/common/podman/bin/run-podman rmi nvcr.io/nvidia/k8s/dcgm-exporter:4.4.2-4.7.0-ubuntu22.04 2>&1 | true" + +# 4. systemd 서비스 삭제 +- name: Remove DCGM Exporter Service File + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "rm -f /lib/systemd/system/cmp-dcgm-exporter.service" + ignore_errors: true + +- name: Remove DCGM Exporter Custom Config File + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "rm -f /cmp-agent/dcgm-exporter-custom.csv" + ignore_errors: true + +- name: Reload systemd daemon + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "systemctl daemon-reload" diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/main.yaml b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/main.yaml new file mode 100644 index 0000000..c87f52a --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/main.yaml @@ -0,0 +1,24 @@ +- name: Detect Remote's architecture + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: uname -m + register: arch_raw + changed_when: false + failed_when: false + +- name: Set architecture variable + set_fact: + arch: "{{ 'amd64' if arch_raw.stdout | trim == 'x86_64' else 'arm64' if arch_raw.stdout | trim == 'aarch64' else arch_raw.stdout | trim }}" + +- name: Debug architecture + debug: + msg: "Architecture is {{ arch }}" + +# DCGM Exporter는 install / uninstall만 지원 (update/restart는 telegraf 에이전트만 해당) +- name: Run DCGM Exporter task based on install method + include_tasks: "dcgm_{{ install_method | default('install') }}.yaml" + when: install_method | default('install') in ['install', 'uninstall'] diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/send_file.yaml b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/send_file.yaml new file mode 100644 index 0000000..1b40d7e --- /dev/null +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/tasks/send_file.yaml @@ -0,0 +1,19 @@ +- name: Copy file to remote temp location + raw: | + scp -r -o StrictHostKeyChecking=no -P {{ target_port }} {% if target_sshkey_path | default('') | length > 0 %}-i {{ target_sshkey_path }} {% endif %}{{ input_file_path }} {{ target_user }}@{{ target_host }}:/tmp/temp_file_{{ request_id }} + delegate_to: 127.0.0.1 + +- name: Move file to final location with proper permissions + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: | + mv -f /tmp/temp_file_{{ request_id }} {{ output_file_path }} + chmod {{ output_file_chmod }} {{ output_file_path }} + chown {{ output_file_owner }}:{{ output_file_group }} {{ output_file_path }}