fix: Fix mc-observability fluent-bit installation

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/playbook.yaml

@@ -7,6 +7,11 @@
     # 보고 mc-o11y-manager가 "linux" / "windows" 둘 중 하나로 채움.
     # default "linux" 로 두는 이유: 옛 호출자(직접 Semaphore template 호출 등)에 대한 호환 + Telegraf/FluentBit/Beyla 기존 케이스.
     os_type: "{{ os_type | default('linux') }}"
+    # enable_gpu: NVIDIA GPU 노드에서 DCGM Exporter를 함께 설치/제거할지 여부.
+    # telegraf 에이전트 설치 시에만 의미가 있으며, telegraf의 prometheus input이
+    # localhost:9400(DCGM Exporter)을 스크랩해 GPU 메트릭을 수집한다.
+    enable_gpu: "{{ enable_gpu | default(false) }}"
+    NVIDIA_CONTAINER_TOOLKIT_VERSION: "1.19.0-1"
   tasks:
     - name: Check variables
       include_role:
@@ -36,6 +41,13 @@
       no_log: true
       when: os_type == 'windows'
 
+    # DCGM Exporter (GPU 메트릭): Linux + telegraf + enable_gpu 인 경우에만 수행.
+    # (o11y-infra dcgm role 포팅 — podman으로 dcgm-exporter 컨테이너 구동, 포트 9400)
+    - name: DCGM Exporter Operation
+      include_role:
+        name: dcgm
+      when: os_type == 'linux' and agent == 'telegraf' and (enable_gpu | bool)
+
     - name: Agent Operation (Linux)
       include_role:
         name: agent

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf

@@ -80,6 +80,6 @@
     name loki
     host ${LOKI_HOST}
     match winevtlog.*
-    labels host=${HOSTNAME}, NS_ID=${NS_ID}, MCI_ID=${MCI_ID}, VM_ID=${VM_ID}
+    labels host=${HOSTNAME}, NS_ID=${NS_ID}, INFRA_ID=${INFRA_ID}, NODE_ID=${NODE_ID}
     label_keys $service, $level, $source
     line_format key_value

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/fluent-bit/etc/variables

@@ -1,4 +1,4 @@
 NS_ID=
-MCI_ID=
-VM_ID=
+INFRA_ID=
+NODE_ID=
 LOKI_HOST=127.0.0.1

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent-windows/agent-install-files/amd64/telegraf/etc/telegraf.conf

@@ -1,7 +1,7 @@
 [global_tags]
   ns_id="@NS_ID"
-  mci_id="@MCI_ID"
-  vm_id="@VM_ID"
+  infra_id="@INFRA_ID"
+  node_id="@NODE_ID"
 
 [agent]
   interval = "30s"

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/fluent-bit.conf

@@ -54,5 +54,5 @@
     name loki
     host ${LOKI_HOST}
     match syslog
-    labels host=${HOSTNAME}, NS_ID=${NS_ID}, MCI_ID=${MCI_ID}, VM_ID=${VM_ID}
+    labels host=${HOSTNAME}, NS_ID=${NS_ID}, INFRA_ID=${INFRA_ID}, NODE_ID=${NODE_ID}
     label_keys $service, $level, $source

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/fluent-bit/etc/variables

@@ -1,4 +1,4 @@
 NS_ID=
-MCI_ID=
-VM_ID=
+INFRA_ID=
+NODE_ID=
 LOKI_HOST=127.0.0.1

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/telegraf/etc/telegraf.conf

@@ -1,7 +1,7 @@
 [global_tags]
   ns_id="@NS_ID"
-  mci_id="@MCI_ID"
-  vm_id="@VM_ID"
+  infra_id="@INFRA_ID"
+  node_id="@NODE_ID"
 
 [agent]
   interval = "30s"

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/fluent-bit.conf

@@ -54,5 +54,5 @@
     name loki
     host ${LOKI_HOST}
     match syslog
-    labels host=${HOSTNAME}, NS_ID=${NS_ID}, MCI_ID=${MCI_ID}, VM_ID=${VM_ID}
+    labels host=${HOSTNAME}, NS_ID=${NS_ID}, INFRA_ID=${INFRA_ID}, NODE_ID=${NODE_ID}
     label_keys $service, $level, $source

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/fluent-bit/etc/variables

@@ -1,4 +1,4 @@
 NS_ID=
-MCI_ID=
-VM_ID=
+INFRA_ID=
+NODE_ID=
 LOKI_HOST=127.0.0.1

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/telegraf/etc/telegraf.conf

@@ -1,7 +1,7 @@
 [global_tags]
   ns_id="@NS_ID"
-  mci_id="@MCI_ID"
-  vm_id="@VM_ID"
+  infra_id="@INFRA_ID"
+  node_id="@NODE_ID"
 
 [agent]
   interval = "30s"

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/cmp-dcgm-exporter.service

@@ -0,0 +1,41 @@
+[Unit]
+Description=CMP DCGM Exporter Service
+Wants=network-online.target
+After=network-online.target
+RequiresMountsFor=%t/containers
+
+[Service]
+Type=notify
+NotifyAccess=all
+User=root
+
+
+Environment=TIME_DIFF_SECONDS=0
+Environment=PODMAN_SYSTEMD_UNIT=%n
+
+ExecStart=/cmp-agent/common/podman/bin/run-podman run \
+    --pid=host \
+    --cgroups=no-conmon \
+    --env TIME_DIFF_SECONDS \
+    --env HOSTNAME=%H \
+    --sdnotify=conmon \
+    --replace \
+    -d \
+    --restart=always \
+    --tz=Asia/Seoul \
+    --network=host \
+    --device=nvidia.com/gpu=all \
+    --name cmp-dcgm \
+    -p 9400:9400 \
+    --mount type=bind,src=/cmp-agent/dcgm-exporter-custom.csv,dst=/cmp-agent/dcgm-exporter-custom.csv,ro=true \
+    nvcr.io/nvidia/k8s/dcgm-exporter:4.4.2-4.7.0-ubuntu22.04 \
+    -f /cmp-agent/dcgm-exporter-custom.csv
+
+ExecStop=/cmp-agent/common/podman/bin/run-podman stop cmp-dcgm
+
+Restart=on-failure
+RestartPreventExitStatus=100
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/amd64/dcgm-exporter-custom.csv

@@ -0,0 +1,92 @@
+# Format
+# If line starts with a '#' it is considered a comment
+# DCGM FIELD, Prometheus metric type, help message
+
+# Clocks
+DCGM_FI_DEV_SM_CLOCK,  gauge, SM clock frequency (in MHz).
+DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).
+# DCGM_EXP_CLOCK_EVENTS_COUNT, gauge, Count of clock events within the user-specified time window (see clock-events-count-window-size param).
+
+# Temperature
+DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
+DCGM_FI_DEV_GPU_TEMP,    gauge, GPU temperature (in C).
+DCGM_FI_DEV_FAN_SPEED, gauge, Fan speed for the device in percent(in %).
+
+# Power
+DCGM_FI_DEV_POWER_USAGE,              gauge, Power draw (in W).
+DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).
+DCGM_FI_DEV_PSTATE, gauge, Performance state.
+
+# PCIE
+DCGM_FI_DEV_PCIE_TX_THROUGHPUT,  counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML.
+DCGM_FI_DEV_PCIE_RX_THROUGHPUT,  counter, Total number of bytes received through PCIe RX (in KB) via NVML.
+DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.
+
+# Utilization (the sample period varies depending on the product)
+DCGM_FI_DEV_GPU_UTIL,      gauge, GPU utilization (in %).
+DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
+DCGM_FI_DEV_ENC_UTIL,      gauge, Encoder utilization (in %).
+DCGM_FI_DEV_DEC_UTIL ,     gauge, Decoder utilization (in %).
+
+# Errors and violations
+DCGM_FI_DEV_XID_ERRORS,           gauge,   Value of the last XID error encountered.
+DCGM_FI_DEV_CLOCKS_EVENT_REASONS, gauge, Clock throttle reasons
+DCGM_EXP_XID_ERRORS_COUNT,        gauge,   Count of XID Errors within user-specified time window (see xid-count-window-size param).
+
+# DCGM_FI_DEV_POWER_VIOLATION,       counter, Throttling duration due to power constraints (in us).
+# DCGM_FI_DEV_THERMAL_VIOLATION,     counter, Throttling duration due to thermal constraints (in us).
+# DCGM_FI_DEV_SYNC_BOOST_VIOLATION,  counter, Throttling duration due to sync-boost constraints (in us).
+# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us).
+# DCGM_FI_DEV_LOW_UTIL_VIOLATION,    counter, Throttling duration due to low utilization (in us).
+# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).
+
+# Memory usage
+DCGM_FI_DEV_FB_FREE,  gauge, Frame buffer memory free (in MB).
+DCGM_FI_DEV_FB_USED,  gauge, Frame buffer memory used (in MB).
+DCGM_FI_DEV_FB_TOTAL, gauge, Frame buffer memory (in MB).
+
+# ECC (Datacenter GPU only, A100, H100, V100, T4)
+DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.
+DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.
+DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.
+DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.
+
+# Retired pages
+# DCGM_FI_DEV_RETIRED_SBE,     counter, Total number of retired pages due to single-bit errors.
+# DCGM_FI_DEV_RETIRED_DBE,     counter, Total number of retired pages due to double-bit errors.
+# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.
+
+# NVLink
+# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors.
+# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors.
+# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL,   counter, Total number of NVLink retries.
+# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors.
+DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL,            counter, Total number of NVLink bandwidth counters for all lanes
+
+# MIG (Multi-Instance GPU)
+DCGM_FI_DEV_MIG_MODE,       gauge, MIG mode status (0=disabled / 1=enabled)
+DCGM_FI_DEV_MIG_MAX_SLICES, gauge, Maximum number of MIG slices supported
+
+# VGPU License status
+DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status
+
+# Remapped rows
+DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors
+DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS,   counter, Number of remapped rows for correctable errors
+DCGM_FI_DEV_ROW_REMAP_FAILURE,           gauge,   Whether remapping of rows has failed
+
+# Static configuration information. These appear as labels on the other metrics
+DCGM_FI_DRIVER_VERSION,      label, Driver Version
+DCGM_FI_DEV_PCI_BUSID, label, PCI attributes for the device
+DCGM_FI_CUDA_DRIVER_VERSION, label, CUDA Driver Version
+DCGM_FI_DEV_VBIOS_VERSION,     label, VBIOS version of the device
+DCGM_FI_NVML_VERSION,          label, NVML Version
+# DCGM_FI_DEV_BRAND,             label, Device Brand
+# DCGM_FI_DEV_SERIAL,            label, Device Serial Number
+# DCGM_FI_DEV_OEM_INFOROM_VER,   label, OEM inforom version
+# DCGM_FI_DEV_ECC_INFOROM_VER,   label, ECC inforom version
+# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version
+# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version
+
+# Process identifier
+DCGM_FI_PROCESS_NAME,         label,   Process name

conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/dcgm/dcgm-install-files/arm64/cmp-dcgm-exporter.service

@@ -0,0 +1,41 @@
+[Unit]
+Description=CMP DCGM Exporter Service
+Wants=network-online.target
+After=network-online.target
+RequiresMountsFor=%t/containers
+
+[Service]
+Type=notify
+NotifyAccess=all
+User=root
+
+
+Environment=TIME_DIFF_SECONDS=0
+Environment=PODMAN_SYSTEMD_UNIT=%n
+
+ExecStart=/cmp-agent/common/podman/bin/run-podman run \
+    --pid=host \
+    --cgroups=no-conmon \
+    --env TIME_DIFF_SECONDS \
+    --env HOSTNAME=%H \
+    --sdnotify=conmon \
+    --replace \
+    -d \
+    --restart=always \
+    --tz=Asia/Seoul \
+    --network=host \
+    --device=nvidia.com/gpu=all \
+    --name cmp-dcgm \
+    -p 9400:9400 \
+    --mount type=bind,src=/cmp-agent/dcgm-exporter-custom.csv,dst=/cmp-agent/dcgm-exporter-custom.csv,ro=true \
+    nvcr.io/nvidia/k8s/dcgm-exporter:4.4.2-4.7.0-ubuntu22.04 \
+    -f /cmp-agent/dcgm-exporter-custom.csv
+
+ExecStop=/cmp-agent/common/podman/bin/run-podman stop cmp-dcgm
+
+Restart=on-failure
+RestartPreventExitStatus=100
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target