# ──────────────────────────────────────────────────────────
# Healthagent defaults — shipped with the package.
# Override any value via /etc/healthagent/config.yaml
# ──────────────────────────────────────────────────────────

modules:
  - gpu
  - systemd
  - network
  - kmsg
  - proc

# ── Network module ──────────────────────────────────────
network:
  services: []

  infiniband:
    state:
      eval: ne
      error: "4: ACTIVE"
      msg: "IB link is not active"
    phys_state:
      eval: ne
      error: "5: LinkUp"
      msg: "IB link not in state LinkUp"
    link_downed:
      eval: window_gt
      window: 10800
      error: 3
      strikes: 1
      msg: "IB link flapped 3+ times within a 3-hour window"
    link_error_recovery:
      eval: gt
      warning: 3
      msg: "IB link error recovery exceeded the threshold"
    operstate:
      eval: ne
      warning: up
      msg: "IPoIB not in state up"
    carrier_down_count:
      eval: gt
      warning: 5
      msg: "IPoIB carrier down count exceeded threshold"
  ethernet:
    carrier_down_count:
      eval: gt
      warning: 3
      msg: "carrier_down_count exceeded threshold"
    operstate:
      eval: ne
      error: up
      msg: "interface not in state up"

# ── GPU module ──────────────────────────────────────────
gpu:
  services:
    - nvidia-imex.service
    - nvidia-dcgm.service
    - nvidia-persistenced.service
  # By default all XIDS are treated as error.
  # XIDS in warning list are downgraded to warning.
  # If error list is explicitly specified, then only
  # specified XIDS will be treated as ERROR.
  xid:
    warning: [43, 63, 13, 31, 66, 94, 154]
    ignore: []
    error: []



  gpudiagnosticcheck:
    prolog:
      tests: short
      params: ""
    epilog:
      tests: medium
      params: ""

  field_watches:
    DCGM_FI_DEV_GPU_TEMP:
      eval: gt
      warning: 93
      category: Thermal
      msg: "GPU {gpu} temperature {value}°C exceeds {threshold}°C"
    DCGM_FI_DEV_CLOCKS_EVENT_REASONS:
      eval: bitmask
      error: 0xE8
      category: Clocks
      msg: "GPU {gpu} clock throttle reasons active: {value:#x} (mask: {threshold:#x})"
    DCGM_FI_DEV_PERSISTENCE_MODE:
      eval: ne
      error: 1
      category: System
      msg: "GPU {gpu} persistence mode not set. Restart nvidia-persistenced or reboot."
    DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION:
      eval: in
      warning: [3]
      error: [1, 2, 4]
      category: System
      msg: "GPU {gpu} recovery action requested (action: {value})"
    DCGM_FI_DEV_ROW_REMAP_FAILURE:
      eval: ne
      error: 0
      category: Memory
      msg: "GPU {gpu} row remap failure detected"
    DCGM_FI_DEV_RETIRED_PENDING:
      eval: gt
      warning: 0
      category: Memory
      msg: "GPU {gpu} has pages pending retirement (reboot recommended)"
    DCGM_FI_DEV_ECC_DBE_VOL_TOTAL:
      eval: gt
      error: 0
      category: Memory
      msg: "GPU {gpu} volatile DBE errors detected: {value}"
    DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER_FLOAT:
      eval: gt
      warning: 1.0e-6
      category: NVLink
      msg: "GPU {gpu} NVLink effective BER {value:.2e} exceeds {threshold:.2e}"
# ── Systemd module ──────────────────────────────────────
systemd:
  services:
    - munge.service
    - slurmd.service
    - slurmctld.service
    - slurmdbd.service
    - slurmrestd.service

# ── Process module ──────────────────────────────────────
proc:
  zombie_per_core: 50
  pid_max_warn_pct: 10
  pid_saturation_pct: 50

# ── Kmsg module ─────────────────────────────────────────
kmsg: {}