{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," $additionalGroupByLabels }}
{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
condition: '{{ true }}'
name: node-exporter
rules:
- alert: NodeFilesystemSpaceFillingUp
  annotations:
    description: 'Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up.'
    runbook_url: '{{ $runbookUrl }}/node/nodefilesystemspacefillingup'
    summary: 'Filesystem is predicted to run out of space within the next 24 hours.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15
    and
      predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
    and
      node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
    )
  for: 1h
  labels:
    severity: warning
- alert: NodeFilesystemSpaceFillingUp
  annotations:
    description: 'Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast.'
    runbook_url: '{{ $runbookUrl }}/node/nodefilesystemspacefillingup'
    summary: 'Filesystem is predicted to run out of space within the next 4 hours.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10
    and
      predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
    and
      node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
    )
  for: 1h
  labels:
    severity: critical
- alert: NodeFilesystemAlmostOutOfSpace
  annotations:
    description: 'Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.'
    runbook_url: '{{ $runbookUrl }}/node/nodefilesystemalmostoutofspace'
    summary: 'Filesystem has less than 5% space left.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
    and
      node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
    )
  for: 30m
  labels:
    severity: warning
- alert: NodeFilesystemAlmostOutOfSpace
  annotations:
    description: 'Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.'
    runbook_url: '{{ $runbookUrl }}/node/nodefilesystemalmostoutofspace'
    summary: 'Filesystem has less than 3% space left.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
    and
      node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
    )
  for: 30m
  labels:
    severity: critical
- alert: NodeFilesystemFilesFillingUp
  annotations:
    description: 'Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up.'
    runbook_url: '{{ $runbookUrl }}/node/nodefilesystemfilesfillingup'
    summary: 'Filesystem is predicted to run out of inodes within the next 24 hours.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40
    and
      predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
    and
      node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
    )
  for: 1h
  labels:
    severity: warning
- alert: NodeFilesystemFilesFillingUp
  annotations:
    description: 'Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast.'
    runbook_url: '{{ $runbookUrl }}/node/nodefilesystemfilesfillingup'
    summary: 'Filesystem is predicted to run out of inodes within the next 4 hours.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20
    and
      predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
    and
      node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
    )
  for: 1h
  labels:
    severity: critical
- alert: NodeFilesystemAlmostOutOfFiles
  annotations:
    description: 'Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.'
    runbook_url: '{{ $runbookUrl }}/node/nodefilesystemalmostoutoffiles'
    summary: 'Filesystem has less than 5% inodes left.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
    and
      node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
    )
  for: 1h
  labels:
    severity: warning
- alert: NodeFilesystemAlmostOutOfFiles
  annotations:
    description: 'Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.'
    runbook_url: '{{ $runbookUrl }}/node/nodefilesystemalmostoutoffiles'
    summary: 'Filesystem has less than 3% inodes left.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
    and
      node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
    )
  for: 1h
  labels:
    severity: critical
- alert: NodeNetworkReceiveErrs
  annotations:
    description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.'
    runbook_url: '{{ $runbookUrl }}/node/nodenetworkreceiveerrs'
    summary: 'Network interface is reporting many receive errors.'
  condition: '{{ true }}'
  expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01
  for: 1h
  labels:
    severity: warning
- alert: NodeNetworkTransmitErrs
  annotations:
    description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.'
    runbook_url: '{{ $runbookUrl }}/node/nodenetworktransmiterrs'
    summary: 'Network interface is reporting many transmit errors.'
  condition: '{{ true }}'
  expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01
  for: 1h
  labels:
    severity: warning
- alert: NodeHighNumberConntrackEntriesUsed
  annotations:
    description: '{{`{{`}} $labels.instance {{`}}`}} {{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.'
    runbook_url: '{{ $runbookUrl }}/node/nodehighnumberconntrackentriesused'
    summary: 'Number of conntrack are getting close to the limit.'
  condition: '{{ true }}'
  expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75
  labels:
    severity: warning
- alert: NodeTextFileCollectorScrapeError
  annotations:
    description: 'Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape.'
    runbook_url: '{{ $runbookUrl }}/node/nodetextfilecollectorscrapeerror'
    summary: 'Node Exporter text file collector failed to scrape.'
  condition: '{{ true }}'
  expr: node_textfile_scrape_error{job="node-exporter"} == 1
  labels:
    severity: warning
- alert: NodeClockSkewDetected
  annotations:
    description: 'Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host.'
    runbook_url: '{{ $runbookUrl }}/node/nodeclockskewdetected'
    summary: 'Clock skew detected.'
  condition: '{{ true }}'
  expr: |-
    (
      node_timex_offset_seconds{job="node-exporter"} > 0.05
    and
      deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0
    )
    or
    (
      node_timex_offset_seconds{job="node-exporter"} < -0.05
    and
      deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0
    )
  for: 10m
  labels:
    severity: warning
- alert: NodeClockNotSynchronising
  annotations:
    description: 'Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host.'
    runbook_url: '{{ $runbookUrl }}/node/nodeclocknotsynchronising'
    summary: 'Clock not synchronising.'
  condition: '{{ true }}'
  expr: |-
    min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0
    and
    node_timex_maxerror_seconds{job="node-exporter"} >= 16
  for: 10m
  labels:
    severity: warning
- alert: NodeRAIDDegraded
  annotations:
    description: 'RAID array ''{{`{{`}} $labels.device {{`}}`}}'' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.'
    runbook_url: '{{ $runbookUrl }}/node/noderaiddegraded'
    summary: 'RAID Array is degraded.'
  condition: '{{ true }}'
  expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0
  for: 15m
  labels:
    severity: critical
- alert: NodeRAIDDiskFailure
  annotations:
    description: 'At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array ''{{`{{`}} $labels.device {{`}}`}}'' needs attention and possibly a disk swap.'
    runbook_url: '{{ $runbookUrl }}/node/noderaiddiskfailure'
    summary: 'Failed device in RAID array.'
  condition: '{{ true }}'
  expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0
  labels:
    severity: warning
- alert: NodeFileDescriptorLimit
  annotations:
    description: 'File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.'
    runbook_url: '{{ $runbookUrl }}/node/nodefiledescriptorlimit'
    summary: 'Kernel is predicted to exhaust file descriptors limit soon.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70
    )
  for: 15m
  labels:
    severity: warning
- alert: NodeFileDescriptorLimit
  annotations:
    description: 'File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.'
    runbook_url: '{{ $runbookUrl }}/node/nodefiledescriptorlimit'
    summary: 'Kernel is predicted to exhaust file descriptors limit soon.'
  condition: '{{ true }}'
  expr: |-
    (
      node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90
    )
  for: 15m
  labels:
    severity: critical
- alert: NodeCPUHighUsage
  annotations:
    description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.

      '
    runbook_url: '{{ $runbookUrl }}/node/nodecpuhighusage'
    summary: 'High CPU usage.'
  condition: '{{ true }}'
  expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!~"idle|iowait"}[2m]))) * 100 > 90
  for: 15m
  labels:
    severity: info
- alert: NodeSystemSaturation
  annotations:
    description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}.

      This might indicate this instance resources saturation and can cause it becoming unresponsive.

      '
    runbook_url: '{{ $runbookUrl }}/node/nodesystemsaturation'
    summary: 'System saturated, load per core is very high.'
  condition: '{{ true }}'
  expr: |-
    node_load1{job="node-exporter"}
    / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2
  for: 15m
  labels:
    severity: warning
- alert: NodeMemoryMajorPagesFaults
  annotations:
    description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}.

      Please check that there is enough memory available at this instance.

      '
    runbook_url: '{{ $runbookUrl }}/node/nodememorymajorpagesfaults'
    summary: 'Memory major page faults are occurring at very high rate.'
  condition: '{{ true }}'
  expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500
  for: 15m
  labels:
    severity: warning
- alert: NodeMemoryHighUtilization
  annotations:
    description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.

      '
    runbook_url: '{{ $runbookUrl }}/node/nodememoryhighutilization'
    summary: 'Host is running out of memory.'
  condition: '{{ true }}'
  expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90
  for: 15m
  labels:
    severity: warning
- alert: NodeDiskIOSaturation
  annotations:
    description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}.

      This symptom might indicate disk saturation.

      '
    runbook_url: '{{ $runbookUrl }}/node/nodediskiosaturation'
    summary: 'Disk IO queue is high.'
  condition: '{{ true }}'
  expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10
  for: 30m
  labels:
    severity: warning
- alert: NodeSystemdServiceFailed
  annotations:
    description: 'Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}}'
    runbook_url: '{{ $runbookUrl }}/node/nodesystemdservicefailed'
    summary: 'Systemd service has entered failed state.'
  condition: '{{ true }}'
  expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1
  for: 5m
  labels:
    severity: warning
- alert: NodeSystemdServiceCrashlooping
  annotations:
    description: 'Systemd service {{`{{`}} $labels.name {{`}}`}} has being restarted too many times at {{`{{`}} $labels.instance {{`}}`}} for the last 15 minutes. Please check if service is crash looping.'
    runbook_url: '{{ $runbookUrl }}/node/nodesystemdservicecrashlooping'
    summary: 'Systemd service keeps restaring, possibly crash looping.'
  condition: '{{ true }}'
  expr: increase(node_systemd_service_restart_total{job="node-exporter"}[5m]) > 2
  for: 15m
  labels:
    severity: warning
- alert: NodeBondingDegraded
  annotations:
    description: 'Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures.'
    runbook_url: '{{ $runbookUrl }}/node/nodebondingdegraded'
    summary: 'Bonding interface is degraded'
  condition: '{{ true }}'
  expr: (node_bonding_slaves - node_bonding_active) != 0
  for: 5m
  labels:
    severity: warning