{{- $Values := (.helm).Values | default .Values }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }}
{{- $defaultDatasource := "prometheus" -}}
{{- range (((($Values.grafana).sidecar).datasources).victoriametrics | default list) }}
  {{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }}
{{- end }}
annotations:
  list:
  - builtIn: 1
    datasource:
      type: datasource
      uid: grafana
    enable: true
    hide: true
    iconColor: rgba(0, 211, 255, 1)
    name: Annotations & Alerts
    target:
      limit: 100
      matchAny: false
      tags: []
      type: dashboard
    type: dashboard
  - datasource:
      type: prometheus
      uid: $ds
    enable: true
    expr: sum(ALERTS{job=~"$job", instance=~"$instance", alertgroup="vmcluster",alertstate="firing",show_at="dashboard"}) by(alertname)
    hide: false
    iconColor: red
    name: alerts
    titleFormat: '{{`{{`}}alertname{{`}}`}}'
  - datasource:
      type: prometheus
      uid: $ds
    enable: true
    expr: sum(vm_app_version{job=~"$job", instance=~"$instance"}) by(version) unless (sum(vm_app_version{job=~"$job", instance=~"$instance"} offset $__interval) by(version))
    hide: true
    iconColor: dark-blue
    name: version change
    textFormat: '{{`{{`}}version{{`}}`}}'
    titleFormat: Version change
  - datasource:
      type: prometheus
      uid: $ds
    enable: true
    expr: sum(changes(vm_app_start_timestamp{job=~"$job", instance=~"$instance"}[$__interval])) by(job)
    hide: false
    iconColor: dark-yellow
    name: restarts
    textFormat: '{{`{{`}}job{{`}}`}} restarted'
condition: {{ ($Values.vmcluster).enabled }}
description: Overview for cluster VictoriaMetrics v1.117.0 or higher
editable: false
fiscalYearStartMonth: 0
graphTooltip: 1
id: 1
links:
- icon: doc
  tags: []
  targetBlank: true
  title: Cluster Wiki
  type: link
  url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/
- icon: external link
  tags: []
  targetBlank: true
  title: Found a bug?
  type: link
  url: https://github.com/VictoriaMetrics/VictoriaMetrics/issues
- icon: external link
  tags: []
  targetBlank: true
  title: New releases
  type: link
  url: https://github.com/VictoriaMetrics/VictoriaMetrics/releases
panels:
- collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    'y': 0
  id: 137
  panels: []
  title: Stats
  type: row
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are in storage
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 0
    'y': 1
  id: 131
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ''
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    exemplar: true
    expr: sum(vm_rows{job=~"$job_storage", type!~"indexdb.*"})
    format: time_series
    instant: true
    interval: ''
    intervalFactor: 1
    legendFormat: ''
    refId: A
  title: Total datapoints
  type: stat
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the datapoints ingestion rate, including replication factor.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 6
    'y': 1
  id: 124
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ''
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    exemplar: false
    expr: sum(rate(vm_vminsert_metrics_read_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]))
    format: time_series
    instant: true
    interval: ''
    intervalFactor: 1
    legendFormat: ''
    range: false
    refId: A
  title: Ingestion rate
  type: stat
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the rate of HTTP read requests.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
      unit: req/s
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 12
    'y': 1
  id: 130
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ''
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    exemplar: true
    expr: sum(rate(vm_http_requests_total{job=~"$job", instance=~"$instance", path=~"/select/.*"}[$__rate_interval]))
    format: time_series
    instant: true
    interval: ''
    intervalFactor: 1
    legendFormat: ''
    refId: A
  title: Read requests
  type: stat
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: 'Total number of available CPUs for all VM components. '
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 18
    'y': 1
  id: 126
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ''
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    exemplar: true
    expr: sum(vm_available_cpu_cores{job=~"$job", instance=~"$instance"})
    format: time_series
    instant: true
    interval: ''
    intervalFactor: 1
    legendFormat: ''
    refId: A
  title: Available CPU
  type: stat
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 0
    'y': 4
  id: 34
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ''
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    exemplar: true
    expr: sum(max_over_time(vm_cache_entries{job=~"$job", instance=~"$instance", type="storage/hour_metric_ids"}[1h]))
    format: time_series
    instant: true
    interval: ''
    intervalFactor: 1
    legendFormat: ''
    refId: A
  title: Active series
  type: stat
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Total amount of used disk space
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
      unit: bytes
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 6
    'y': 4
  id: 35
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ''
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    exemplar: false
    expr: sum(vm_data_size_bytes{job=~"$job_storage"})
    format: time_series
    instant: true
    interval: ''
    intervalFactor: 1
    legendFormat: ''
    refId: A
  title: Disk space usage
  type: stat
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Average disk usage per datapoint.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      decimals: 2
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
      unit: bytes
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 12
    'y': 4
  id: 112
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ''
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    exemplar: true
    expr: sum(vm_data_size_bytes{job=~"$job_storage"}) / sum(vm_rows{job=~"$job_storage", type!~"indexdb.*"})
    format: time_series
    instant: true
    interval: ''
    intervalFactor: 1
    legendFormat: ''
    refId: A
  title: Bytes per point
  type: stat
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Total size of available memory for all VM components.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
      unit: bytes
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 18
    'y': 4
  id: 128
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ''
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    exemplar: true
    expr: sum(vm_available_memory_bytes{job=~"$job", instance=~"$instance"})
    format: time_series
    instant: true
    interval: ''
    intervalFactor: 1
    legendFormat: ''
    refId: A
  title: Available memory
  type: stat
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      custom:
        align: auto
        cellOptions:
          type: auto
        inspect: false
        minWidth: 50
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
        - color: red
          value: 80
    overrides:
    - matcher:
        id: byName
        options: Time
      properties:
      - id: custom.hidden
        value: true
    - matcher:
        id: byName
        options: Value
      properties:
      - id: displayName
        value: Count
  gridPos:
    h: 5
    w: 8
    x: 0
    'y': 7
  id: 149
  options:
    cellHeight: sm
    footer:
      countRows: false
      fields: ''
      reducer:
      - sum
      show: false
    showHeader: true
    sortBy:
    - desc: true
      displayName: Count
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    exemplar: false
    expr: sum(vm_app_version{job=~"$job", instance=~"$instance"}) by(job, short_version)
    format: table
    instant: true
    range: false
    refId: A
  title: ''
  type: table
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ''
        axisPlacement: auto
        axisSoftMin: 0
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: stepAfter
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: 'off'
      decimals: 0
      links: []
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
        - color: red
          value: 80
      unit: none
    overrides: []
  gridPos:
    h: 5
    w: 16
    x: 8
    'y': 7
  id: 62
  options:
    legend:
      calcs:
      - lastNotNull
      displayMode: table
      placement: right
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    expr: sum(min_over_time(up{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job)
    format: time_series
    instant: false
    legendFormat: '{{`{{`}}job{{`}}`}}'
    refId: A
  title: Uptime ($job)
  type: timeseries
- collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    'y': 12
  id: 10
  panels: []
  title: Overview
  type: row
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are inserted into cluster per second by protocol before the replication. Check vminsert metrics if there are any issues with ingestion.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ''
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineStyle:
          fill: solid
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: normal
        thresholdsStyle:
          mode: 'off'
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    'y': 13
  id: 2
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    exemplar: true
    expr: sum(rate(vm_rows_inserted_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (type) > 0
    interval: ''
    legendFormat: '{{`{{`}}type{{`}}`}}'
    range: true
    refId: A
  title: Datapoints ingestion rate ($instance)
  type: timeseries
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: '* `*` - unsupported query path

    * `/write` - insert into VM

    * `/metrics` - query VM system metrics

    * `/query` - query instant values

    * `/query_range` - query over a range of time

    * `/series` - match a certain label set

    * `/label/{}/values` - query a list of label values (variables mostly)'
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ''
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: 'off'
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    'y': 13
  id: 6
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    expr: sum(rate(vm_http_requests_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (path) > 0
    format: time_series
    intervalFactor: 1
    legendFormat: '{{`{{`}}path{{`}}`}}'
    range: true
    refId: A
  title: Requests rate ($instance)
  type: timeseries
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour across all storage nodes. High value may result in ingestion slowdown and high memory usage.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ''
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: 'off'
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    'y': 21
  id: 12
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: sum(vm_cache_entries{job=~"$job", instance=~"$instance", type="storage/hour_metric_ids"})
    format: time_series
    intervalFactor: 1
    legendFormat: Active time series
    refId: A
  title: Active time series ($instance)
  type: timeseries
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: 'The less time it takes is better.

    * `*` - unsupported query path

    * `/write` - insert into VM

    * `/metrics` - query VM system metrics

    * `/query` - query instant values

    * `/query_range` - query over a range of time

    * `/series` - match a certain label set

    * `/label/{}/values` - query a list of label values (variables mostly)'
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ''
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: 'off'
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
        - color: red
          value: 80
      unit: s
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    'y': 21
  id: 8
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    expr: max(vm_request_duration_seconds{job=~"$job", instance=~"$instance", quantile="0.99"}) by (path) > 0
    format: time_series
    intervalFactor: 1
    legendFormat: '{{`{{`}}path{{`}}`}}'
    range: true
    refId: A
  title: Query duration 0.99 quantile ($instance)
  type: timeseries
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: '* `*` - unsupported query path

    * `/write` - insert into VM

    * `/metrics` - query VM system metrics

    * `/query` - query instant values

    * `/query_range` - query over a range of time

    * `/series` - match a certain label set

    * `/label/{}/values` - query a list of label values (variables mostly)'
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ''
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: 'off'
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    'y': 29
  id: 52
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    expr: sum(rate(vm_http_request_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job, path) > 0
    format: time_series
    intervalFactor: 1
    legendFormat: '{{`{{`}}path{{`}}`}} ({{`{{`}}job{{`}}`}})'
    range: true
    refId: A
  title: Requests error rate ($instance)
  type: timeseries
- datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ''
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: bars
        fillOpacity: 100
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: 'off'
      links:
      - targetBlank: true
        title: Drilldown
        url: /d/oS7Bi_0Wz?viewPanel=203&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: null
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    'y': 29
  id: 104
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 11.5.0
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    editorMode: code
    exemplar: true
    expr: sum(rate(vm_log_messages_total{job=~"$job",instance=~"$instance", level!="info"}[$__rate_interval])) by (job, level) > 0
    format: time_series
    hide: false
    interval: 5m
    intervalFactor: 1
    legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}level{{`}}`}}'
    range: true
    refId: A
  title: Logging rate
  type: timeseries
- collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    'y': 37
  id: 46
  panels:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Percentage of used RSS memory (resident).

      The RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).

      The application''s performance will significantly degrade when memory usage is close to 100%.


      Click on the line and choose Drilldown to show memory usage per instance'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2293
    id: 66
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            max_over_time(process_resident_memory_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job", instance=~"$instance"}
        ) by(job)
      interval: ''
      legendFormat: __auto
      range: true
      refId: A
    title: RSS memory % usage ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2293
    id: 64
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            rate(process_cpu_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job", instance=~"$instance"}
        ) by(job)
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    title: CPU ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.

      Safe memory usage % considered to be below 80%


      Click on the line and choose Drilldown to show memory usage per instance'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2301
    id: 138
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            max_over_time(process_resident_memory_anon_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job", instance=~"$instance"}
        ) by(job)
      interval: ''
      legendFormat: __auto
      range: true
      refId: A
    title: RSS anonymous memory % usage ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).


      The lower the better.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2301
    id: 219
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(rate(process_pressure_cpu_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
      format: time_series
      interval: ''
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(rate(process_pressure_cpu_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
      range: true
      refId: B
    title: CPU pressure
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).


      The lower the better.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2309
    id: 220
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(rate(process_pressure_memory_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
      format: time_series
      interval: ''
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(rate(process_pressure_memory_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
      range: true
      refId: B
    title: Memory pressure
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the number of bytes read/write from the storage layer.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: Bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read .*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2309
    id: 122
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(rate(process_io_storage_read_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: read {{`{{`}}job{{`}}`}}
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(process_io_storage_written_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: write {{`{{`}}job{{`}}`}}
      range: true
      refId: B
    title: Disk writes/reads ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the percentage of open file descriptors compared to the limit set in the OS.

      Reaching the limit of open files can cause various issues and must be prevented.


      See how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides:
      - matcher:
          id: byRegexp
          options: /max.*/
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2317
    id: 117
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            max_over_time(process_open_fds{job=~"$job", instance=~"$instance"}[$__rate_interval])
            /
            process_max_fds{job=~"$job", instance=~"$instance"}
        ) by(job)
      format: time_series
      interval: ''
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    title: Open FDs usage % ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the number of read/write syscalls such as read, pread, write, pwrite.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byRegexp
          options: /read .*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2317
    id: 204
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(rate(process_io_read_syscalls_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: read {{`{{`}}job{{`}}`}}
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(process_io_write_syscalls_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: write {{`{{`}}job{{`}}`}}
      range: true
      refId: B
    title: Disk write/read calls ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2325
    id: 68
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(max_over_time(go_goroutines{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
      format: time_series
      interval: ''
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    title: Goroutines ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).


      The lower the better.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2325
    id: 221
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(rate(process_pressure_io_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
      format: time_series
      interval: ''
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(rate(process_pressure_io_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
      range: true
      refId: B
    title: IO pressure
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2333
    id: 70
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(max_over_time(process_num_threads{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
      format: time_series
      interval: ''
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    title: Threads ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2333
    id: 119
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(max_over_time(vm_tcplistener_conns{job=~"$job", instance=~"$instance"}[$__interval])) by(job)
      interval: ''
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    title: TCP connections ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the percent of CPU spent on garbage collection.


      If % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.


      Try searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ '
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2341
    id: 210
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: "max(\n  rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)"
      format: time_series
      interval: ''
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    title: CPU spent on GC ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2341
    id: 120
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(rate(vm_tcplistener_accepts_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
      interval: ''
      legendFormat: __auto
      range: true
      refId: A
    title: TCP connections rate ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2349
    id: 218
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(rate(go_memstats_alloc_bytes_total{job=~"$job"}[$__rate_interval])) by (job)
      format: time_series
      interval: ''
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    title: Memory allocations rate
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of            insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.1
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2349
    id: 213
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job, instance, le))) by(job)
      format: time_series
      interval: ''
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    title: Go scheduling latency
    type: timeseries
  title: Resource usage ($job)
  type: row
- collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    'y': 38
  id: 106
  panels:
  - description: ''
    fieldConfig:
      defaults: {}
      overrides: []
    gridPos:
      h: 2
      w: 24
      x: 0
      'y': 1519
    id: 211
    options:
      code:
        language: plaintext
        showLineNumbers: false
        showMiniMap: false
      content: See [Troubleshooting](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) docs.
      mode: markdown
    pluginVersion: 11.5.0
    title: ''
    transparent: true
    type: text
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the rate and total number of new series created over last 24h.


      High [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM''s or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#cardinality) "explosions".


      The higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.


      To investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format


      Good references to read:

      * https://www.robustperception.io/cardinality-is-key

      * https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: new series over 24h
        properties:
        - id: custom.axisPlacement
          value: right
        - id: custom.axisSoftMin
          value: 0
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 1521
    id: 102
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      exemplar: true
      expr: sum(rate(vm_new_timeseries_created_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]))
      interval: ''
      legendFormat: churn rate
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      exemplar: true
      expr: sum(increase(vm_new_timeseries_created_total{job=~"$job_storage", instance=~"$instance"}[24h]))
      hide: false
      interval: ''
      legendFormat: new series over 24h
      refId: B
    title: Churn rate ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "The percentage of [slow inserts](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \n\nThe lower the better. \n\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \n* Ingestion of completely new, not seen before time series;\n* [Re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\n* Not enough memory to maintain big enough caches for the current workload.\n\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series). \n\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line+area
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: transparent
            value: null
          - color: red
            value: 0.1
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 1521
    id: 108
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: "max(\n    rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval]) \n    / rate(vm_rows_added_to_storage_total{job=~\"$job_storage\"}[$__rate_interval])\n)"
      interval: ''
      legendFormat: slow inserts
      range: true
      refId: A
    title: Slow inserts
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: bars
          fillOpacity: 100
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 1529
    id: 170
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(increase(vm_assisted_merges_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(type) > 0
      format: time_series
      interval: 5m
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    title: Assisted merges ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.


      The less value is better.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 1529
    id: 107
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: |-
        sum(rate(vm_slow_queries_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval]))
        /
        sum(rate(vm_http_requests_total{job=~"$job_select", instance=~"$instance", path=~"/select/.*"}[$__rate_interval]))
      interval: ''
      legendFormat: slow queries %
      range: true
      refId: A
    title: Slow queries % ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 0
      'y': 1537
    id: 144
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: "max(\n    vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} \n    /\n    vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(type)"
      interval: ''
      legendFormat: __auto
      range: true
      refId: A
    title: Cache usage % by type ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows cache miss ratio. Lower is better.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        max: 1
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 12
      'y': 1537
    id: 58
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            rate(vm_cache_misses_total{job=~"$job", instance=~"$instance"}[$__rate_interval])
            /
            rate(vm_cache_requests_total{job=~"$job", instance=~"$instance"}[$__rate_interval])
        ) by(type)
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    title: Cache miss ratio ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 1546
    id: 142
    links:
    - targetBlank: true
      title: Readonly mode
      url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(vm_rpc_vmstorage_is_read_only{job=~"$job_insert", instance=~"$instance"}) by(instance, addr) > 0
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}addr{{`}}`}}'
      range: true
      refId: A
    title: Storage in readonly status for vminsert ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.


      Should be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn''t set.


      Deduplication happens in two places:

      * `type="merge"` - during [background merges](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage) by vmstorages. Deduplication during merges is permanent.

      * `type="select"` - during [read queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data) by vmselects. Deduplication happens in-flight and has no permanent effect.


      The more samples need to be deduplicated, the higher will be resource usage on vmselects or vmstorages.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 1546
    id: 215
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(rate(vm_deduplicated_samples_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type) >0
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    title: Deduplication rate ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows how many samples were ignored or dropped on insertion due to various reasons:

      * timestamp out of retention period or timestamp in future;

      * invalid metric name;

      * exceeding limit for labels length or number;

      * dropped by [relabeling configuration](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#relabeling).'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 1554
    id: 135
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(increase(vm_rows_ignored_total{job=~"$job", instance=~"$instance"}[1h])) by (reason)
      hide: false
      interval: ''
      legendFormat: __auto
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(increase(vm_relabel_metrics_dropped_total{job=~"$job", instance=~"$instance"}[1h]))
      hide: false
      instant: false
      legendFormat: relabeling
      range: true
      refId: B
    title: Samples dropped for last 1h ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the number of partial results served by `vmselects`.


      `vmselect` continues serving queries if at least a single vmstorage nodes is available. It marks responses as `partial` for queries served from the remaining healthy vmstorage nodes.


      If you prefer consistency over availability then run vmselect nodes with `-search.denyPartialResponse` cmd-line flag. In this case vmselect returns an error if at least a single vmstorage node is unavailable.


      See more at [cluster availability](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability).'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: stepAfter
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: none
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 1554
    id: 217
    options:
      legend:
        calcs:
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(increase(vm_partial_results_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) > 0
      format: time_series
      instant: false
      legendFormat: partial results
      refId: A
    title: Partial query results ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: thresholds
        custom:
          align: auto
          cellOptions:
            type: auto
          inspect: false
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
      overrides:
      - matcher:
          id: byName
          options: Value
        properties:
        - id: custom.hidden
          value: true
      - matcher:
          id: byName
          options: Time
        properties:
        - id: custom.hidden
          value: true
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 1562
    id: 183
    options:
      cellHeight: sm
      footer:
        countRows: false
        fields: ''
        reducer:
        - sum
        show: false
      showHeader: true
      sortBy:
      - desc: true
        displayName: job
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: false
      expr: sum(flag{is_set="true", job=~"$job", instance=~"$instance"}) by(job, instance, name, value)
      format: table
      instant: true
      legendFormat: __auto
      range: false
      refId: A
    title: Non-default flags
    transformations:
    - id: groupBy
      options:
        fields:
          instance:
            aggregations:
            - uniqueValues
            operation: aggregate
          job:
            aggregations: []
            operation: groupby
          name:
            aggregations: []
            operation: groupby
          value:
            aggregations: []
            operation: groupby
    type: table
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn''t restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. '
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: stepAfter
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: none
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 1562
    id: 214
    options:
      legend:
        calcs:
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(changes(vm_app_start_timestamp{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
      format: time_series
      instant: false
      legendFormat: '{{`{{`}}job{{`}}`}}'
      refId: A
    title: Restarts ($job)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: ${ds}
    description: 'Shows the amount of data scheduled for [downsampling](https://docs.victoriametrics.com/#downsampling) or [retention filters](https://docs.victoriametrics.com/#retention-filters).


      Applying downsampling or retention filters happens in background. `vmstorage` schedules this process periodically when finds data partitions that satisfy configured retention or downsampling rules.


      This process could consume additional CPU and Disk IO resources.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: auto
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: null
          - color: red
            value: 80
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 1570
    id: 222
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: single
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      editorMode: code
      expr: sum(vm_downsampling_partitions_scheduled_size_bytes{job=~"$job", instance=~"$instance"}) > 0
      legendFormat: downsampling
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      editorMode: code
      expr: sum(vm_retention_partitions_scheduled_size_bytes{job=~"$job", instance=~"$instance"}) > 0
      hide: false
      instant: false
      legendFormat: retention filters
      range: true
      refId: B
    title: Partitions scheduled for re-processing
    type: timeseries
  title: Troubleshooting
  type: row
- collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    'y': 39
  id: 48
  panels:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 0
      'y': 4941
    id: 76
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_rpc_rows_pushed_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
      format: time_series
      intervalFactor: 1
      legendFormat: Pushed
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_rpc_rows_sent_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
      format: time_series
      intervalFactor: 1
      legendFormat: Sent
      range: true
      refId: E
    title: Rows ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Rate of RPC errors by type:

      * `Connection` - the number of connection errors to vmstorage node

      * `Dial` -  the number of dial errors to vmstorage node.

      * `Handshake` - the number of handshake errors to vmstorage node

      * `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 12
      'y': 4941
    id: 86
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_rpc_connection_errors_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
      format: time_series
      intervalFactor: 1
      legendFormat: Connection
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: sum(rate(vm_rpc_dial_errors_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
      format: time_series
      intervalFactor: 1
      legendFormat: Dial
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: sum(rate(vm_rpc_handshake_errors_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
      format: time_series
      intervalFactor: 1
      legendFormat: Handshake
      refId: E
    title: RPC errors ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: rps
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4950
    id: 80
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(addr) > 0
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    title: 'Rows ($instance) rerouted to '
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: rps
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4950
    id: 78
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      exemplar: true
      expr: sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(addr) > 0
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: '{{`{{`}}addr{{`}}`}}'
      refId: A
    title: Rows ($instance) rerouted from
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of rows or bytes that vminsert internal buffer contains at the moment.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: bytes
        properties:
        - id: unit
          value: bytes
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4958
    id: 82
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: sum(vm_rpc_buf_pending_bytes{job=~"$job", instance=~"$instance"})
      legendFormat: bytes
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: sum(vm_rpc_rows_pending{job=~"$job", instance=~"$instance"})
      legendFormat: rows
      refId: B
    title: Pending
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Network usage by internal VictoriaMetrics RPC protocol
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4958
    id: 74
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcpdialer_written_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) * 8
      legendFormat: network usage
      range: true
      refId: A
    title: RPC network usage ($instance)
    type: timeseries
  title: Interconnection ($job)
  type: row
- collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    'y': 40
  id: 24
  panels:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the number of datapoints ingested into storage nodes per second. This metric doesn't show all stored datapoints since some of them may be dropped because of wrong timestamps or decode errors.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: normal
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2387
    id: 100
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_vminsert_metrics_read_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]))
      format: time_series
      intervalFactor: 1
      legendFormat: ingestion rate
      range: true
      refId: A
    title: Ingestion rate ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.

      Safe memory usage % considered to be below 80%


      Click on the line and choose Drilldown to show memory usage per instance'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 7
      w: 12
      x: 12
      'y': 2387
    id: 167
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            max_over_time(process_resident_memory_anon_bytes{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job_storage", instance=~"$instance"}
        )
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        min(
            max_over_time(process_resident_memory_anon_bytes{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job_storage", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        quantile(0.5,
            max_over_time(process_resident_memory_anon_bytes{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job_storage", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    title: Memory (anon) usage % ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the number of concurrently executed read requests, where:

      * `max` - equal to `-search.maxConcurrentRequest` cmd-line flag;

      * `current` - current number of concurrent select requests executed by storage.


      Each concurrent select request requires RAM, CPU and disk IO resources during processing. Too many concurrent requests could result into OOM exceptions.


      When `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).


      See more at https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#resource-usage-limits'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2394
    id: 133
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: "max(\n    max_over_time(vm_vmselect_concurrent_requests_current{job=~\"$job_storage\", \n    instance=~\"$instance\"}[$__rate_interval])\n)"
      interval: ''
      legendFormat: current
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: min(vm_vmselect_concurrent_requests_capacity{job=~"$job_storage", instance=~"$instance"})
      hide: false
      interval: ''
      legendFormat: max
      range: true
      refId: B
    title: Concurrent selects ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: ''
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 7
      w: 12
      x: 0
      'y': 2395
    id: 151
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: false
      expr: |-
        max(
            rate(process_cpu_seconds_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job_storage", instance=~"$instance"}
        )
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: false
      expr: |-
        min(
            rate(process_cpu_seconds_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job_storage", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: false
      expr: |-
        quantile(0.5,
            rate(process_cpu_seconds_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job_storage", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    title: CPU usage % ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row)."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2402
    id: 212
    links:
    - targetBlank: true
      title: Related discussion
      url: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: "max(\n    max_over_time(vm_concurrent_insert_current{job=~\"$job_storage\", \n    instance=~\"$instance\"}[$__rate_interval])\n)"
      interval: ''
      legendFormat: current
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: min(vm_concurrent_insert_capacity{job=~"$job_storage", instance=~"$instance"})
      hide: false
      interval: ''
      legendFormat: max
      range: true
      refId: B
    title: Concurrent flushes on disk ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of rows merged per second by storage nodes. Merge speed depends on available CPU and disk IO bandwidth.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2402
    id: 55
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_rows_merged_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(type)
      legendFormat: __auto
      range: true
      refId: A
    title: Merge speed
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "The max number of on-going merges across storage nodes.\n The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.\n It is expected to have high numbers for `storage/small` metric."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2410
    id: 54
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(max_over_time(vm_active_merges{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(type)
      legendFormat: __auto
      range: true
      refId: A
    title: Active merges ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage).

      Increase in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.

      * `indexdb` - inverted index

      * `storage/small` - recently added parts of data ingested into storage (hot data)

      * `storage/big` -  small parts gradually merged into bigger parts (cold data)'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2410
    id: 22
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(vm_parts{job=~"$job_storage", instance=~"$instance", type=~"indexdb.*"}) by(type)
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(vm_last_partition_parts{job=~"$job_storage", instance=~"$instance"}) by(type)
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: B
    title: LSM parts max by type ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=200&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2418
    id: 20
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: "max(\n    sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n    (\n        sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n        sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n    ) \n)"
      format: time_series
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: "min(\n    sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n    (\n        sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n        sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n    ) \n)"
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: "quantile(0.5,\n    sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n    (\n        sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n        sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n    ) \n)"
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    title: Disk space usage %  ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: pending index entries
        properties:
        - id: unit
          value: none
        - id: decimals
          value: 3
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2418
    id: 14
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(max_over_time(vm_pending_rows{job=~"$job_storage", instance=~"$instance", type="storage"}[$__rate_interval]))
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: pending datapoints
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(max_over_time(vm_pending_rows{job=~"$job_storage", instance=~"$instance", type="indexdb"}[$__rate_interval]))
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: pending index entries
      range: true
      refId: B
    title: Pending datapoints ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly.


      The sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=201&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2426
    id: 202
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: "max(\n    sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)\n    / \n    sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)"
      format: time_series
      intervalFactor: 1
      legendFormat: indexdb
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: "max(\n    sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)\n    / \n    sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)"
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: datapoints
      range: true
      refId: B
    title: Disk space usage % by type  ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Value > 0 means vmstorage is in readonly mode.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2426
    id: 141
    links:
    - targetBlank: true
      title: Readonly mode
      url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: false
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: vm_storage_is_read_only{job=~"$job_storage", instance=~"$instance"} > 0
      interval: ''
      legendFormat: '{{`{{`}} instance {{`}}`}}'
      range: true
      refId: A
    title: Readonly mode
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows network usage by vmstorage services.

      * Writes show traffic sent to clients.

      * Reads show traffic received from clients.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 2434
    id: 206
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcplistener_read_bytes_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(name) * 8 > 0
      format: time_series
      intervalFactor: 1
      legendFormat: read from {{`{{`}}name{{`}}`}}
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcplistener_written_bytes_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(name) * 8 > 0
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to {{`{{`}}name{{`}}`}}
      range: true
      refId: B
    title: Network usage ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the max number of existing [snapshots](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots) across vmstorages.


      Recently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.


      It is recommended deleting old snapshots when they are no longer needed to free up disk space.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 2434
    id: 216
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: false
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: max(vm_snapshots{job=~"$job_storage", instance=~"$instance"})
      interval: ''
      legendFormat: snapshots
      range: true
      refId: A
    title: Number of snapshots
    type: timeseries
  title: vmstorage ($instance)
  type: row
- collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    'y': 41
  id: 42
  panels:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Request rate accepted by vmselect nodes
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4954
    id: 92
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_http_requests_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (path) > 0
      format: time_series
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    title: Requests rate ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the max number of concurrent selects across instances.

      * `max` limit can be configured via `search.maxConcurrentRequests` flag

      * `current` shows the current number of goroutines busy with processing requests


      When `current` hits `max` constantly, it means one or more vmselect nodes are overloaded with number of requests. If you observe that CPU for vmselects is saturated, consider adding more vmselect replicas or increase CPU resources. If CPU and Memory panels show a plenty of free resources - try increasing `-search.maxConcurrentRequests`. Please note, the higher is `-search.maxConcurrentRequests`, the higher could be [peak memory usage](https://docs.victoriametrics.com/victoriametrics/troubleshooting/#out-of-memory-errors).'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
        - id: custom.fillOpacity
          value: 0
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4954
    id: 95
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(max_over_time(vm_concurrent_select_current{job=~"$job_select", instance=~"$instance"}[$__rate_interval]))
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: current
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: min(vm_concurrent_select_capacity{job=~"$job_select", instance=~"$instance"})
      format: time_series
      intervalFactor: 1
      legendFormat: max
      refId: B
    title: Concurrent selects ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4962
    id: 163
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            rate(process_cpu_seconds_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job_select", instance=~"$instance"}
        )
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        min(
            rate(process_cpu_seconds_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job_select", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        quantile(0.5,
            rate(process_cpu_seconds_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job_select", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    title: CPU usage % ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.

      Safe memory usage % considered to be below 80%


      Click on the line and choose Drilldown to show memory usage per instance'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4962
    id: 165
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            max_over_time(process_resident_memory_anon_bytes{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job_select", instance=~"$instance"}
        )
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        min(
            max_over_time(process_resident_memory_anon_bytes{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job_select", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        quantile(0.5,
            max_over_time(process_resident_memory_anon_bytes{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job_select", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    title: Memory (anon) usage % ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 99th percentile of the number of time series read per query.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4970
    id: 178
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: series
      range: true
      refId: A
    title: Series read per query ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per queried time series.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4970
    id: 180
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: datapoints
      range: true
      refId: A
    title: Datapoints read per series ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per query.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4978
    id: 179
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: datapoints
      range: true
      refId: A
    title: Datapoints read per query ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: '99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scanner per query.


      This number can exceed number of DatapointsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4978
    id: 181
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: datapoints
      range: true
      refId: A
    title: Datapoints scanned per query ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4986
    id: 93
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcplistener_read_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0
      format: time_series
      intervalFactor: 1
      legendFormat: read from client
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcplistener_written_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to client
      range: true
      refId: B
    title: 'Network usage: clients ($instance)'
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows network usage between vmselects and vmstorages.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4986
    id: 207
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcpdialer_read_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0
      format: time_series
      intervalFactor: 1
      legendFormat: read from vmstorage
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcpdialer_written_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to vmstorage
      range: true
      refId: B
    title: 'Network usage: vmstorage ($instance)'
    type: timeseries
  title: vmselect ($instance)
  type: row
- collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    'y': 42
  id: 40
  panels:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: '* `*` - unsupported query path

      * `/write` - insert into VM

      * `/metrics` - query VM system metrics

      * `/query` - query instant values

      * `/query_range` - query over a range of time

      * `/series` - match a certain label set

      * `/label/{}/values` - query a list of label values (variables mostly)'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4955
    id: 97
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_http_requests_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) by (path) > 0
      format: time_series
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    title: Requests rate ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the max number of ongoing insertions.

      * `max` - equal to number of CPU * 2 by default. May be configured with `maxConcurrentInserts` flag;

      * `current` - current number of goroutines busy with processing requests.


      `-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.


      When `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
        - id: custom.fillOpacity
          value: 0
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4955
    id: 99
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(max_over_time(vm_concurrent_insert_current{job=~"$job_insert", instance=~"$instance"}[$__rate_interval]))
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: current
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      exemplar: true
      expr: min(vm_concurrent_insert_capacity{job=~"$job_insert", instance=~"$instance"})
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: max
      refId: B
    title: Concurrent inserts ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4963
    id: 185
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            rate(process_cpu_seconds_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job_insert", instance=~"$instance"}
        )
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        min(
            rate(process_cpu_seconds_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job_insert", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        quantile(0.5,
            rate(process_cpu_seconds_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
            /
            process_cpu_cores_available{job=~"$job_insert", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    title: CPU usage % ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.

      Safe memory usage % considered to be below 80%


      Click on the line and choose Drilldown to show memory usage per instance'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4963
    id: 187
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        max(
            max_over_time(process_resident_memory_anon_bytes{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job_insert", instance=~"$instance"}
        )
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        min(
            max_over_time(process_resident_memory_anon_bytes{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job_insert", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: |-
        quantile(0.5,
            max_over_time(process_resident_memory_anon_bytes{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
            /
            vm_available_memory_bytes{job=~"$job_insert", instance=~"$instance"}
        )
      format: time_series
      hide: false
      interval: ''
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    title: Memory (anon) usage % ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the saturation level of connection between vminsert and vmstorage components. \n\nIf the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that either vminsert or vmstorage nodes are struggling with the load. Verify CPU/mem saturation of both components and network saturation between them.\nIf vminsert resources are saturated - consider adding more resources or scale vminserts horizontally.\n\nIf vminsert resources and network are fine, check vmstorage metrics for anomalies."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line+area
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: transparent
          - color: red
            value: 0.9
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4971
    id: 139
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: max(rate(vm_rpc_send_duration_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(addr)
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    title: Storage connection saturation ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows when vmstorage node is unreachable for vminsert.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4971
    id: 114
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: vm_rpc_vmstorage_is_reachable{job=~"$job", instance=~"$instance"} != 1
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}addr{{`}}`}}'
      range: true
      refId: A
    title: Storage reachability ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows network usage between vminserts and clients, such as vmagent, Prometheus, or any other client pushing metrics to vminsert.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 4979
    id: 208
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcplistener_read_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0
      format: time_series
      intervalFactor: 1
      legendFormat: read from client
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcplistener_written_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to client
      range: true
      refId: B
    title: 'Network usage: clients ($instance)'
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows network usage between vminserts and vmstorages.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4979
    id: 209
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcpdialer_read_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0
      format: time_series
      intervalFactor: 1
      legendFormat: read from vmstorage
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(rate(vm_tcpdialer_written_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to vmstorage
      range: true
      refId: B
    title: 'Network usage: vmstorage ($instance)'
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: ''
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 4987
    id: 88
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    title: Rows per insert ($instance)
    type: timeseries
  title: vminsert ($instance)
  type: row
- collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    'y': 43
  id: 194
  panels:
  - fieldConfig:
      defaults: {}
      overrides: []
    gridPos:
      h: 2
      w: 24
      x: 0
      'y': 5004
    id: 198
    options:
      code:
        language: plaintext
        showLineNumbers: false
        showMiniMap: false
      content: Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.
      mode: markdown
    pluginVersion: 9.2.7
    title: ''
    transparent: true
    type: text
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the used memory (resident).

      The application''s performance will significantly degrade when memory usage is close to 100%.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 5006
    id: 189
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: max_over_time(process_resident_memory_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
      interval: ''
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    title: RSS memory usage ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 5006
    id: 190
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: max_over_time(process_resident_memory_anon_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
      interval: ''
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    title: RSS anonymous memory usage ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: ''
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: short
      overrides: []
    gridPos:
      h: 7
      w: 12
      x: 0
      'y': 5014
    id: 192
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: false
      expr: sum(rate(process_cpu_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, instance)
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    title: CPU usage ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the approx time needed to reach 100% of disk capacity based on the following params:

      * free disk space (after -storage.minFreeDiskSpaceBytes);

      * row ingestion rate;

      * compression.'
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: s
      overrides: []
    gridPos:
      h: 7
      w: 12
      x: 12
      'y': 5014
    id: 196
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - min
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: "(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n    (rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) - \n        sum(rate(vm_deduplicated_samples_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])) without (type)) * \n    (\n        sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type) /\n        sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type)\n    )\n    +\n    rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) * \n    (\n        sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"}) /\n        sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"})\n    )\n) > 0"
      format: time_series
      interval: ''
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}}'
      range: true
      refId: A
    title: Storage full ETA ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 5021
    id: 200
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(vm_data_size_bytes{job=~"$job", instance=~"$instance"}) by(job, instance)
      format: time_series
      intervalFactor: 1
      legendFormat: ''
      range: true
      refId: A
    title: Disk space usage ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: ''
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      'y': 5021
    id: 201
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(vm_data_size_bytes{job=~"$job", instance=~"$instance", type=~"indexdb.*"}) by(job, instance)
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (indexdb)'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      expr: sum(vm_data_size_bytes{job=~"$job", instance=~"$instance", type!~"indexdb.*"}) by(job, instance)
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (datapoints)'
      range: true
      refId: B
    title: Disk space usage by type ($instance)
    type: timeseries
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ''
          axisPlacement: auto
          barAlignment: 0
          drawStyle: bars
          fillOpacity: 100
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: 'off'
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      'y': 5029
    id: 203
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      editorMode: code
      exemplar: true
      expr: sum(rate(vm_log_messages_total{job=~"$job",instance=~"$instance", level!="info"}[$__rate_interval])) by (job, instance, level, location) > 0
      format: time_series
      hide: false
      interval: 5m
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    title: Logging rate
    type: timeseries
  title: Drilldown
  type: row
preload: false
refresh: ''
schemaVersion: 40
tags:
- victoriametrics
- vm-k8s-stack
templating:
  list:
  - current:
      text: VictoriaMetrics - cluster
      value: PAF93674D0B4E9963
    includeAll: false
    name: ds
    options: []
    query: {{ $defaultDatasource }}
    refresh: 1
    regex: ''
    type: datasource
  - current: {}
    datasource:
      type: prometheus
      uid: $ds
    definition: label_values(vm_app_version{version=~"^vm(insert|select|storage).*"}, job)
    includeAll: true
    multi: true
    name: job
    options: []
    query:
      query: label_values(vm_app_version{version=~"^vm(insert|select|storage).*"}, job)
      refId: VictoriaMetrics-job-Variable-Query
    refresh: 1
    regex: ''
    type: query
  - current: {}
    datasource:
      type: prometheus
      uid: $ds
    definition: label_values(vm_app_version{job=~"$job", version=~"^vminsert.*"}, job)
    hide: 2
    includeAll: true
    multi: true
    name: job_insert
    options: []
    query:
      query: label_values(vm_app_version{job=~"$job", version=~"^vminsert.*"}, job)
      refId: StandardVariableQuery
    refresh: 1
    regex: ''
    type: query
  - current: {}
    datasource:
      type: prometheus
      uid: $ds
    definition: label_values(vm_app_version{job=~"$job", version=~"^vmselect.*"}, job)
    hide: 2
    includeAll: true
    multi: true
    name: job_select
    options: []
    query:
      query: label_values(vm_app_version{job=~"$job", version=~"^vmselect.*"}, job)
      refId: StandardVariableQuery
    refresh: 1
    regex: ''
    type: query
  - current: {}
    datasource:
      type: prometheus
      uid: $ds
    definition: label_values(vm_app_version{job=~"$job", version=~"^vmstorage.*"}, job)
    hide: 2
    includeAll: true
    multi: true
    name: job_storage
    options: []
    query:
      query: label_values(vm_app_version{job=~"$job", version=~"^vmstorage.*"}, job)
      refId: StandardVariableQuery
    refresh: 1
    regex: ''
    type: query
  - allValue: .*
    current: {}
    datasource:
      type: prometheus
      uid: $ds
    definition: label_values(vm_app_version{job=~"$job"}, instance)
    includeAll: true
    multi: true
    name: instance
    options: []
    query:
      query: label_values(vm_app_version{job=~"$job"}, instance)
      refId: VictoriaMetrics-instance-Variable-Query
    refresh: 1
    regex: ''
    type: query
  - baseFilters: []
    datasource:
      type: prometheus
      uid: $ds
    filters: []
    name: adhoc
    type: adhoc
time:
  from: now-3h
  to: now
timepicker:
  refresh_intervals:
  - 10s
  - 30s
  - 1m
  - 5m
  - 15m
  - 30m
  - 1h
  - 2h
  - 1d
timezone: {{ default "utc" ($Values.defaultDashboards).defaultTimezone }}
title: VictoriaMetrics - cluster
uid: oS7Bi_0Wz
version: 1
weekStart: ''
