gitea_admin/infra
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
68d0ddbb73
infra/charts/victoria-metrics-k8s-stack/files/dashboards/generated/victoriametrics-cluster.yaml
Konstantin Averkiev c45fd1a6ac added vm stack
2025-07-08 17:29:32 +03:00

8343 lines
209 KiB
YAML
Raw Blame History

{{- $Values := (.helm).Values | default .Values }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }}
{{- $defaultDatasource := "prometheus" -}}
{{- range (((($Values.grafana).sidecar).datasources).victoriametrics | default list) }}
{{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }}
{{- end }}
annotations:
list:
- builtIn: 1
datasource:
type: datasource
uid: grafana
enable: true
hide: true
iconColor: rgba(0, 211, 255, 1)
name: Annotations & Alerts
target:
limit: 100
matchAny: false
tags: []
type: dashboard
type: dashboard
- datasource:
type: prometheus
uid: $ds
enable: true
expr: sum(ALERTS{job=~"$job", instance=~"$instance", alertgroup="vmcluster",alertstate="firing",show_at="dashboard"}) by(alertname)
hide: false
iconColor: red
name: alerts
titleFormat: '{{`{{`}}alertname{{`}}`}}'
- datasource:
type: prometheus
uid: $ds
enable: true
expr: sum(vm_app_version{job=~"$job", instance=~"$instance"}) by(version) unless (sum(vm_app_version{job=~"$job", instance=~"$instance"} offset $__interval) by(version))
hide: true
iconColor: dark-blue
name: version change
textFormat: '{{`{{`}}version{{`}}`}}'
titleFormat: Version change
- datasource:
type: prometheus
uid: $ds
enable: true
expr: sum(changes(vm_app_start_timestamp{job=~"$job", instance=~"$instance"}[$__interval])) by(job)
hide: false
iconColor: dark-yellow
name: restarts
textFormat: '{{`{{`}}job{{`}}`}} restarted'
condition: {{ ($Values.vmcluster).enabled }}
description: Overview for cluster VictoriaMetrics v1.117.0 or higher
editable: false
fiscalYearStartMonth: 0
graphTooltip: 1
id: 1
links:
- icon: doc
tags: []
targetBlank: true
title: Cluster Wiki
type: link
url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/
- icon: external link
tags: []
targetBlank: true
title: Found a bug?
type: link
url: https://github.com/VictoriaMetrics/VictoriaMetrics/issues
- icon: external link
tags: []
targetBlank: true
title: New releases
type: link
url: https://github.com/VictoriaMetrics/VictoriaMetrics/releases
panels:
- collapsed: false
gridPos:
h: 1
w: 24
x: 0
'y': 0
id: 137
panels: []
title: Stats
type: row
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are in storage
fieldConfig:
defaults:
color:
mode: thresholds
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
unit: short
overrides: []
gridPos:
h: 3
w: 6
x: 0
'y': 1
id: 131
maxDataPoints: 100
options:
colorMode: value
graphMode: area
justifyMode: auto
orientation: horizontal
percentChangeColorMode: standard
reduceOptions:
calcs:
- lastNotNull
fields: ''
values: false
showPercentChange: false
text: {}
textMode: auto
wideLayout: true
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
exemplar: true
expr: sum(vm_rows{job=~"$job_storage", type!~"indexdb.*"})
format: time_series
instant: true
interval: ''
intervalFactor: 1
legendFormat: ''
refId: A
title: Total datapoints
type: stat
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the datapoints ingestion rate, including replication factor.
fieldConfig:
defaults:
color:
mode: thresholds
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
unit: short
overrides: []
gridPos:
h: 3
w: 6
x: 6
'y': 1
id: 124
maxDataPoints: 100
options:
colorMode: value
graphMode: area
justifyMode: auto
orientation: horizontal
percentChangeColorMode: standard
reduceOptions:
calcs:
- lastNotNull
fields: ''
values: false
showPercentChange: false
text: {}
textMode: auto
wideLayout: true
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: false
expr: sum(rate(vm_vminsert_metrics_read_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]))
format: time_series
instant: true
interval: ''
intervalFactor: 1
legendFormat: ''
range: false
refId: A
title: Ingestion rate
type: stat
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the rate of HTTP read requests.
fieldConfig:
defaults:
color:
mode: thresholds
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
unit: req/s
overrides: []
gridPos:
h: 3
w: 6
x: 12
'y': 1
id: 130
maxDataPoints: 100
options:
colorMode: value
graphMode: area
justifyMode: auto
orientation: horizontal
percentChangeColorMode: standard
reduceOptions:
calcs:
- lastNotNull
fields: ''
values: false
showPercentChange: false
text: {}
textMode: auto
wideLayout: true
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(rate(vm_http_requests_total{job=~"$job", instance=~"$instance", path=~"/select/.*"}[$__rate_interval]))
format: time_series
instant: true
interval: ''
intervalFactor: 1
legendFormat: ''
refId: A
title: Read requests
type: stat
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Total number of available CPUs for all VM components. '
fieldConfig:
defaults:
color:
mode: thresholds
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
unit: short
overrides: []
gridPos:
h: 3
w: 6
x: 18
'y': 1
id: 126
maxDataPoints: 100
options:
colorMode: value
graphMode: area
justifyMode: auto
orientation: horizontal
percentChangeColorMode: standard
reduceOptions:
calcs:
- lastNotNull
fields: ''
values: false
showPercentChange: false
text: {}
textMode: auto
wideLayout: true
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(vm_available_cpu_cores{job=~"$job", instance=~"$instance"})
format: time_series
instant: true
interval: ''
intervalFactor: 1
legendFormat: ''
refId: A
title: Available CPU
type: stat
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown.
fieldConfig:
defaults:
color:
mode: thresholds
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
unit: short
overrides: []
gridPos:
h: 3
w: 6
x: 0
'y': 4
id: 34
maxDataPoints: 100
options:
colorMode: value
graphMode: area
justifyMode: auto
orientation: horizontal
percentChangeColorMode: standard
reduceOptions:
calcs:
- lastNotNull
fields: ''
values: false
showPercentChange: false
text: {}
textMode: auto
wideLayout: true
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(max_over_time(vm_cache_entries{job=~"$job", instance=~"$instance", type="storage/hour_metric_ids"}[1h]))
format: time_series
instant: true
interval: ''
intervalFactor: 1
legendFormat: ''
refId: A
title: Active series
type: stat
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Total amount of used disk space
fieldConfig:
defaults:
color:
mode: thresholds
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
unit: bytes
overrides: []
gridPos:
h: 3
w: 6
x: 6
'y': 4
id: 35
maxDataPoints: 100
options:
colorMode: value
graphMode: area
justifyMode: auto
orientation: horizontal
percentChangeColorMode: standard
reduceOptions:
calcs:
- lastNotNull
fields: ''
values: false
showPercentChange: false
text: {}
textMode: auto
wideLayout: true
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
exemplar: false
expr: sum(vm_data_size_bytes{job=~"$job_storage"})
format: time_series
instant: true
interval: ''
intervalFactor: 1
legendFormat: ''
refId: A
title: Disk space usage
type: stat
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Average disk usage per datapoint.
fieldConfig:
defaults:
color:
mode: thresholds
decimals: 2
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
unit: bytes
overrides: []
gridPos:
h: 3
w: 6
x: 12
'y': 4
id: 112
maxDataPoints: 100
options:
colorMode: value
graphMode: area
justifyMode: auto
orientation: horizontal
percentChangeColorMode: standard
reduceOptions:
calcs:
- lastNotNull
fields: ''
values: false
showPercentChange: false
text: {}
textMode: auto
wideLayout: true
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
exemplar: true
expr: sum(vm_data_size_bytes{job=~"$job_storage"}) / sum(vm_rows{job=~"$job_storage", type!~"indexdb.*"})
format: time_series
instant: true
interval: ''
intervalFactor: 1
legendFormat: ''
refId: A
title: Bytes per point
type: stat
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Total size of available memory for all VM components.
fieldConfig:
defaults:
color:
mode: thresholds
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
unit: bytes
overrides: []
gridPos:
h: 3
w: 6
x: 18
'y': 4
id: 128
maxDataPoints: 100
options:
colorMode: value
graphMode: area
justifyMode: auto
orientation: horizontal
percentChangeColorMode: standard
reduceOptions:
calcs:
- lastNotNull
fields: ''
values: false
showPercentChange: false
text: {}
textMode: auto
wideLayout: true
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(vm_available_memory_bytes{job=~"$job", instance=~"$instance"})
format: time_series
instant: true
interval: ''
intervalFactor: 1
legendFormat: ''
refId: A
title: Available memory
type: stat
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: thresholds
custom:
align: auto
cellOptions:
type: auto
inspect: false
minWidth: 50
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
overrides:
- matcher:
id: byName
options: Time
properties:
- id: custom.hidden
value: true
- matcher:
id: byName
options: Value
properties:
- id: displayName
value: Count
gridPos:
h: 5
w: 8
x: 0
'y': 7
id: 149
options:
cellHeight: sm
footer:
countRows: false
fields: ''
reducer:
- sum
show: false
showHeader: true
sortBy:
- desc: true
displayName: Count
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: false
expr: sum(vm_app_version{job=~"$job", instance=~"$instance"}) by(job, short_version)
format: table
instant: true
range: false
refId: A
title: ''
type: table
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
axisSoftMin: 0
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: stepAfter
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: none
overrides: []
gridPos:
h: 5
w: 16
x: 8
'y': 7
id: 62
options:
legend:
calcs:
- lastNotNull
displayMode: table
placement: right
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(min_over_time(up{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job)
format: time_series
instant: false
legendFormat: '{{`{{`}}job{{`}}`}}'
refId: A
title: Uptime ($job)
type: timeseries
- collapsed: false
gridPos:
h: 1
w: 24
x: 0
'y': 12
id: 10
panels: []
title: Overview
type: row
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are inserted into cluster per second by protocol before the replication. Check vminsert metrics if there are any issues with ingestion.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineStyle:
fill: solid
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: normal
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 13
id: 2
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(rate(vm_rows_inserted_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (type) > 0
interval: ''
legendFormat: '{{`{{`}}type{{`}}`}}'
range: true
refId: A
title: Datapoints ingestion rate ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: '* `*` - unsupported query path
* `/write` - insert into VM
* `/metrics` - query VM system metrics
* `/query` - query instant values
* `/query_range` - query over a range of time
* `/series` - match a certain label set
* `/label/{}/values` - query a list of label values (variables mostly)'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 13
id: 6
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_http_requests_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (path) > 0
format: time_series
intervalFactor: 1
legendFormat: '{{`{{`}}path{{`}}`}}'
range: true
refId: A
title: Requests rate ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour across all storage nodes. High value may result in ingestion slowdown and high memory usage.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 21
id: 12
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
expr: sum(vm_cache_entries{job=~"$job", instance=~"$instance", type="storage/hour_metric_ids"})
format: time_series
intervalFactor: 1
legendFormat: Active time series
refId: A
title: Active time series ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'The less time it takes is better.
* `*` - unsupported query path
* `/write` - insert into VM
* `/metrics` - query VM system metrics
* `/query` - query instant values
* `/query_range` - query over a range of time
* `/series` - match a certain label set
* `/label/{}/values` - query a list of label values (variables mostly)'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: s
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 21
id: 8
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(vm_request_duration_seconds{job=~"$job", instance=~"$instance", quantile="0.99"}) by (path) > 0
format: time_series
intervalFactor: 1
legendFormat: '{{`{{`}}path{{`}}`}}'
range: true
refId: A
title: Query duration 0.99 quantile ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: '* `*` - unsupported query path
* `/write` - insert into VM
* `/metrics` - query VM system metrics
* `/query` - query instant values
* `/query_range` - query over a range of time
* `/series` - match a certain label set
* `/label/{}/values` - query a list of label values (variables mostly)'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 29
id: 52
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_http_request_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job, path) > 0
format: time_series
intervalFactor: 1
legendFormat: '{{`{{`}}path{{`}}`}} ({{`{{`}}job{{`}}`}})'
range: true
refId: A
title: Requests error rate ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: bars
fillOpacity: 100
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=203&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 29
id: 104
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(rate(vm_log_messages_total{job=~"$job",instance=~"$instance", level!="info"}[$__rate_interval])) by (job, level) > 0
format: time_series
hide: false
interval: 5m
intervalFactor: 1
legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}level{{`}}`}}'
range: true
refId: A
title: Logging rate
type: timeseries
- collapsed: true
gridPos:
h: 1
w: 24
x: 0
'y': 37
id: 46
panels:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Percentage of used RSS memory (resident).
The RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).
The application''s performance will significantly degrade when memory usage is close to 100%.
Click on the line and choose Drilldown to show memory usage per instance'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2293
id: 66
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
max_over_time(process_resident_memory_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job", instance=~"$instance"}
) by(job)
interval: ''
legendFormat: __auto
range: true
refId: A
title: RSS memory % usage ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2293
id: 64
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
rate(process_cpu_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job", instance=~"$instance"}
) by(job)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
title: CPU ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
Safe memory usage % considered to be below 80%
Click on the line and choose Drilldown to show memory usage per instance'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2301
id: 138
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
max_over_time(process_resident_memory_anon_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job", instance=~"$instance"}
) by(job)
interval: ''
legendFormat: __auto
range: true
refId: A
title: RSS anonymous memory % usage ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
The lower the better.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: s
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2301
id: 219
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(rate(process_pressure_cpu_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
format: time_series
interval: ''
intervalFactor: 2
legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(rate(process_pressure_cpu_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
format: time_series
hide: false
interval: ''
intervalFactor: 2
legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
range: true
refId: B
title: CPU pressure
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
The lower the better.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: s
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2309
id: 220
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(rate(process_pressure_memory_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
format: time_series
interval: ''
intervalFactor: 2
legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(rate(process_pressure_memory_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
format: time_series
hide: false
interval: ''
intervalFactor: 2
legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
range: true
refId: B
title: Memory pressure
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the number of bytes read/write from the storage layer.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: Bps
overrides:
- matcher:
id: byRegexp
options: /read .*/
properties:
- id: custom.transform
value: negative-Y
gridPos:
h: 8
w: 12
x: 12
'y': 2309
id: 122
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(rate(process_io_storage_read_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: read {{`{{`}}job{{`}}`}}
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(process_io_storage_written_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: write {{`{{`}}job{{`}}`}}
range: true
refId: B
title: Disk writes/reads ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the percentage of open file descriptors compared to the limit set in the OS.
Reaching the limit of open files can cause various issues and must be prevented.
See how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 2
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: percentunit
overrides:
- matcher:
id: byRegexp
options: /max.*/
properties:
- id: color
value:
fixedColor: '#C4162A'
mode: fixed
gridPos:
h: 8
w: 12
x: 0
'y': 2317
id: 117
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
max_over_time(process_open_fds{job=~"$job", instance=~"$instance"}[$__rate_interval])
/
process_max_fds{job=~"$job", instance=~"$instance"}
) by(job)
format: time_series
interval: ''
intervalFactor: 2
legendFormat: '{{`{{`}}job{{`}}`}}'
range: true
refId: A
title: Open FDs usage % ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the number of read/write syscalls such as read, pread, write, pwrite.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides:
- matcher:
id: byRegexp
options: /read .*/
properties:
- id: custom.transform
value: negative-Y
gridPos:
h: 8
w: 12
x: 12
'y': 2317
id: 204
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(rate(process_io_read_syscalls_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: read {{`{{`}}job{{`}}`}}
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(process_io_write_syscalls_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: write {{`{{`}}job{{`}}`}}
range: true
refId: B
title: Disk write/read calls ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2325
id: 68
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(max_over_time(go_goroutines{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
format: time_series
interval: ''
intervalFactor: 2
legendFormat: __auto
range: true
refId: A
title: Goroutines ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
The lower the better.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: s
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2325
id: 221
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(rate(process_pressure_io_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
format: time_series
interval: ''
intervalFactor: 2
legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(rate(process_pressure_io_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job)
format: time_series
hide: false
interval: ''
intervalFactor: 2
legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
range: true
refId: B
title: IO pressure
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2333
id: 70
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(max_over_time(process_num_threads{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
format: time_series
interval: ''
intervalFactor: 2
legendFormat: __auto
range: true
refId: A
title: Threads ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2333
id: 119
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(max_over_time(vm_tcplistener_conns{job=~"$job", instance=~"$instance"}[$__interval])) by(job)
interval: ''
legendFormat: '{{`{{`}}job{{`}}`}}'
range: true
refId: A
title: TCP connections ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the percent of CPU spent on garbage collection.
If % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.
Try searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ '
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2341
id: 210
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)"
format: time_series
interval: ''
intervalFactor: 2
legendFormat: __auto
range: true
refId: A
title: CPU spent on GC ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2341
id: 120
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(rate(vm_tcplistener_accepts_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
interval: ''
legendFormat: __auto
range: true
refId: A
title: TCP connections rate ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: bytes
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2349
id: 218
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(rate(go_memstats_alloc_bytes_total{job=~"$job"}[$__rate_interval])) by (job)
format: time_series
interval: ''
intervalFactor: 2
legendFormat: __auto
range: true
refId: A
title: Memory allocations rate
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing."
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 0.1
unit: s
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2349
id: 213
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job, instance, le))) by(job)
format: time_series
interval: ''
intervalFactor: 2
legendFormat: __auto
range: true
refId: A
title: Go scheduling latency
type: timeseries
title: Resource usage ($job)
type: row
- collapsed: true
gridPos:
h: 1
w: 24
x: 0
'y': 38
id: 106
panels:
- description: ''
fieldConfig:
defaults: {}
overrides: []
gridPos:
h: 2
w: 24
x: 0
'y': 1519
id: 211
options:
code:
language: plaintext
showLineNumbers: false
showMiniMap: false
content: See [Troubleshooting](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) docs.
mode: markdown
pluginVersion: 11.5.0
title: ''
transparent: true
type: text
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the rate and total number of new series created over last 24h.
High [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM''s or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#cardinality) "explosions".
The higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.
To investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format
Good references to read:
* https://www.robustperception.io/cardinality-is-key
* https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides:
- matcher:
id: byName
options: new series over 24h
properties:
- id: custom.axisPlacement
value: right
- id: custom.axisSoftMin
value: 0
gridPos:
h: 8
w: 12
x: 0
'y': 1521
id: 102
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
exemplar: true
expr: sum(rate(vm_new_timeseries_created_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]))
interval: ''
legendFormat: churn rate
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
exemplar: true
expr: sum(increase(vm_new_timeseries_created_total{job=~"$job_storage", instance=~"$instance"}[24h]))
hide: false
interval: ''
legendFormat: new series over 24h
refId: B
title: Churn rate ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: "The percentage of [slow inserts](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \n\nThe lower the better. \n\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \n* Ingestion of completely new, not seen before time series;\n* [Re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\n* Not enough memory to maintain big enough caches for the current workload.\n\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series). \n\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details."
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line+area
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: transparent
value: null
- color: red
value: 0.1
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 1521
id: 108
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: "max(\n rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval]) \n / rate(vm_rows_added_to_storage_total{job=~\"$job_storage\"}[$__rate_interval])\n)"
interval: ''
legendFormat: slow inserts
range: true
refId: A
title: Slow inserts
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: bars
fillOpacity: 100
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 1529
id: 170
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(increase(vm_assisted_merges_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(type) > 0
format: time_series
interval: 5m
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
title: Assisted merges ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.
The less value is better.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
axisSoftMin: 0
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 1529
id: 107
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: |-
sum(rate(vm_slow_queries_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval]))
/
sum(rate(vm_http_requests_total{job=~"$job_select", instance=~"$instance", path=~"/select/.*"}[$__rate_interval]))
interval: ''
legendFormat: slow queries %
range: true
refId: A
title: Slow queries % ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized."
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: percentunit
overrides: []
gridPos:
h: 9
w: 12
x: 0
'y': 1537
id: 144
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: "max(\n vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} \n /\n vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(type)"
interval: ''
legendFormat: __auto
range: true
refId: A
title: Cache usage % by type ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows cache miss ratio. Lower is better.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
max: 1
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: percentunit
overrides: []
gridPos:
h: 9
w: 12
x: 12
'y': 1537
id: 58
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
rate(vm_cache_misses_total{job=~"$job", instance=~"$instance"}[$__rate_interval])
/
rate(vm_cache_requests_total{job=~"$job", instance=~"$instance"}[$__rate_interval])
) by(type)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
title: Cache miss ratio ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 1546
id: 142
links:
- targetBlank: true
title: Readonly mode
url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(vm_rpc_vmstorage_is_read_only{job=~"$job_insert", instance=~"$instance"}) by(instance, addr) > 0
format: time_series
interval: ''
intervalFactor: 1
legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}addr{{`}}`}}'
range: true
refId: A
title: Storage in readonly status for vminsert ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.
Should be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn''t set.
Deduplication happens in two places:
* `type="merge"` - during [background merges](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage) by vmstorages. Deduplication during merges is permanent.
* `type="select"` - during [read queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data) by vmselects. Deduplication happens in-flight and has no permanent effect.
The more samples need to be deduplicated, the higher will be resource usage on vmselects or vmstorages.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 1546
id: 215
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(rate(vm_deduplicated_samples_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type) >0
format: time_series
interval: ''
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
title: Deduplication rate ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows how many samples were ignored or dropped on insertion due to various reasons:
* timestamp out of retention period or timestamp in future;
* invalid metric name;
* exceeding limit for labels length or number;
* dropped by [relabeling configuration](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#relabeling).'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 1554
id: 135
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(increase(vm_rows_ignored_total{job=~"$job", instance=~"$instance"}[1h])) by (reason)
hide: false
interval: ''
legendFormat: __auto
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(increase(vm_relabel_metrics_dropped_total{job=~"$job", instance=~"$instance"}[1h]))
hide: false
instant: false
legendFormat: relabeling
range: true
refId: B
title: Samples dropped for last 1h ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the number of partial results served by `vmselects`.
`vmselect` continues serving queries if at least a single vmstorage nodes is available. It marks responses as `partial` for queries served from the remaining healthy vmstorage nodes.
If you prefer consistency over availability then run vmselect nodes with `-search.denyPartialResponse` cmd-line flag. In this case vmselect returns an error if at least a single vmstorage node is unavailable.
See more at [cluster availability](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability).'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
axisSoftMin: 0
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: stepAfter
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: none
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 1554
id: 217
options:
legend:
calcs:
- lastNotNull
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(increase(vm_partial_results_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) > 0
format: time_series
instant: false
legendFormat: partial results
refId: A
title: Partial query results ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: thresholds
custom:
align: auto
cellOptions:
type: auto
inspect: false
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
overrides:
- matcher:
id: byName
options: Value
properties:
- id: custom.hidden
value: true
- matcher:
id: byName
options: Time
properties:
- id: custom.hidden
value: true
gridPos:
h: 8
w: 12
x: 0
'y': 1562
id: 183
options:
cellHeight: sm
footer:
countRows: false
fields: ''
reducer:
- sum
show: false
showHeader: true
sortBy:
- desc: true
displayName: job
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: false
expr: sum(flag{is_set="true", job=~"$job", instance=~"$instance"}) by(job, instance, name, value)
format: table
instant: true
legendFormat: __auto
range: false
refId: A
title: Non-default flags
transformations:
- id: groupBy
options:
fields:
instance:
aggregations:
- uniqueValues
operation: aggregate
job:
aggregations: []
operation: groupby
name:
aggregations: []
operation: groupby
value:
aggregations: []
operation: groupby
type: table
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn''t restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. '
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
axisSoftMin: 0
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: stepAfter
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: none
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 1562
id: 214
options:
legend:
calcs:
- lastNotNull
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(changes(vm_app_start_timestamp{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
format: time_series
instant: false
legendFormat: '{{`{{`}}job{{`}}`}}'
refId: A
title: Restarts ($job)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: ${ds}
description: 'Shows the amount of data scheduled for [downsampling](https://docs.victoriametrics.com/#downsampling) or [retention filters](https://docs.victoriametrics.com/#retention-filters).
Applying downsampling or retention filters happens in background. `vmstorage` schedules this process periodically when finds data partitions that satisfy configured retention or downsampling rules.
This process could consume additional CPU and Disk IO resources.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
axisSoftMin: 0
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: auto
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
mappings: []
thresholds:
mode: absolute
steps:
- color: green
value: null
- color: red
value: 80
unit: bytes
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 1570
id: 222
options:
legend:
calcs: []
displayMode: list
placement: bottom
showLegend: true
tooltip:
hideZeros: false
mode: single
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: ${ds}
editorMode: code
expr: sum(vm_downsampling_partitions_scheduled_size_bytes{job=~"$job", instance=~"$instance"}) > 0
legendFormat: downsampling
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: ${ds}
editorMode: code
expr: sum(vm_retention_partitions_scheduled_size_bytes{job=~"$job", instance=~"$instance"}) > 0
hide: false
instant: false
legendFormat: retention filters
range: true
refId: B
title: Partitions scheduled for re-processing
type: timeseries
title: Troubleshooting
type: row
- collapsed: true
gridPos:
h: 1
w: 24
x: 0
'y': 39
id: 48
panels:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor."
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 9
w: 12
x: 0
'y': 4941
id: 76
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_rpc_rows_pushed_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
format: time_series
intervalFactor: 1
legendFormat: Pushed
range: true
refId: B
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_rpc_rows_sent_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
format: time_series
intervalFactor: 1
legendFormat: Sent
range: true
refId: E
title: Rows ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Rate of RPC errors by type:
* `Connection` - the number of connection errors to vmstorage node
* `Dial` - the number of dial errors to vmstorage node.
* `Handshake` - the number of handshake errors to vmstorage node
* `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 9
w: 12
x: 12
'y': 4941
id: 86
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_rpc_connection_errors_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
format: time_series
intervalFactor: 1
legendFormat: Connection
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
expr: sum(rate(vm_rpc_dial_errors_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
format: time_series
intervalFactor: 1
legendFormat: Dial
refId: B
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
expr: sum(rate(vm_rpc_handshake_errors_total{job=~"$job",instance=~"$instance"}[$__rate_interval]))
format: time_series
intervalFactor: 1
legendFormat: Handshake
refId: E
title: RPC errors ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: rps
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 4950
id: 80
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(addr) > 0
format: time_series
interval: ''
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
title: 'Rows ($instance) rerouted to '
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: rps
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 4950
id: 78
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
exemplar: true
expr: sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(addr) > 0
format: time_series
interval: ''
intervalFactor: 1
legendFormat: '{{`{{`}}addr{{`}}`}}'
refId: A
title: Rows ($instance) rerouted from
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: The number of rows or bytes that vminsert internal buffer contains at the moment.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides:
- matcher:
id: byName
options: bytes
properties:
- id: unit
value: bytes
gridPos:
h: 8
w: 12
x: 0
'y': 4958
id: 82
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
expr: sum(vm_rpc_buf_pending_bytes{job=~"$job", instance=~"$instance"})
legendFormat: bytes
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
expr: sum(vm_rpc_rows_pending{job=~"$job", instance=~"$instance"})
legendFormat: rows
refId: B
title: Pending
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Network usage by internal VictoriaMetrics RPC protocol
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: bps
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 4958
id: 74
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcpdialer_written_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) * 8
legendFormat: network usage
range: true
refId: A
title: RPC network usage ($instance)
type: timeseries
title: Interconnection ($job)
type: row
- collapsed: true
gridPos:
h: 1
w: 24
x: 0
'y': 40
id: 24
panels:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the number of datapoints ingested into storage nodes per second. This metric doesn't show all stored datapoints since some of them may be dropped because of wrong timestamps or decode errors.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: normal
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2387
id: 100
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_vminsert_metrics_read_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]))
format: time_series
intervalFactor: 1
legendFormat: ingestion rate
range: true
refId: A
title: Ingestion rate ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
Safe memory usage % considered to be below 80%
Click on the line and choose Drilldown to show memory usage per instance'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 0.9
unit: percentunit
overrides: []
gridPos:
h: 7
w: 12
x: 12
'y': 2387
id: 167
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
max_over_time(process_resident_memory_anon_bytes{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job_storage", instance=~"$instance"}
)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: max
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
min(
max_over_time(process_resident_memory_anon_bytes{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job_storage", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: min
range: true
refId: B
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
quantile(0.5,
max_over_time(process_resident_memory_anon_bytes{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job_storage", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: median
range: true
refId: C
title: Memory (anon) usage % ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the number of concurrently executed read requests, where:
* `max` - equal to `-search.maxConcurrentRequest` cmd-line flag;
* `current` - current number of concurrent select requests executed by storage.
Each concurrent select request requires RAM, CPU and disk IO resources during processing. Too many concurrent requests could result into OOM exceptions.
When `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).
See more at https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#resource-usage-limits'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides:
- matcher:
id: byName
options: max
properties:
- id: color
value:
fixedColor: '#C4162A'
mode: fixed
gridPos:
h: 8
w: 12
x: 12
'y': 2394
id: 133
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: "max(\n max_over_time(vm_vmselect_concurrent_requests_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)"
interval: ''
legendFormat: current
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: min(vm_vmselect_concurrent_requests_capacity{job=~"$job_storage", instance=~"$instance"})
hide: false
interval: ''
legendFormat: max
range: true
refId: B
title: Concurrent selects ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: ''
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 0.9
unit: percentunit
overrides: []
gridPos:
h: 7
w: 12
x: 0
'y': 2395
id: 151
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: false
expr: |-
max(
rate(process_cpu_seconds_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job_storage", instance=~"$instance"}
)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: max
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: false
expr: |-
min(
rate(process_cpu_seconds_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job_storage", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: min
range: true
refId: B
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: false
expr: |-
quantile(0.5,
rate(process_cpu_seconds_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job_storage", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: median
range: true
refId: C
title: CPU usage % ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row)."
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides:
- matcher:
id: byName
options: max
properties:
- id: color
value:
fixedColor: '#C4162A'
mode: fixed
gridPos:
h: 8
w: 12
x: 0
'y': 2402
id: 212
links:
- targetBlank: true
title: Related discussion
url: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: "max(\n max_over_time(vm_concurrent_insert_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)"
interval: ''
legendFormat: current
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: min(vm_concurrent_insert_capacity{job=~"$job_storage", instance=~"$instance"})
hide: false
interval: ''
legendFormat: max
range: true
refId: B
title: Concurrent flushes on disk ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: The number of rows merged per second by storage nodes. Merge speed depends on available CPU and disk IO bandwidth.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2402
id: 55
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_rows_merged_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(type)
legendFormat: __auto
range: true
refId: A
title: Merge speed
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: "The max number of on-going merges across storage nodes.\n The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.\n It is expected to have high numbers for `storage/small` metric."
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2410
id: 54
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(max_over_time(vm_active_merges{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(type)
legendFormat: __auto
range: true
refId: A
title: Active merges ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage).
Increase in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.
* `indexdb` - inverted index
* `storage/small` - recently added parts of data ingested into storage (hot data)
* `storage/big` - small parts gradually merged into bigger parts (cold data)'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2410
id: 22
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(vm_parts{job=~"$job_storage", instance=~"$instance", type=~"indexdb.*"}) by(type)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(vm_last_partition_parts{job=~"$job_storage", instance=~"$instance"}) by(type)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: __auto
range: true
refId: B
title: LSM parts max by type ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=200&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2418
id: 20
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)"
format: time_series
intervalFactor: 1
legendFormat: max
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: "min(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)"
format: time_series
hide: false
intervalFactor: 1
legendFormat: min
range: true
refId: B
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: "quantile(0.5,\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)"
format: time_series
hide: false
intervalFactor: 1
legendFormat: median
range: true
refId: C
title: Disk space usage % ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides:
- matcher:
id: byName
options: pending index entries
properties:
- id: unit
value: none
- id: decimals
value: 3
gridPos:
h: 8
w: 12
x: 12
'y': 2418
id: 14
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: none
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(max_over_time(vm_pending_rows{job=~"$job_storage", instance=~"$instance", type="storage"}[$__rate_interval]))
format: time_series
hide: false
intervalFactor: 1
legendFormat: pending datapoints
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(max_over_time(vm_pending_rows{job=~"$job_storage", instance=~"$instance", type="indexdb"}[$__rate_interval]))
format: time_series
hide: false
intervalFactor: 1
legendFormat: pending index entries
range: true
refId: B
title: Pending datapoints ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly.
The sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=201&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 2426
id: 202
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)"
format: time_series
intervalFactor: 1
legendFormat: indexdb
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)"
format: time_series
hide: false
intervalFactor: 1
legendFormat: datapoints
range: true
refId: B
title: Disk space usage % by type ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Value > 0 means vmstorage is in readonly mode.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2426
id: 141
links:
- targetBlank: true
title: Readonly mode
url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode
options:
legend:
calcs: []
displayMode: list
placement: bottom
showLegend: false
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: vm_storage_is_read_only{job=~"$job_storage", instance=~"$instance"} > 0
interval: ''
legendFormat: '{{`{{`}} instance {{`}}`}}'
range: true
refId: A
title: Readonly mode
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows network usage by vmstorage services.
* Writes show traffic sent to clients.
* Reads show traffic received from clients.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: bps
overrides:
- matcher:
id: byRegexp
options: /read.*/
properties:
- id: custom.transform
value: negative-Y
gridPos:
h: 8
w: 12
x: 0
'y': 2434
id: 206
options:
legend:
calcs:
- mean
- lastNotNull
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcplistener_read_bytes_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(name) * 8 > 0
format: time_series
intervalFactor: 1
legendFormat: read from {{`{{`}}name{{`}}`}}
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcplistener_written_bytes_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(name) * 8 > 0
format: time_series
hide: false
intervalFactor: 1
legendFormat: write to {{`{{`}}name{{`}}`}}
range: true
refId: B
title: Network usage ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the max number of existing [snapshots](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots) across vmstorages.
Recently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.
It is recommended deleting old snapshots when they are no longer needed to free up disk space.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 2434
id: 216
options:
legend:
calcs: []
displayMode: list
placement: bottom
showLegend: false
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: max(vm_snapshots{job=~"$job_storage", instance=~"$instance"})
interval: ''
legendFormat: snapshots
range: true
refId: A
title: Number of snapshots
type: timeseries
title: vmstorage ($instance)
type: row
- collapsed: true
gridPos:
h: 1
w: 24
x: 0
'y': 41
id: 42
panels:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Request rate accepted by vmselect nodes
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 4954
id: 92
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_http_requests_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (path) > 0
format: time_series
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
title: Requests rate ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the max number of concurrent selects across instances.
* `max` limit can be configured via `search.maxConcurrentRequests` flag
* `current` shows the current number of goroutines busy with processing requests
When `current` hits `max` constantly, it means one or more vmselect nodes are overloaded with number of requests. If you observe that CPU for vmselects is saturated, consider adding more vmselect replicas or increase CPU resources. If CPU and Memory panels show a plenty of free resources - try increasing `-search.maxConcurrentRequests`. Please note, the higher is `-search.maxConcurrentRequests`, the higher could be [peak memory usage](https://docs.victoriametrics.com/victoriametrics/troubleshooting/#out-of-memory-errors).'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides:
- matcher:
id: byName
options: max
properties:
- id: color
value:
fixedColor: '#C4162A'
mode: fixed
- id: custom.fillOpacity
value: 0
gridPos:
h: 8
w: 12
x: 12
'y': 4954
id: 95
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(max_over_time(vm_concurrent_select_current{job=~"$job_select", instance=~"$instance"}[$__rate_interval]))
format: time_series
interval: ''
intervalFactor: 1
legendFormat: current
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
expr: min(vm_concurrent_select_capacity{job=~"$job_select", instance=~"$instance"})
format: time_series
intervalFactor: 1
legendFormat: max
refId: B
title: Concurrent selects ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 0.9
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 4962
id: 163
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
rate(process_cpu_seconds_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job_select", instance=~"$instance"}
)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: max
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
min(
rate(process_cpu_seconds_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job_select", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: min
range: true
refId: B
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
quantile(0.5,
rate(process_cpu_seconds_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job_select", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: median
range: true
refId: C
title: CPU usage % ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
Safe memory usage % considered to be below 80%
Click on the line and choose Drilldown to show memory usage per instance'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 0.9
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 4962
id: 165
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
max_over_time(process_resident_memory_anon_bytes{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job_select", instance=~"$instance"}
)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: max
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
min(
max_over_time(process_resident_memory_anon_bytes{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job_select", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: min
range: true
refId: B
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
quantile(0.5,
max_over_time(process_resident_memory_anon_bytes{job=~"$job_select", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job_select", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: median
range: true
refId: C
title: Memory (anon) usage % ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 99th percentile of the number of time series read per query.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 2
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 4970
id: 178
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
format: time_series
interval: ''
intervalFactor: 1
legendFormat: series
range: true
refId: A
title: Series read per query ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per queried time series.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 2
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 4970
id: 180
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
format: time_series
interval: ''
intervalFactor: 1
legendFormat: datapoints
range: true
refId: A
title: Datapoints read per series ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per query.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 2
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 4978
id: 179
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
format: time_series
interval: ''
intervalFactor: 1
legendFormat: datapoints
range: true
refId: A
title: Datapoints read per query ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: '99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scanner per query.
This number can exceed number of DatapointsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 2
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 4978
id: 181
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
format: time_series
interval: ''
intervalFactor: 1
legendFormat: datapoints
range: true
refId: A
title: Datapoints scanned per query ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: bps
overrides:
- matcher:
id: byRegexp
options: /read.*/
properties:
- id: custom.transform
value: negative-Y
gridPos:
h: 8
w: 12
x: 0
'y': 4986
id: 93
options:
legend:
calcs:
- mean
- lastNotNull
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcplistener_read_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0
format: time_series
intervalFactor: 1
legendFormat: read from client
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcplistener_written_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0
format: time_series
hide: false
intervalFactor: 1
legendFormat: write to client
range: true
refId: B
title: 'Network usage: clients ($instance)'
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows network usage between vmselects and vmstorages.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: bps
overrides:
- matcher:
id: byRegexp
options: /read.*/
properties:
- id: custom.transform
value: negative-Y
gridPos:
h: 8
w: 12
x: 12
'y': 4986
id: 207
options:
legend:
calcs:
- mean
- lastNotNull
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcpdialer_read_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0
format: time_series
intervalFactor: 1
legendFormat: read from vmstorage
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcpdialer_written_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0
format: time_series
hide: false
intervalFactor: 1
legendFormat: write to vmstorage
range: true
refId: B
title: 'Network usage: vmstorage ($instance)'
type: timeseries
title: vmselect ($instance)
type: row
- collapsed: true
gridPos:
h: 1
w: 24
x: 0
'y': 42
id: 40
panels:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: '* `*` - unsupported query path
* `/write` - insert into VM
* `/metrics` - query VM system metrics
* `/query` - query instant values
* `/query_range` - query over a range of time
* `/series` - match a certain label set
* `/label/{}/values` - query a list of label values (variables mostly)'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 4955
id: 97
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_http_requests_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) by (path) > 0
format: time_series
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
title: Requests rate ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the max number of ongoing insertions.
* `max` - equal to number of CPU * 2 by default. May be configured with `maxConcurrentInserts` flag;
* `current` - current number of goroutines busy with processing requests.
`-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.
When `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 10
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides:
- matcher:
id: byName
options: max
properties:
- id: color
value:
fixedColor: '#C4162A'
mode: fixed
- id: custom.fillOpacity
value: 0
gridPos:
h: 8
w: 12
x: 12
'y': 4955
id: 99
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(max_over_time(vm_concurrent_insert_current{job=~"$job_insert", instance=~"$instance"}[$__rate_interval]))
format: time_series
interval: ''
intervalFactor: 1
legendFormat: current
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
exemplar: true
expr: min(vm_concurrent_insert_capacity{job=~"$job_insert", instance=~"$instance"})
format: time_series
interval: ''
intervalFactor: 1
legendFormat: max
refId: B
title: Concurrent inserts ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 0.9
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 4963
id: 185
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
rate(process_cpu_seconds_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job_insert", instance=~"$instance"}
)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: max
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
min(
rate(process_cpu_seconds_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job_insert", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: min
range: true
refId: B
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
quantile(0.5,
rate(process_cpu_seconds_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
/
process_cpu_cores_available{job=~"$job_insert", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: median
range: true
refId: C
title: CPU usage % ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
Safe memory usage % considered to be below 80%
Click on the line and choose Drilldown to show memory usage per instance'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links:
- targetBlank: true
title: Drilldown
url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 0.9
unit: percentunit
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 4963
id: 187
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
max(
max_over_time(process_resident_memory_anon_bytes{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job_insert", instance=~"$instance"}
)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: max
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
min(
max_over_time(process_resident_memory_anon_bytes{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job_insert", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: min
range: true
refId: B
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: |-
quantile(0.5,
max_over_time(process_resident_memory_anon_bytes{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])
/
vm_available_memory_bytes{job=~"$job_insert", instance=~"$instance"}
)
format: time_series
hide: false
interval: ''
intervalFactor: 1
legendFormat: median
range: true
refId: C
title: Memory (anon) usage % ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: "Shows the saturation level of connection between vminsert and vmstorage components. \n\nIf the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that either vminsert or vmstorage nodes are struggling with the load. Verify CPU/mem saturation of both components and network saturation between them.\nIf vminsert resources are saturated - consider adding more resources or scale vminserts horizontally.\n\nIf vminsert resources and network are fine, check vmstorage metrics for anomalies."
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line+area
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: transparent
- color: red
value: 0.9
unit: s
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 4971
id: 139
options:
legend:
calcs:
- mean
- lastNotNull
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: max(rate(vm_rpc_send_duration_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(addr)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
title: Storage connection saturation ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows when vmstorage node is unreachable for vminsert.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 0
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 4971
id: 114
options:
legend:
calcs:
- mean
- lastNotNull
displayMode: table
placement: bottom
showLegend: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: vm_rpc_vmstorage_is_reachable{job=~"$job", instance=~"$instance"} != 1
format: time_series
interval: ''
intervalFactor: 1
legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}addr{{`}}`}}'
range: true
refId: A
title: Storage reachability ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows network usage between vminserts and clients, such as vmagent, Prometheus, or any other client pushing metrics to vminsert.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: bps
overrides:
- matcher:
id: byRegexp
options: /read.*/
properties:
- id: custom.transform
value: negative-Y
gridPos:
h: 8
w: 12
x: 0
'y': 4979
id: 208
options:
legend:
calcs:
- mean
- lastNotNull
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcplistener_read_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0
format: time_series
intervalFactor: 1
legendFormat: read from client
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcplistener_written_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0
format: time_series
hide: false
intervalFactor: 1
legendFormat: write to client
range: true
refId: B
title: 'Network usage: clients ($instance)'
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows network usage between vminserts and vmstorages.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: bps
overrides:
- matcher:
id: byRegexp
options: /read.*/
properties:
- id: custom.transform
value: negative-Y
gridPos:
h: 8
w: 12
x: 12
'y': 4979
id: 209
options:
legend:
calcs:
- mean
- lastNotNull
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcpdialer_read_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0
format: time_series
intervalFactor: 1
legendFormat: read from vmstorage
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(rate(vm_tcpdialer_written_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0
format: time_series
hide: false
intervalFactor: 1
legendFormat: write to vmstorage
range: true
refId: B
title: 'Network usage: vmstorage ($instance)'
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: ''
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
decimals: 2
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 4987
id: 88
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange)))
format: time_series
interval: ''
intervalFactor: 1
legendFormat: max
range: true
refId: A
title: Rows per insert ($instance)
type: timeseries
title: vminsert ($instance)
type: row
- collapsed: true
gridPos:
h: 1
w: 24
x: 0
'y': 43
id: 194
panels:
- fieldConfig:
defaults: {}
overrides: []
gridPos:
h: 2
w: 24
x: 0
'y': 5004
id: 198
options:
code:
language: plaintext
showLineNumbers: false
showMiniMap: false
content: Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.
mode: markdown
pluginVersion: 9.2.7
title: ''
transparent: true
type: text
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the used memory (resident).
The application''s performance will significantly degrade when memory usage is close to 100%.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: decbytes
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 5006
id: 189
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: max_over_time(process_resident_memory_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
interval: ''
legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
range: true
refId: A
title: RSS memory usage ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: decbytes
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 5006
id: 190
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: max_over_time(process_resident_memory_anon_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
interval: ''
legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
range: true
refId: A
title: RSS anonymous memory usage ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: ''
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: short
overrides: []
gridPos:
h: 7
w: 12
x: 0
'y': 5014
id: 192
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: false
expr: sum(rate(process_cpu_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, instance)
format: time_series
interval: ''
intervalFactor: 1
legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
range: true
refId: A
title: CPU usage ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: 'Shows the approx time needed to reach 100% of disk capacity based on the following params:
* free disk space (after -storage.minFreeDiskSpaceBytes);
* row ingestion rate;
* compression.'
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: s
overrides: []
gridPos:
h: 7
w: 12
x: 12
'y': 5014
id: 196
options:
legend:
calcs:
- mean
- lastNotNull
- min
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: none
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: "(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n (rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) - \n sum(rate(vm_deduplicated_samples_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])) without (type)) * \n (\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type) /\n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type)\n )\n +\n rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) * \n (\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"}) /\n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"})\n )\n) > 0"
format: time_series
interval: ''
intervalFactor: 1
legendFormat: '{{`{{`}}instance{{`}}`}}'
range: true
refId: A
title: Storage full ETA ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisBorderShow: false
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
barWidthFactor: 0.6
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
insertNulls: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: decbytes
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 5021
id: 200
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
hideZeros: false
mode: multi
sort: desc
pluginVersion: 11.5.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(vm_data_size_bytes{job=~"$job", instance=~"$instance"}) by(job, instance)
format: time_series
intervalFactor: 1
legendFormat: ''
range: true
refId: A
title: Disk space usage ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: ''
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: line
fillOpacity: 0
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: line
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
unit: decbytes
overrides: []
gridPos:
h: 8
w: 12
x: 12
'y': 5021
id: 201
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(vm_data_size_bytes{job=~"$job", instance=~"$instance", type=~"indexdb.*"}) by(job, instance)
format: time_series
intervalFactor: 1
legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (indexdb)'
range: true
refId: A
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
expr: sum(vm_data_size_bytes{job=~"$job", instance=~"$instance", type!~"indexdb.*"}) by(job, instance)
format: time_series
hide: false
intervalFactor: 1
legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (datapoints)'
range: true
refId: B
title: Disk space usage by type ($instance)
type: timeseries
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
fieldConfig:
defaults:
color:
mode: palette-classic
custom:
axisCenteredZero: false
axisColorMode: text
axisLabel: ''
axisPlacement: auto
barAlignment: 0
drawStyle: bars
fillOpacity: 100
gradientMode: none
hideFrom:
legend: false
tooltip: false
viz: false
lineInterpolation: linear
lineWidth: 1
pointSize: 5
scaleDistribution:
type: linear
showPoints: never
spanNulls: false
stacking:
group: A
mode: none
thresholdsStyle:
mode: 'off'
links: []
mappings: []
min: 0
thresholds:
mode: absolute
steps:
- color: green
- color: red
value: 80
unit: short
overrides: []
gridPos:
h: 8
w: 12
x: 0
'y': 5029
id: 203
options:
legend:
calcs:
- mean
- lastNotNull
- max
displayMode: table
placement: bottom
showLegend: true
sortBy: Last *
sortDesc: true
tooltip:
mode: multi
sort: desc
pluginVersion: 9.1.0
targets:
- datasource:
type: {{ $defaultDatasource }}
uid: $ds
editorMode: code
exemplar: true
expr: sum(rate(vm_log_messages_total{job=~"$job",instance=~"$instance", level!="info"}[$__rate_interval])) by (job, instance, level, location) > 0
format: time_series
hide: false
interval: 5m
intervalFactor: 1
legendFormat: __auto
range: true
refId: A
title: Logging rate
type: timeseries
title: Drilldown
type: row
preload: false
refresh: ''
schemaVersion: 40
tags:
- victoriametrics
- vm-k8s-stack
templating:
list:
- current:
text: VictoriaMetrics - cluster
value: PAF93674D0B4E9963
includeAll: false
name: ds
options: []
query: {{ $defaultDatasource }}
refresh: 1
regex: ''
type: datasource
- current: {}
datasource:
type: prometheus
uid: $ds
definition: label_values(vm_app_version{version=~"^vm(insert|select|storage).*"}, job)
includeAll: true
multi: true
name: job
options: []
query:
query: label_values(vm_app_version{version=~"^vm(insert|select|storage).*"}, job)
refId: VictoriaMetrics-job-Variable-Query
refresh: 1
regex: ''
type: query
- current: {}
datasource:
type: prometheus
uid: $ds
definition: label_values(vm_app_version{job=~"$job", version=~"^vminsert.*"}, job)
hide: 2
includeAll: true
multi: true
name: job_insert
options: []
query:
query: label_values(vm_app_version{job=~"$job", version=~"^vminsert.*"}, job)
refId: StandardVariableQuery
refresh: 1
regex: ''
type: query
- current: {}
datasource:
type: prometheus
uid: $ds
definition: label_values(vm_app_version{job=~"$job", version=~"^vmselect.*"}, job)
hide: 2
includeAll: true
multi: true
name: job_select
options: []
query:
query: label_values(vm_app_version{job=~"$job", version=~"^vmselect.*"}, job)
refId: StandardVariableQuery
refresh: 1
regex: ''
type: query
- current: {}
datasource:
type: prometheus
uid: $ds
definition: label_values(vm_app_version{job=~"$job", version=~"^vmstorage.*"}, job)
hide: 2
includeAll: true
multi: true
name: job_storage
options: []
query:
query: label_values(vm_app_version{job=~"$job", version=~"^vmstorage.*"}, job)
refId: StandardVariableQuery
refresh: 1
regex: ''
type: query
- allValue: .*
current: {}
datasource:
type: prometheus
uid: $ds
definition: label_values(vm_app_version{job=~"$job"}, instance)
includeAll: true
multi: true
name: instance
options: []
query:
query: label_values(vm_app_version{job=~"$job"}, instance)
refId: VictoriaMetrics-instance-Variable-Query
refresh: 1
regex: ''
type: query
- baseFilters: []
datasource:
type: prometheus
uid: $ds
filters: []
name: adhoc
type: adhoc
time:
from: now-3h
to: now
timepicker:
refresh_intervals:
- 10s
- 30s
- 1m
- 5m
- 15m
- 30m
- 1h
- 2h
- 1d
timezone: {{ default "utc" ($Values.defaultDashboards).defaultTimezone }}
title: VictoriaMetrics - cluster
uid: oS7Bi_0Wz
version: 1
weekStart: ''
Reference in New Issue View Git Blame Copy Permalink