6277 lines
155 KiB
YAML
6277 lines
155 KiB
YAML
{{- $Values := (.helm).Values | default .Values }}
|
|
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
|
|
{{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }}
|
|
{{- $defaultDatasource := "prometheus" -}}
|
|
{{- range (((($Values.grafana).sidecar).datasources).victoriametrics | default list) }}
|
|
{{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }}
|
|
{{- end }}
|
|
annotations:
|
|
list:
|
|
- builtIn: 1
|
|
datasource:
|
|
type: datasource
|
|
uid: grafana
|
|
enable: true
|
|
hide: true
|
|
iconColor: rgba(0, 211, 255, 1)
|
|
name: Annotations & Alerts
|
|
target:
|
|
limit: 100
|
|
matchAny: false
|
|
tags: []
|
|
type: dashboard
|
|
type: dashboard
|
|
- datasource:
|
|
type: prometheus
|
|
uid: $ds
|
|
enable: true
|
|
expr: sum(vm_app_version{job=~"$job", instance=~"$instance"}) by(short_version) unless (sum(vm_app_version{job=~"$job", instance=~"$instance"} offset $__interval) by(short_version))
|
|
hide: true
|
|
iconColor: dark-blue
|
|
name: version
|
|
textFormat: '{{`{{`}}short_version{{`}}`}}'
|
|
titleFormat: Version change
|
|
- datasource:
|
|
type: prometheus
|
|
uid: $ds
|
|
enable: true
|
|
expr: sum(changes(vm_app_start_timestamp{job=~"$job", instance=~"$instance"}[$__interval])) by(job, instance)
|
|
hide: false
|
|
iconColor: dark-yellow
|
|
name: restarts
|
|
textFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} restarted'
|
|
condition: {{ true }}
|
|
description: Overview for VictoriaMetrics vmagent v1.117.0 or higher
|
|
editable: false
|
|
fiscalYearStartMonth: 0
|
|
graphTooltip: 1
|
|
id: 2
|
|
links:
|
|
- icon: doc
|
|
tags: []
|
|
targetBlank: true
|
|
title: vmagent wiki
|
|
tooltip: ''
|
|
type: link
|
|
url: https://docs.victoriametrics.com/victoriametrics/vmagent/
|
|
- icon: external link
|
|
tags: []
|
|
targetBlank: true
|
|
title: Found a bug?
|
|
type: link
|
|
url: https://github.com/VictoriaMetrics/VictoriaMetrics/issues
|
|
- icon: external link
|
|
tags: []
|
|
targetBlank: true
|
|
title: New releases
|
|
type: link
|
|
url: https://github.com/VictoriaMetrics/VictoriaMetrics/releases
|
|
- asDropdown: false
|
|
icon: external link
|
|
includeVars: false
|
|
keepTime: false
|
|
tags: []
|
|
targetBlank: true
|
|
title: Troubleshooting
|
|
tooltip: ''
|
|
type: link
|
|
url: https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
|
|
panels:
|
|
- collapsed: false
|
|
gridPos:
|
|
h: 1
|
|
w: 24
|
|
x: 0
|
|
'y': 0
|
|
id: 105
|
|
panels: []
|
|
title: Stats
|
|
type: row
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the rate of [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scraped from configured targets.
|
|
fieldConfig:
|
|
defaults:
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
overrides: []
|
|
gridPos:
|
|
h: 3
|
|
w: 6
|
|
x: 0
|
|
'y': 1
|
|
id: 103
|
|
options:
|
|
colorMode: value
|
|
graphMode: area
|
|
justifyMode: auto
|
|
orientation: auto
|
|
percentChangeColorMode: standard
|
|
reduceOptions:
|
|
calcs:
|
|
- last
|
|
fields: ''
|
|
values: false
|
|
showPercentChange: false
|
|
text: {}
|
|
textMode: auto
|
|
wideLayout: true
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_promscrape_scraped_samples_sum{job=~"$job", instance=~"$instance"}[$__rate_interval]))
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Samples scraped/s
|
|
type: stat
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the number of targets scraped per second.
|
|
fieldConfig:
|
|
defaults:
|
|
decimals: 1
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
overrides: []
|
|
gridPos:
|
|
h: 3
|
|
w: 6
|
|
x: 6
|
|
'y': 1
|
|
id: 134
|
|
options:
|
|
colorMode: value
|
|
graphMode: none
|
|
justifyMode: auto
|
|
orientation: auto
|
|
percentChangeColorMode: standard
|
|
reduceOptions:
|
|
calcs:
|
|
- last
|
|
fields: ''
|
|
values: false
|
|
showPercentChange: false
|
|
text: {}
|
|
textMode: auto
|
|
wideLayout: true
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_promscrape_scrapes_total{job=~"$job", instance=~"$instance"}[$__rate_interval]))
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Targets scraped/s
|
|
type: stat
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows number of generated error messages in logs over last 30m. Non-zero value may be a sign of connectivity or misconfiguration errors.
|
|
fieldConfig:
|
|
defaults:
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 1
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 3
|
|
w: 6
|
|
x: 12
|
|
'y': 1
|
|
id: 16
|
|
links:
|
|
- targetBlank: true
|
|
title: Troubleshooting
|
|
url: https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
|
|
options:
|
|
colorMode: value
|
|
graphMode: area
|
|
justifyMode: auto
|
|
orientation: auto
|
|
percentChangeColorMode: standard
|
|
reduceOptions:
|
|
calcs:
|
|
- last
|
|
fields: ''
|
|
values: false
|
|
showPercentChange: false
|
|
text: {}
|
|
textMode: auto
|
|
wideLayout: true
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
expr: sum(increase(vm_log_messages_total{job=~"$job", instance=~"$instance", level!="info"}[30m]))
|
|
interval: ''
|
|
legendFormat: ''
|
|
refId: A
|
|
title: Log errors (30m)
|
|
type: stat
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Total number of available CPUs for selected vmagents. '
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: thresholds
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 3
|
|
w: 6
|
|
x: 18
|
|
'y': 1
|
|
id: 152
|
|
maxDataPoints: 100
|
|
options:
|
|
colorMode: value
|
|
graphMode: area
|
|
justifyMode: auto
|
|
orientation: horizontal
|
|
percentChangeColorMode: standard
|
|
reduceOptions:
|
|
calcs:
|
|
- lastNotNull
|
|
fields: ''
|
|
values: false
|
|
showPercentChange: false
|
|
text: {}
|
|
textMode: auto
|
|
wideLayout: true
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(vm_available_cpu_cores{job=~"$job", instance=~"$instance"})
|
|
format: time_series
|
|
instant: true
|
|
interval: ''
|
|
intervalFactor: 1
|
|
legendFormat: ''
|
|
refId: A
|
|
title: Available CPU
|
|
type: stat
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the rate of ingested [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
|
|
fieldConfig:
|
|
defaults:
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
overrides: []
|
|
gridPos:
|
|
h: 3
|
|
w: 6
|
|
x: 0
|
|
'y': 4
|
|
id: 102
|
|
options:
|
|
colorMode: value
|
|
graphMode: area
|
|
justifyMode: auto
|
|
orientation: auto
|
|
percentChangeColorMode: standard
|
|
reduceOptions:
|
|
calcs:
|
|
- last
|
|
fields: ''
|
|
values: false
|
|
showPercentChange: false
|
|
text: {}
|
|
textMode: auto
|
|
wideLayout: true
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vmagent_rows_inserted_total{job=~"$job", instance=~"$instance"}[$__rate_interval]))
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Samples ingested/s
|
|
type: stat
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows total number of all configured scrape targets in state `up` or `down`.\n\nSee `http://vmagent-host:8429/targets` to get list of all targets. \n"
|
|
fieldConfig:
|
|
defaults:
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
overrides:
|
|
- matcher:
|
|
id: byName
|
|
options: down
|
|
properties:
|
|
- id: thresholds
|
|
value:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 1
|
|
gridPos:
|
|
h: 3
|
|
w: 6
|
|
x: 6
|
|
'y': 4
|
|
id: 72
|
|
options:
|
|
colorMode: value
|
|
graphMode: area
|
|
justifyMode: auto
|
|
orientation: auto
|
|
percentChangeColorMode: standard
|
|
reduceOptions:
|
|
calcs:
|
|
- last
|
|
fields: ''
|
|
values: false
|
|
showPercentChange: false
|
|
text: {}
|
|
textMode: auto
|
|
wideLayout: true
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(vm_promscrape_targets{job=~"$job", instance=~"$instance", status="up"})
|
|
interval: ''
|
|
legendFormat: up
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(vm_promscrape_targets{job=~"$job", instance=~"$instance", status="down"})
|
|
hide: false
|
|
interval: ''
|
|
legendFormat: down
|
|
range: true
|
|
refId: B
|
|
title: Scrape targets
|
|
type: stat
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Persistent queue size shows size of pending samples in bytes which hasn't been flushed to remote storage yet. \nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored."
|
|
fieldConfig:
|
|
defaults:
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 10485760
|
|
unit: bytes
|
|
overrides: []
|
|
gridPos:
|
|
h: 3
|
|
w: 6
|
|
x: 12
|
|
'y': 4
|
|
id: 56
|
|
options:
|
|
colorMode: value
|
|
graphMode: area
|
|
justifyMode: auto
|
|
orientation: auto
|
|
percentChangeColorMode: standard
|
|
reduceOptions:
|
|
calcs:
|
|
- last
|
|
fields: ''
|
|
values: false
|
|
showPercentChange: false
|
|
text: {}
|
|
textMode: auto
|
|
wideLayout: true
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
expr: sum(vmagent_remotewrite_pending_data_bytes{job=~"$job", instance=~"$instance"})
|
|
interval: ''
|
|
legendFormat: ''
|
|
refId: A
|
|
title: Persistent queue size
|
|
type: stat
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Total size of available memory for selected vmagents.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: thresholds
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
unit: bytes
|
|
overrides: []
|
|
gridPos:
|
|
h: 3
|
|
w: 6
|
|
x: 18
|
|
'y': 4
|
|
id: 153
|
|
maxDataPoints: 100
|
|
options:
|
|
colorMode: value
|
|
graphMode: area
|
|
justifyMode: auto
|
|
orientation: horizontal
|
|
percentChangeColorMode: standard
|
|
reduceOptions:
|
|
calcs:
|
|
- lastNotNull
|
|
fields: ''
|
|
values: false
|
|
showPercentChange: false
|
|
text: {}
|
|
textMode: auto
|
|
wideLayout: true
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(vm_available_memory_bytes{job=~"$job", instance=~"$instance"})
|
|
format: time_series
|
|
instant: true
|
|
interval: ''
|
|
intervalFactor: 1
|
|
legendFormat: ''
|
|
refId: A
|
|
title: Available memory
|
|
type: stat
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: thresholds
|
|
custom:
|
|
align: auto
|
|
cellOptions:
|
|
type: auto
|
|
inspect: false
|
|
minWidth: 50
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
overrides:
|
|
- matcher:
|
|
id: byName
|
|
options: Time
|
|
properties:
|
|
- id: custom.hidden
|
|
value: true
|
|
- matcher:
|
|
id: byName
|
|
options: Value
|
|
properties:
|
|
- id: displayName
|
|
value: Count
|
|
gridPos:
|
|
h: 5
|
|
w: 8
|
|
x: 0
|
|
'y': 7
|
|
id: 101
|
|
options:
|
|
cellHeight: sm
|
|
footer:
|
|
countRows: false
|
|
fields: ''
|
|
reducer:
|
|
- sum
|
|
show: false
|
|
showHeader: true
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: false
|
|
expr: sum(vm_app_version{job=~"$job", instance=~"$instance"}) by(job, short_version)
|
|
format: table
|
|
instant: true
|
|
range: false
|
|
refId: A
|
|
title: ''
|
|
type: table
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: stepAfter
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 5
|
|
w: 16
|
|
x: 8
|
|
'y': 7
|
|
id: 13
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- lastNotNull
|
|
displayMode: table
|
|
placement: right
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(up{job=~"$job", instance=~"$instance"}) by (job)
|
|
format: time_series
|
|
instant: false
|
|
interval: ''
|
|
legendFormat: __auto
|
|
refId: A
|
|
title: Uptime
|
|
type: timeseries
|
|
- collapsed: false
|
|
gridPos:
|
|
h: 1
|
|
w: 24
|
|
x: 0
|
|
'y': 12
|
|
id: 24
|
|
panels: []
|
|
title: Overview
|
|
type: row
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows in/out [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.\n\nClick on the line and choose Drilldown to show CPU usage per instance\n"
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links:
|
|
- title: Drilldown
|
|
url: /d/G7Z9GzMGz?viewPanel=123&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides:
|
|
- matcher:
|
|
id: byRegexp
|
|
options: /out .*/
|
|
properties:
|
|
- id: custom.transform
|
|
value: negative-Y
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 13
|
|
id: 5
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate({__name__=~"vmagent_rows_inserted_total|vm_promscrape_scraped_samples_sum", job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
|
|
hide: false
|
|
interval: ''
|
|
legendFormat: in {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
|
|
interval: ''
|
|
legendFormat: out {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: B
|
|
title: Samples rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show the persistent queue size per instance.\n"
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links:
|
|
- targetBlank: true
|
|
title: Drilldown
|
|
url: /d/G7Z9GzMGz?viewPanel=125&var-url=${__field.labels.url}&var-ds=$ds&var-instance=$instance&var-job=$job&${__url_time_range}
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: bytes
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 13
|
|
id: 17
|
|
links:
|
|
- title: Troubleshooting
|
|
url: https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(vmagent_remotewrite_pending_data_bytes{job=~"$job", instance=~"$instance", url=~"$url"}) by (job, url) > 2e6
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}job{{`}}`}} => {{`{{`}}url{{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
title: Persistent queue size ($instance) to ($url)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: bars
|
|
fillOpacity: 100
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: normal
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 21
|
|
id: 107
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vm_log_messages_total{job=~"$job",instance=~"$instance", level!="info"}[$__rate_interval])) by (job, level) > 0
|
|
format: time_series
|
|
hide: false
|
|
interval: 5m
|
|
intervalFactor: 1
|
|
legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}level{{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
title: Logging rate
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows rate of dropped [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) from persistent queue. vmagent drops samples from queue if in-memory and on-disk queues are full and it is unable to flush them to remote storage.
|
|
|
|
The max size of on-disk queue is configured by `-remoteWrite.maxDiskUsagePerURL` flag.'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: decbytes
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 21
|
|
id: 49
|
|
links:
|
|
- targetBlank: true
|
|
title: Troubleshooting
|
|
url: https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_persistentqueue_bytes_dropped_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (path) > 0
|
|
interval: ''
|
|
legendFormat: '{{`{{`}} path {{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
title: Persistent queue dropped rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the rate of requests served by vmagent HTTP server.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 29
|
|
id: 15
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vmagent_http_requests_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, path) > 0
|
|
interval: ''
|
|
legendFormat: ''
|
|
range: true
|
|
refId: A
|
|
title: Requests rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Errors rate shows rate for multiple metrics that track possible errors in vmagent, such as network or parsing errors.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 29
|
|
id: 69
|
|
links:
|
|
- targetBlank: true
|
|
title: Troubleshooting
|
|
url: https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vmagent_http_request_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, protocol) > 0
|
|
interval: ''
|
|
legendFormat: requests:{{`{{`}}protocol{{`}}`}} ({{`{{`}}job{{`}}`}})
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_protoparser_read_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type) > 0
|
|
interval: ''
|
|
legendFormat: 'parse: {{`{{`}}type{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: B
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_ingestserver_request_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type) > 0
|
|
interval: ''
|
|
legendFormat: 'ingest: {{`{{`}}type{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: C
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_protoparser_unmarshal_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type) > 0
|
|
interval: ''
|
|
legendFormat: 'unmarshal: {{`{{`}}type{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: D
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_promscrape_dial_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
|
|
interval: ''
|
|
legendFormat: scrape dial ({{`{{`}}job{{`}}`}})
|
|
range: true
|
|
refId: E
|
|
title: Errors rate ($instance)
|
|
type: timeseries
|
|
- collapsed: true
|
|
gridPos:
|
|
h: 1
|
|
w: 24
|
|
x: 0
|
|
'y': 37
|
|
id: 45
|
|
panels:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Percentage of used RSS memory (resident).
|
|
|
|
The RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).
|
|
|
|
The application''s performance will significantly degrade when memory usage is close to 100%.
|
|
|
|
|
|
Click on the line and choose Drilldown to show memory usage per instance'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links:
|
|
- targetBlank: true
|
|
title: Drilldown
|
|
url: /d/G7Z9GzMGz?viewPanel=117&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: percentunit
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 38
|
|
id: 111
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: |-
|
|
max(
|
|
max_over_time(process_resident_memory_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
|
|
/
|
|
vm_available_memory_bytes{job=~"$job", instance=~"$instance"}
|
|
) by(job)
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: RSS memory % usage ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links:
|
|
- targetBlank: true
|
|
title: Drilldown
|
|
url: /d/G7Z9GzMGz?viewPanel=119&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: percentunit
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 38
|
|
id: 157
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: false
|
|
expr: |-
|
|
max(
|
|
rate(process_cpu_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])
|
|
/
|
|
process_cpu_cores_available{job=~"$job", instance=~"$instance"}
|
|
) by(instance)
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 1
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: CPU % usage
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
|
|
|
|
Safe memory usage % considered to be below 80%
|
|
|
|
|
|
Click on the line and choose Drilldown to show memory usage per instance'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: percentunit
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 46
|
|
id: 155
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: false
|
|
expr: |-
|
|
max(
|
|
max_over_time(process_resident_memory_anon_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
|
|
/
|
|
vm_available_memory_bytes{job=~"$job", instance=~"$instance"}
|
|
) by(instance)
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: RSS anonymous memory % usage
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
|
|
|
|
|
|
The lower the better.'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
unit: s
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 46
|
|
id: 158
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_pressure_cpu_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job, instance)
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} - waiting'
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_pressure_cpu_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job, instance)
|
|
format: time_series
|
|
hide: false
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} - stalled'
|
|
range: true
|
|
refId: B
|
|
title: CPU pressure
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
|
|
|
|
|
|
The lower the better.'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
unit: s
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 54
|
|
id: 156
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_pressure_memory_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job, instance)
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} - waiting'
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_pressure_memory_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job, instance)
|
|
format: time_series
|
|
hide: false
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} - stalled'
|
|
range: true
|
|
refId: B
|
|
title: Memory pressure
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.
|
|
|
|
|
|
Click on the line and choose Drilldown to show CPU usage per instance'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links:
|
|
- targetBlank: true
|
|
title: Drilldown
|
|
url: /d/G7Z9GzMGz?viewPanel=121&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: bytes
|
|
overrides:
|
|
- matcher:
|
|
id: byName
|
|
options: read
|
|
properties:
|
|
- id: custom.transform
|
|
value: negative-Y
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 54
|
|
id: 81
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_io_storage_read_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
|
|
format: time_series
|
|
hide: false
|
|
interval: ''
|
|
intervalFactor: 1
|
|
legendFormat: read {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_io_storage_written_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
|
|
format: time_series
|
|
hide: false
|
|
interval: ''
|
|
intervalFactor: 1
|
|
legendFormat: write {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: B
|
|
title: Disk writes/reads ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 62
|
|
id: 39
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(max_over_time(go_goroutines{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Goroutines ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
|
|
|
|
|
|
The lower the better.'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
unit: s
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 62
|
|
id: 159
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_pressure_io_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job, instance)
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} - waiting'
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_pressure_io_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job, instance)
|
|
format: time_series
|
|
hide: false
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} - stalled'
|
|
range: true
|
|
refId: B
|
|
title: IO pressure
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 70
|
|
id: 41
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(max_over_time(process_num_threads{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
|
|
format: time_series
|
|
intervalFactor: 2
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Threads ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Network usage shows the bytes rate for data accepted by vmagent and pushed via remotewrite protocol.
|
|
|
|
Discrepancies are possible because of different protocols used for ingesting, scraping and writing data.'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: bps
|
|
overrides:
|
|
- matcher:
|
|
id: byRegexp
|
|
options: /out.*/
|
|
properties:
|
|
- id: custom.transform
|
|
value: negative-Y
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 70
|
|
id: 7
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8 \n+ sum(rate(vm_promscrape_conn_bytes_read_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8"
|
|
interval: ''
|
|
legendFormat: in {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) * 8
|
|
interval: ''
|
|
legendFormat: out {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: B
|
|
title: Network usage ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the percent of CPU spent on garbage collection.
|
|
|
|
|
|
If % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.
|
|
|
|
|
|
Try searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ '
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: percentunit
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 78
|
|
id: 135
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)"
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: CPU spent on GC ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing."
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 0.1
|
|
unit: s
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 78
|
|
id: 149
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~"$job"}[$__rate_interval])) by (job, instance, le))) by(job)
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Go scheduling latency
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
unit: bytes
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 86
|
|
id: 154
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(go_memstats_alloc_bytes_total{job=~"$job"}[$__rate_interval])) by (job, instance)
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Memory allocations rate
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Panel shows the percentage of open file descriptors in the OS per instance.
|
|
|
|
Reaching the limit of open files (100%) can cause various issues and must be prevented.
|
|
|
|
|
|
See how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
decimals: 5
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: percentunit
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 86
|
|
id: 83
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: |-
|
|
max(
|
|
max_over_time(process_open_fds{job=~"$job", instance=~"$instance"}[$__rate_interval])
|
|
/
|
|
process_max_fds{job=~"$job", instance=~"$instance"}
|
|
) by(job)
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 2
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Open FDs usage % ($instance)
|
|
type: timeseries
|
|
title: Resource usage
|
|
type: row
|
|
- collapsed: true
|
|
gridPos:
|
|
h: 1
|
|
w: 24
|
|
x: 0
|
|
'y': 38
|
|
id: 94
|
|
panels:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows top 10 jobs by the number of new series registered by vmagent over the 5min range. These jobs generate the most of the churn rate.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1580
|
|
id: 92
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: false
|
|
expr: topk(10, sum(sum_over_time(scrape_series_added[5m])) by (job)) > 0
|
|
interval: ''
|
|
legendFormat: '{{`{{`}} job {{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
title: Top 10 jobs by unique samples
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows top 10 instances by the number of new series registered by vmagent over the 5min range. These instances generate the most of the churn rate.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1580
|
|
id: 95
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: false
|
|
expr: topk(10, sum(sum_over_time(scrape_series_added[5m])) by (instance)) > 0
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Top 10 instances by unique samples
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows write saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMax: 2
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: transparent
|
|
- color: red
|
|
value: 0.9
|
|
unit: s
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1588
|
|
id: 98
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: max(rate(vm_persistentqueue_write_duration_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job)
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Persistent queue write saturation ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows read saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMax: 2
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: transparent
|
|
- color: red
|
|
value: 0.9
|
|
unit: s
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1588
|
|
id: 99
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: max(rate(vm_persistentqueue_read_duration_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job)
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Persistent queue read saturation ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the rate of dropped data blocks in cases when remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses.
|
|
|
|
|
|
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: normal
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1596
|
|
id: 79
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vmagent_remotewrite_packets_dropped_total{job=~"$job", instance=~"$instance", url=~"$url"}[$__rate_interval])) by(job, url) > 0
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Data blocks dropped ($instance) to ($url)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows the rate of dropped [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) due to relabeling. \nMetric tracks drops for `-remoteWrite.relabelConfig` configuration only."
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: normal
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1596
|
|
id: 18
|
|
links:
|
|
- targetBlank: true
|
|
title: Relabeling
|
|
url: https://docs.victoriametrics.com/victoriametrics/relabeling/
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vmagent_remotewrite_relabel_metrics_dropped_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, url) > 0
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: B
|
|
title: Rows dropped by relabeling ($instance) to ($url)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the rate of parsed datapoints from write or scrape requests.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1604
|
|
id: 127
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vm_protoparser_rows_read_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type) > 0
|
|
interval: ''
|
|
legendFormat: '{{`{{`}} type {{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
title: Datapoints rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Tracks the rate of dropped invalid rows because of errors while unmarshaling write requests. The exact errors messages will be printed in logs.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1604
|
|
id: 50
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vm_rows_invalid_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type) > 0
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}type{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
title: Invalid datapoints rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: thresholds
|
|
custom:
|
|
align: auto
|
|
cellOptions:
|
|
type: auto
|
|
inspect: false
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
overrides:
|
|
- matcher:
|
|
id: byName
|
|
options: Value
|
|
properties:
|
|
- id: custom.hidden
|
|
value: true
|
|
- matcher:
|
|
id: byName
|
|
options: Time
|
|
properties:
|
|
- id: custom.hidden
|
|
value: true
|
|
gridPos:
|
|
h: 7
|
|
w: 12
|
|
x: 0
|
|
'y': 1612
|
|
id: 129
|
|
options:
|
|
cellHeight: sm
|
|
footer:
|
|
countRows: false
|
|
fields: ''
|
|
reducer:
|
|
- sum
|
|
show: false
|
|
showHeader: true
|
|
sortBy:
|
|
- desc: true
|
|
displayName: job
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: false
|
|
expr: sum(flag{is_set="true", job=~"$job", instance=~"$instance"}) by(job, instance, name, value)
|
|
format: table
|
|
instant: true
|
|
legendFormat: __auto
|
|
range: false
|
|
refId: A
|
|
title: Non-default flags
|
|
transformations:
|
|
- id: groupBy
|
|
options:
|
|
fields:
|
|
instance:
|
|
aggregations: []
|
|
job:
|
|
aggregations: []
|
|
operation: groupby
|
|
name:
|
|
aggregations: []
|
|
operation: groupby
|
|
value:
|
|
aggregations: []
|
|
operation: groupby
|
|
type: table
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn''t restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. '
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMin: 0
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: stepAfter
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
decimals: 0
|
|
links: []
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: none
|
|
overrides: []
|
|
gridPos:
|
|
h: 7
|
|
w: 12
|
|
x: 12
|
|
'y': 1612
|
|
id: 150
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- lastNotNull
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(changes(vm_app_start_timestamp{job=~"$job", instance=~"$instance"}[$__rate_interval]) > 0) by(job)
|
|
format: time_series
|
|
instant: false
|
|
legendFormat: '{{`{{`}}job{{`}}`}}'
|
|
refId: A
|
|
title: Restarts ($job)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows how many samples were ignored on insertion due to various reasons. See the reason of rejection application logs.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1619
|
|
id: 151
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(increase(vm_rows_ignored_total{job=~"$job", instance=~"$instance"}[1h])) by (reason)
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Rows ignored for last 1h ($instance)
|
|
type: timeseries
|
|
title: Troubleshooting
|
|
type: row
|
|
- collapsed: true
|
|
gridPos:
|
|
h: 1
|
|
w: 24
|
|
x: 0
|
|
'y': 39
|
|
id: 28
|
|
panels:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 7
|
|
w: 12
|
|
x: 0
|
|
'y': 1825
|
|
id: 48
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(vm_promscrape_targets{job=~"$job", instance=~"$instance", status="up"}) by(job, type) > 0
|
|
format: time_series
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}type{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
title: Scrape targets UP(By Type)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 10
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 7
|
|
w: 12
|
|
x: 12
|
|
'y': 1825
|
|
id: 76
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(vm_promscrape_targets{job=~"$job", instance=~"$instance", status="down"}) by(job, type) > 0
|
|
format: time_series
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}type{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
title: Scrape targets DOWN(By Type)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 7
|
|
w: 12
|
|
x: 0
|
|
'y': 1832
|
|
id: 132
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(vm_promscrape_scrape_pool_targets{job=~"$job", instance=~"$instance", status="up"}) by(job, scrape_job) > 0
|
|
format: time_series
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}job{{`}}`}}: {{`{{`}}scrape_job{{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
title: Scrape targets UP(By Job)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 10
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 7
|
|
w: 12
|
|
x: 12
|
|
'y': 1832
|
|
id: 133
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(vm_promscrape_scrape_pool_targets{job=~"$job", instance=~"$instance", status="down"}) by(job, scrape_job) > 0
|
|
format: time_series
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}job{{`}}`}}: {{`{{`}}scrape_job{{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
title: Scrape targets DOWN(By Job)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the number of scrapes per second.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1839
|
|
id: 20
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vm_promscrape_scrapes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Scrape rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the number of datapoints scraped per second.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1839
|
|
id: 126
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vm_promscrape_scraped_samples_sum{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Scraped datapoints rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: bytes
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1847
|
|
id: 46
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_response_size_bytes_bucket{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, vmrange))) by(job)
|
|
format: time_series
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Scrape response size 0.99 quantile ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: s
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1847
|
|
id: 148
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, vmrange))) by(job)
|
|
format: time_series
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Scrape duration 0.99 quantile ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 10
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1855
|
|
id: 31
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_promscrape_scrapes_failed_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
|
|
interval: ''
|
|
legendFormat: scrapes failed ({{`{{`}}job{{`}}`}})
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_promscrape_scrapes_timed_out_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
|
|
interval: ''
|
|
legendFormat: timeouts ({{`{{`}}job{{`}}`}})
|
|
range: true
|
|
refId: B
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_promscrape_scrapes_gunzip_failed_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
|
|
interval: ''
|
|
legendFormat: gunzip fails ({{`{{`}}job{{`}}`}})
|
|
range: true
|
|
refId: C
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_promscrape_dial_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
|
|
interval: ''
|
|
legendFormat: dial fails ({{`{{`}}job{{`}}`}})
|
|
range: true
|
|
refId: D
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_promscrape_max_scrape_size_exceeded_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
|
|
hide: false
|
|
interval: ''
|
|
legendFormat: max scrape size exceeded ({{`{{`}}job{{`}}`}})
|
|
range: true
|
|
refId: E
|
|
title: Scrape fails ($instance)
|
|
type: timeseries
|
|
title: Scraping
|
|
type: row
|
|
- collapsed: true
|
|
gridPos:
|
|
h: 1
|
|
w: 24
|
|
x: 0
|
|
'y': 40
|
|
id: 71
|
|
panels:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the rate of write requests served by ingestserver (UDP, TCP connections) and HTTP server.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1547
|
|
id: 73
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 9.2.6
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vm_ingestserver_requests_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type, net) > 0
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}net{{`}}`}}: {{`{{`}} type {{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vmagent_http_requests_total{job=~"$job", instance=~"$instance", protocol!=""}[$__rate_interval])) by(job, protocol) > 0
|
|
interval: ''
|
|
legendFormat: '{{`{{`}} protocol {{`}}`}}: http ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: B
|
|
title: Requests rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the rate of rows ingested in vmagent via push protocols.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1547
|
|
id: 131
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 9.2.6
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vmagent_rows_inserted_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type) > 0
|
|
interval: ''
|
|
legendFormat: '{{`{{`}} type {{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
title: Rows rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows how many concurrent inserts are taking place.\n\nIf the number of concurrent inserts hitting the `limit` or is close to the `limit` constantly - it might be a sign of a resource shortage.\n\n If vmagent's CPU usage and remote write connection saturation are at normal level, it might be that `-maxConcurrentInserts` cmd-line flag need to be increased."
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1823
|
|
id: 130
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 9.2.6
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: max_over_time(vm_concurrent_insert_current{job=~"$job", instance=~"$instance"}[$__rate_interval])
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: min(vm_concurrent_insert_capacity{job=~"$job", instance=~"$instance"}) by(job)
|
|
interval: ''
|
|
legendFormat: limit ({{`{{`}}job{{`}}`}})
|
|
range: true
|
|
refId: B
|
|
title: Concurrent inserts ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the rate of write errors in ingestserver (UDP, TCP connections) and HTTP server.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1823
|
|
id: 77
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 9.2.6
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
exemplar: true
|
|
expr: sum(rate(vm_ingestserver_request_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(type, net) > 0
|
|
interval: ''
|
|
legendFormat: '{{`{{`}} type {{`}}`}} ({{`{{`}}net{{`}}`}})'
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
exemplar: true
|
|
expr: sum(rate(vmagent_http_request_errors_total{job=~"$job", instance=~"$instance", protocol!=""}[$__rate_interval])) by(protocol) > 0
|
|
interval: ''
|
|
legendFormat: '{{`{{`}} protocol {{`}}`}} (http)'
|
|
refId: B
|
|
title: Error rate ($instance)
|
|
type: timeseries
|
|
title: Ingestion
|
|
type: row
|
|
- collapsed: true
|
|
gridPos:
|
|
h: 1
|
|
w: 24
|
|
x: 0
|
|
'y': 41
|
|
id: 136
|
|
panels:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows the number of matched samples by the aggregation rule. \n\nThe more samples is matched, the more work this aggregation rule does. The matching rule is specified via `match` param.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). "
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMin: 0
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: auto
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1870
|
|
id: 146
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- min
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
mode: single
|
|
sort: none
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_streamaggr_matched_samples_total{job=~"$job",instance=~"$instance", url=~"$url"}[$__rate_interval])) without (instance, pod) > 0
|
|
instant: false
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Matched samples ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "The rate of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples "
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMin: 0
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: auto
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1870
|
|
id: 143
|
|
options:
|
|
legend:
|
|
calcs: []
|
|
displayMode: list
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
mode: single
|
|
sort: none
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_streamaggr_ignored_samples_total{job=~"$job",instance=~"$instance", url=~"$url"}[$__rate_interval]) > 0) without (instance, pod)
|
|
instant: false
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Ignored samples ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows the number of produced samples by the aggregation rule. \n\nNumber of produced samples depend on params like `by`, `without`, `interval`, etc.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). "
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMin: 0
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: auto
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1878
|
|
id: 147
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- min
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
mode: single
|
|
sort: none
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vm_streamaggr_output_samples_total{job=~"$job",instance=~"$instance", url=~"$url"}[$__rate_interval])) without (instance, pod) > 0
|
|
instant: false
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Produced samples ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation"
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMin: 0
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: auto
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
min: -5
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1878
|
|
id: 139
|
|
options:
|
|
legend:
|
|
calcs: []
|
|
displayMode: list
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
mode: single
|
|
sort: none
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: increase(vm_streamaggr_flush_timeouts_total{job=~"$job",instance=~"$instance", url=~"$url"}[$__rate_interval]) > 0
|
|
instant: false
|
|
legendFormat: 'aggregation: {{`{{`}}url{{`}}`}} ({{`{{`}}job{{`}}`}}): {{`{{`}}path{{`}}`}}:{{`{{`}}position{{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: increase(vm_streamaggr_dedup_flush_timeouts_total{job=~"$job",instance=~"$instance", url=~"$url"}[$__rate_interval]) > 0
|
|
hide: false
|
|
instant: false
|
|
legendFormat: 'deduplication: {{`{{`}}url{{`}}`}} ({{`{{`}}job{{`}}`}}): {{`{{`}}path{{`}}`}}:{{`{{`}}position{{`}}`}}'
|
|
range: true
|
|
refId: B
|
|
title: Flush timeouts ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the max lag between samples timestamps within one batch passed to the aggregation input.
|
|
|
|
|
|
Lower is better.
|
|
|
|
|
|
Too high lag or lag exceeding the interval might be a sign that data was delayed before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.
|
|
|
|
|
|
See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMin: 0
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: auto
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: s
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1886
|
|
id: 142
|
|
options:
|
|
legend:
|
|
calcs: []
|
|
displayMode: list
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
mode: single
|
|
sort: none
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~"$job",instance=~"$instance", url=~"$url"}[$__rate_interval])) without (instance, pod))
|
|
instant: false
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Samples lag 0.99 quantile ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel."
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMin: 0
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: auto
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: s
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1886
|
|
id: 137
|
|
options:
|
|
legend:
|
|
calcs: []
|
|
displayMode: list
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
mode: single
|
|
sort: none
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: histogram_quantile(0.99, rate(vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~"$job",instance=~"$instance", url=~"$url"}[$__rate_interval]))
|
|
instant: false
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Dedup flush duration 0.99 quantile ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the size of Label Compressor in number of entries.
|
|
|
|
|
|
Labels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.
|
|
|
|
|
|
Rapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
axisSoftMin: 0
|
|
barAlignment: 0
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: auto
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
overrides:
|
|
- matcher:
|
|
id: byRegexp
|
|
options: /bytes.*/
|
|
properties:
|
|
- id: custom.axisPlacement
|
|
value: right
|
|
- id: unit
|
|
value: bytes
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1926
|
|
id: 141
|
|
options:
|
|
legend:
|
|
calcs: []
|
|
displayMode: list
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
mode: single
|
|
sort: none
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: max(vm_streamaggr_labels_compressor_items_count{job=~"$job",instance=~"$instance"}) by(job, instance)
|
|
hide: false
|
|
instant: false
|
|
legendFormat: 'items: {{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: max(vm_streamaggr_labels_compressor_size_bytes{job=~"$job", instance=~"$instance"}) by(job, instance)
|
|
hide: false
|
|
instant: false
|
|
legendFormat: 'bytes: {{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: B
|
|
title: Labels compressor ($instance)
|
|
type: timeseries
|
|
title: Streaming aggregation
|
|
type: row
|
|
- collapsed: true
|
|
gridPos:
|
|
h: 1
|
|
w: 24
|
|
x: 0
|
|
'y': 42
|
|
id: 58
|
|
panels:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the rate of requests to configured remote write endpoints by url and status code.
|
|
|
|
|
|
Remote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.
|
|
|
|
|
|
'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
decimals: 2
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1828
|
|
id: 60
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vmagent_remotewrite_requests_total{job=~"$job", instance=~"$instance", url=~"$url"}[$__rate_interval])) by(job, url, status_code) > 0
|
|
interval: ''
|
|
legendFormat: ''
|
|
range: true
|
|
refId: A
|
|
title: Requests rate ($instance) to ($url)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the global rate for number of written bytes via remote write connections.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: decbytes
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1828
|
|
id: 66
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Bytes write rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows requests retry rate by url. Number of retries is unlimited but protected with delays up to 1m between attempts.
|
|
|
|
|
|
Remote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.
|
|
|
|
|
|
'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1836
|
|
id: 61
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(rate(vmagent_remotewrite_retries_count_total{job=~"$job", instance=~"$instance", url=~"$url"}[$__rate_interval])) by(url) > 0
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Retry rate ($instance) to ($url)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows current number of established connections to remote write endpoints.
|
|
|
|
|
|
'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1836
|
|
id: 65
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(max_over_time(vmagent_remotewrite_conns{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job)
|
|
interval: ''
|
|
legendFormat: __auto
|
|
range: true
|
|
refId: A
|
|
title: Connections ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the current limit usage of unique series over an hourly period. Vmagent will start to drop series once the limit is reached.
|
|
|
|
|
|
Please note, panel will be blank if `remoteWrite.maxHourlySeries` is not set.'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
mappings: []
|
|
max: 100
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: transparent
|
|
- color: red
|
|
value: 0.9
|
|
unit: percent
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1844
|
|
id: 88
|
|
options:
|
|
legend:
|
|
calcs: []
|
|
displayMode: list
|
|
placement: bottom
|
|
showLegend: false
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: "max(\n vmagent_hourly_series_limit_current_series{job=~\"$job\", instance=~\"$instance\"} \n / \n vmagent_hourly_series_limit_max_series{job=~\"$job\", instance=~\"$instance\"}\n ) by(job) * 100"
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}job{{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
title: Hourly series limit
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows saturation of every connection to remote storage. If the threshold of 90% is reached, then the connection is saturated (busy or slow) by more than 90%, so vmagent won't be able to keep up and can start buffering data. \n\nThis usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase the number of connections per each remote storage.\n"
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: transparent
|
|
- color: red
|
|
value: 0.9
|
|
unit: percentunit
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 1844
|
|
id: 84
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: |-
|
|
max(
|
|
rate(vmagent_remotewrite_send_duration_seconds_total{job=~"$job", instance=~"$instance", url=~"$url"}[$__rate_interval])
|
|
/
|
|
vmagent_remotewrite_queues{job=~"$job", instance=~"$instance", url=~"$url"}
|
|
) by(job, url)
|
|
interval: ''
|
|
legendFormat: ''
|
|
range: true
|
|
refId: A
|
|
title: Remote write connection saturation ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the current limit usage of unique series over a daily period. Vmagent will start to drop series once the limit is reached.
|
|
|
|
|
|
Please note, panel will be blank if `remoteWrite.maxDailySeries` is not set.'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: line
|
|
mappings: []
|
|
max: 100
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: transparent
|
|
- color: red
|
|
value: 0.9
|
|
unit: percentunit
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 1852
|
|
id: 90
|
|
options:
|
|
legend:
|
|
calcs: []
|
|
displayMode: list
|
|
placement: bottom
|
|
showLegend: false
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: "max(\n vmagent_daily_series_limit_current_series{job=~\"$job\",instance=~\"$instance\"} \n / \n vmagent_daily_series_limit_max_series{job=~\"$job\",instance=~\"$instance\"}\n) by(job)"
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}job{{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
title: Daily series limit
|
|
type: timeseries
|
|
title: Remote write
|
|
type: row
|
|
- collapsed: true
|
|
gridPos:
|
|
h: 1
|
|
w: 24
|
|
x: 0
|
|
'y': 43
|
|
id: 113
|
|
panels:
|
|
- fieldConfig:
|
|
defaults: {}
|
|
overrides: []
|
|
gridPos:
|
|
h: 2
|
|
w: 24
|
|
x: 0
|
|
'y': 100
|
|
id: 115
|
|
options:
|
|
code:
|
|
language: plaintext
|
|
showLineNumbers: false
|
|
showMiniMap: false
|
|
content: Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.
|
|
mode: markdown
|
|
pluginVersion: 11.5.0
|
|
title: ''
|
|
transparent: true
|
|
type: text
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: ''
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
unit: short
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 102
|
|
id: 119
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: false
|
|
expr: sum(rate(process_cpu_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, instance)
|
|
format: time_series
|
|
interval: ''
|
|
intervalFactor: 1
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
title: CPU usage ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: 'Shows the used memory (resident).
|
|
|
|
The application''s performance will significantly degrade when memory usage is close to 100%.'
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
unit: decbytes
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 102
|
|
id: 117
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: desc
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: max_over_time(process_resident_memory_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval])
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
|
|
range: true
|
|
refId: A
|
|
title: RSS memory usage ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows the persistent queue size of pending samples in bytes which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true."
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
min: 0
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: bytes
|
|
overrides: []
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 110
|
|
id: 125
|
|
links:
|
|
- title: Troubleshooting
|
|
url: https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
exemplar: true
|
|
expr: sum(vmagent_remotewrite_pending_data_bytes{job=~"$job", instance=~"$instance", url=~"$url"}) by (instance, url)
|
|
interval: ''
|
|
legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}url{{`}}`}}'
|
|
range: true
|
|
refId: A
|
|
title: Persistent queue size ($instance) to ($url)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: "Shows in/out samples rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target."
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: short
|
|
overrides:
|
|
- matcher:
|
|
id: byRegexp
|
|
options: /out .*/
|
|
properties:
|
|
- id: custom.transform
|
|
value: negative-Y
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 12
|
|
'y': 110
|
|
id: 123
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: |-
|
|
sum(rate(vm_promscrape_scraped_samples_sum{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, instance)
|
|
+ sum(rate(vmagent_rows_inserted_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, instance)
|
|
hide: false
|
|
interval: ''
|
|
legendFormat: in {{`{{`}}instance{{`}}`}} {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, instance)
|
|
interval: ''
|
|
legendFormat: out {{`{{`}}instance{{`}}`}} {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: B
|
|
title: Samples rate ($instance)
|
|
type: timeseries
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
description: Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.
|
|
fieldConfig:
|
|
defaults:
|
|
color:
|
|
mode: palette-classic
|
|
custom:
|
|
axisBorderShow: false
|
|
axisCenteredZero: false
|
|
axisColorMode: text
|
|
axisLabel: ''
|
|
axisPlacement: auto
|
|
barAlignment: 0
|
|
barWidthFactor: 0.6
|
|
drawStyle: line
|
|
fillOpacity: 0
|
|
gradientMode: none
|
|
hideFrom:
|
|
legend: false
|
|
tooltip: false
|
|
viz: false
|
|
insertNulls: false
|
|
lineInterpolation: linear
|
|
lineWidth: 1
|
|
pointSize: 5
|
|
scaleDistribution:
|
|
type: linear
|
|
showPoints: never
|
|
spanNulls: false
|
|
stacking:
|
|
group: A
|
|
mode: none
|
|
thresholdsStyle:
|
|
mode: 'off'
|
|
links: []
|
|
mappings: []
|
|
thresholds:
|
|
mode: absolute
|
|
steps:
|
|
- color: green
|
|
value: null
|
|
- color: red
|
|
value: 80
|
|
unit: bytes
|
|
overrides:
|
|
- matcher:
|
|
id: byName
|
|
options: read
|
|
properties:
|
|
- id: custom.transform
|
|
value: negative-Y
|
|
gridPos:
|
|
h: 8
|
|
w: 12
|
|
x: 0
|
|
'y': 118
|
|
id: 121
|
|
options:
|
|
legend:
|
|
calcs:
|
|
- mean
|
|
- lastNotNull
|
|
- max
|
|
displayMode: table
|
|
placement: bottom
|
|
showLegend: true
|
|
sortBy: Last *
|
|
sortDesc: true
|
|
tooltip:
|
|
hideZeros: false
|
|
mode: multi
|
|
sort: none
|
|
pluginVersion: 11.5.0
|
|
targets:
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_io_storage_read_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, instance)
|
|
format: time_series
|
|
hide: false
|
|
interval: ''
|
|
intervalFactor: 1
|
|
legendFormat: read {{`{{`}}instance{{`}}`}} {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: A
|
|
- datasource:
|
|
type: {{ $defaultDatasource }}
|
|
uid: $ds
|
|
editorMode: code
|
|
expr: sum(rate(process_io_storage_written_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job,instance)
|
|
format: time_series
|
|
hide: false
|
|
interval: ''
|
|
intervalFactor: 1
|
|
legendFormat: write {{`{{`}}instance{{`}}`}} {{`{{`}}job{{`}}`}}
|
|
range: true
|
|
refId: B
|
|
title: Disk writes/reads ($instance)
|
|
type: timeseries
|
|
title: Drilldown
|
|
type: row
|
|
preload: false
|
|
refresh: ''
|
|
schemaVersion: 40
|
|
tags:
|
|
- victoriametrics
|
|
- vm-k8s-stack
|
|
templating:
|
|
list:
|
|
- current: {}
|
|
includeAll: false
|
|
name: ds
|
|
options: []
|
|
query: {{ $defaultDatasource }}
|
|
refresh: 1
|
|
regex: ''
|
|
type: datasource
|
|
- current: {}
|
|
datasource:
|
|
type: prometheus
|
|
uid: $ds
|
|
definition: label_values(vm_app_version{version=~"^vmagent.*"}, job)
|
|
includeAll: true
|
|
multi: true
|
|
name: job
|
|
options: []
|
|
query:
|
|
query: label_values(vm_app_version{version=~"^vmagent.*"}, job)
|
|
refId: VictoriaMetrics-job-Variable-Query
|
|
refresh: 1
|
|
regex: ''
|
|
type: query
|
|
- allValue: .*
|
|
current: {}
|
|
datasource:
|
|
type: prometheus
|
|
uid: $ds
|
|
definition: label_values(vm_app_version{job=~"$job"}, instance)
|
|
includeAll: true
|
|
multi: true
|
|
name: instance
|
|
options: []
|
|
query:
|
|
query: label_values(vm_app_version{job=~"$job"}, instance)
|
|
refId: VictoriaMetrics-instance-Variable-Query
|
|
refresh: 1
|
|
regex: ''
|
|
type: query
|
|
- allValue: .*
|
|
current: {}
|
|
datasource:
|
|
type: prometheus
|
|
uid: $ds
|
|
definition: label_values(vmagent_remotewrite_requests_total{job=~"$job", instance=~"$instance"}, url)
|
|
description: The remote write URLs
|
|
includeAll: true
|
|
multi: true
|
|
name: url
|
|
options: []
|
|
query:
|
|
query: label_values(vmagent_remotewrite_requests_total{job=~"$job", instance=~"$instance"}, url)
|
|
refId: StandardVariableQuery
|
|
refresh: 1
|
|
regex: ''
|
|
type: query
|
|
- baseFilters: []
|
|
datasource:
|
|
type: prometheus
|
|
uid: $ds
|
|
filters: []
|
|
name: adhoc
|
|
type: adhoc
|
|
time:
|
|
from: now-3h
|
|
to: now
|
|
timepicker:
|
|
refresh_intervals:
|
|
- 10s
|
|
- 30s
|
|
- 1m
|
|
- 5m
|
|
- 15m
|
|
- 30m
|
|
- 1h
|
|
- 2h
|
|
- 1d
|
|
timezone: {{ default "utc" ($Values.defaultDashboards).defaultTimezone }}
|
|
title: VictoriaMetrics - vmagent
|
|
uid: G7Z9GzMGz
|
|
version: 1
|
|
weekStart: ''
|