{{- $Values := (.helm).Values | default .Values }} {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }} {{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }} {{- $defaultDatasource := "prometheus" -}} {{- range (((($Values.grafana).sidecar).datasources).victoriametrics | default list) }} {{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }} {{- end }} annotations: list: - builtIn: 1 datasource: type: datasource uid: grafana enable: true hide: true iconColor: rgba(0, 211, 255, 1) name: Annotations & Alerts target: limit: 100 matchAny: false tags: [] type: dashboard type: dashboard - datasource: type: prometheus uid: $ds enable: true expr: sum(ALERTS{job=~"$job", instance=~"$instance", alertgroup="vmcluster",alertstate="firing",show_at="dashboard"}) by(alertname) hide: false iconColor: red name: alerts titleFormat: '{{`{{`}}alertname{{`}}`}}' - datasource: type: prometheus uid: $ds enable: true expr: sum(vm_app_version{job=~"$job", instance=~"$instance"}) by(version) unless (sum(vm_app_version{job=~"$job", instance=~"$instance"} offset $__interval) by(version)) hide: true iconColor: dark-blue name: version change textFormat: '{{`{{`}}version{{`}}`}}' titleFormat: Version change - datasource: type: prometheus uid: $ds enable: true expr: sum(changes(vm_app_start_timestamp{job=~"$job", instance=~"$instance"}[$__interval])) by(job) hide: false iconColor: dark-yellow name: restarts textFormat: '{{`{{`}}job{{`}}`}} restarted' condition: {{ ($Values.vmcluster).enabled }} description: Overview for cluster VictoriaMetrics v1.117.0 or higher editable: false fiscalYearStartMonth: 0 graphTooltip: 1 id: 1 links: - icon: doc tags: [] targetBlank: true title: Cluster Wiki type: link url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/ - icon: external link tags: [] targetBlank: true title: Found a bug? type: link url: https://github.com/VictoriaMetrics/VictoriaMetrics/issues - icon: external link tags: [] targetBlank: true title: New releases type: link url: https://github.com/VictoriaMetrics/VictoriaMetrics/releases panels: - collapsed: false gridPos: h: 1 w: 24 x: 0 'y': 0 id: 137 panels: [] title: Stats type: row - datasource: type: {{ $defaultDatasource }} uid: $ds description: How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are in storage fieldConfig: defaults: color: mode: thresholds mappings: [] thresholds: mode: absolute steps: - color: green value: null unit: short overrides: [] gridPos: h: 3 w: 6 x: 0 'y': 1 id: 131 maxDataPoints: 100 options: colorMode: value graphMode: area justifyMode: auto orientation: horizontal percentChangeColorMode: standard reduceOptions: calcs: - lastNotNull fields: '' values: false showPercentChange: false text: {} textMode: auto wideLayout: true pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds exemplar: true expr: sum(vm_rows{job=~"$job_storage", type!~"indexdb.*"}) format: time_series instant: true interval: '' intervalFactor: 1 legendFormat: '' refId: A title: Total datapoints type: stat - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the datapoints ingestion rate, including replication factor. fieldConfig: defaults: color: mode: thresholds mappings: [] thresholds: mode: absolute steps: - color: green value: null unit: short overrides: [] gridPos: h: 3 w: 6 x: 6 'y': 1 id: 124 maxDataPoints: 100 options: colorMode: value graphMode: area justifyMode: auto orientation: horizontal percentChangeColorMode: standard reduceOptions: calcs: - lastNotNull fields: '' values: false showPercentChange: false text: {} textMode: auto wideLayout: true pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: false expr: sum(rate(vm_vminsert_metrics_read_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) format: time_series instant: true interval: '' intervalFactor: 1 legendFormat: '' range: false refId: A title: Ingestion rate type: stat - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the rate of HTTP read requests. fieldConfig: defaults: color: mode: thresholds mappings: [] thresholds: mode: absolute steps: - color: green value: null unit: req/s overrides: [] gridPos: h: 3 w: 6 x: 12 'y': 1 id: 130 maxDataPoints: 100 options: colorMode: value graphMode: area justifyMode: auto orientation: horizontal percentChangeColorMode: standard reduceOptions: calcs: - lastNotNull fields: '' values: false showPercentChange: false text: {} textMode: auto wideLayout: true pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(rate(vm_http_requests_total{job=~"$job", instance=~"$instance", path=~"/select/.*"}[$__rate_interval])) format: time_series instant: true interval: '' intervalFactor: 1 legendFormat: '' refId: A title: Read requests type: stat - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Total number of available CPUs for all VM components. ' fieldConfig: defaults: color: mode: thresholds mappings: [] thresholds: mode: absolute steps: - color: green value: null unit: short overrides: [] gridPos: h: 3 w: 6 x: 18 'y': 1 id: 126 maxDataPoints: 100 options: colorMode: value graphMode: area justifyMode: auto orientation: horizontal percentChangeColorMode: standard reduceOptions: calcs: - lastNotNull fields: '' values: false showPercentChange: false text: {} textMode: auto wideLayout: true pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(vm_available_cpu_cores{job=~"$job", instance=~"$instance"}) format: time_series instant: true interval: '' intervalFactor: 1 legendFormat: '' refId: A title: Available CPU type: stat - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown. fieldConfig: defaults: color: mode: thresholds mappings: [] thresholds: mode: absolute steps: - color: green value: null unit: short overrides: [] gridPos: h: 3 w: 6 x: 0 'y': 4 id: 34 maxDataPoints: 100 options: colorMode: value graphMode: area justifyMode: auto orientation: horizontal percentChangeColorMode: standard reduceOptions: calcs: - lastNotNull fields: '' values: false showPercentChange: false text: {} textMode: auto wideLayout: true pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(max_over_time(vm_cache_entries{job=~"$job", instance=~"$instance", type="storage/hour_metric_ids"}[1h])) format: time_series instant: true interval: '' intervalFactor: 1 legendFormat: '' refId: A title: Active series type: stat - datasource: type: {{ $defaultDatasource }} uid: $ds description: Total amount of used disk space fieldConfig: defaults: color: mode: thresholds mappings: [] thresholds: mode: absolute steps: - color: green value: null unit: bytes overrides: [] gridPos: h: 3 w: 6 x: 6 'y': 4 id: 35 maxDataPoints: 100 options: colorMode: value graphMode: area justifyMode: auto orientation: horizontal percentChangeColorMode: standard reduceOptions: calcs: - lastNotNull fields: '' values: false showPercentChange: false text: {} textMode: auto wideLayout: true pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds exemplar: false expr: sum(vm_data_size_bytes{job=~"$job_storage"}) format: time_series instant: true interval: '' intervalFactor: 1 legendFormat: '' refId: A title: Disk space usage type: stat - datasource: type: {{ $defaultDatasource }} uid: $ds description: Average disk usage per datapoint. fieldConfig: defaults: color: mode: thresholds decimals: 2 mappings: [] thresholds: mode: absolute steps: - color: green value: null unit: bytes overrides: [] gridPos: h: 3 w: 6 x: 12 'y': 4 id: 112 maxDataPoints: 100 options: colorMode: value graphMode: area justifyMode: auto orientation: horizontal percentChangeColorMode: standard reduceOptions: calcs: - lastNotNull fields: '' values: false showPercentChange: false text: {} textMode: auto wideLayout: true pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds exemplar: true expr: sum(vm_data_size_bytes{job=~"$job_storage"}) / sum(vm_rows{job=~"$job_storage", type!~"indexdb.*"}) format: time_series instant: true interval: '' intervalFactor: 1 legendFormat: '' refId: A title: Bytes per point type: stat - datasource: type: {{ $defaultDatasource }} uid: $ds description: Total size of available memory for all VM components. fieldConfig: defaults: color: mode: thresholds mappings: [] thresholds: mode: absolute steps: - color: green value: null unit: bytes overrides: [] gridPos: h: 3 w: 6 x: 18 'y': 4 id: 128 maxDataPoints: 100 options: colorMode: value graphMode: area justifyMode: auto orientation: horizontal percentChangeColorMode: standard reduceOptions: calcs: - lastNotNull fields: '' values: false showPercentChange: false text: {} textMode: auto wideLayout: true pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(vm_available_memory_bytes{job=~"$job", instance=~"$instance"}) format: time_series instant: true interval: '' intervalFactor: 1 legendFormat: '' refId: A title: Available memory type: stat - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: thresholds custom: align: auto cellOptions: type: auto inspect: false minWidth: 50 mappings: [] thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 overrides: - matcher: id: byName options: Time properties: - id: custom.hidden value: true - matcher: id: byName options: Value properties: - id: displayName value: Count gridPos: h: 5 w: 8 x: 0 'y': 7 id: 149 options: cellHeight: sm footer: countRows: false fields: '' reducer: - sum show: false showHeader: true sortBy: - desc: true displayName: Count pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: false expr: sum(vm_app_version{job=~"$job", instance=~"$instance"}) by(job, short_version) format: table instant: true range: false refId: A title: '' type: table - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto axisSoftMin: 0 barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: stepAfter lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: none overrides: [] gridPos: h: 5 w: 16 x: 8 'y': 7 id: 62 options: legend: calcs: - lastNotNull displayMode: table placement: right showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(min_over_time(up{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job) format: time_series instant: false legendFormat: '{{`{{`}}job{{`}}`}}' refId: A title: Uptime ($job) type: timeseries - collapsed: false gridPos: h: 1 w: 24 x: 0 'y': 12 id: 10 panels: [] title: Overview type: row - datasource: type: {{ $defaultDatasource }} uid: $ds description: How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are inserted into cluster per second by protocol before the replication. Check vminsert metrics if there are any issues with ingestion. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineStyle: fill: solid lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: normal thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 13 id: 2 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(rate(vm_rows_inserted_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (type) > 0 interval: '' legendFormat: '{{`{{`}}type{{`}}`}}' range: true refId: A title: Datapoints ingestion rate ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: '* `*` - unsupported query path * `/write` - insert into VM * `/metrics` - query VM system metrics * `/query` - query instant values * `/query_range` - query over a range of time * `/series` - match a certain label set * `/label/{}/values` - query a list of label values (variables mostly)' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 13 id: 6 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_http_requests_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (path) > 0 format: time_series intervalFactor: 1 legendFormat: '{{`{{`}}path{{`}}`}}' range: true refId: A title: Requests rate ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour across all storage nodes. High value may result in ingestion slowdown and high memory usage. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 21 id: 12 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds expr: sum(vm_cache_entries{job=~"$job", instance=~"$instance", type="storage/hour_metric_ids"}) format: time_series intervalFactor: 1 legendFormat: Active time series refId: A title: Active time series ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'The less time it takes is better. * `*` - unsupported query path * `/write` - insert into VM * `/metrics` - query VM system metrics * `/query` - query instant values * `/query_range` - query over a range of time * `/series` - match a certain label set * `/label/{}/values` - query a list of label values (variables mostly)' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: s overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 21 id: 8 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(vm_request_duration_seconds{job=~"$job", instance=~"$instance", quantile="0.99"}) by (path) > 0 format: time_series intervalFactor: 1 legendFormat: '{{`{{`}}path{{`}}`}}' range: true refId: A title: Query duration 0.99 quantile ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: '* `*` - unsupported query path * `/write` - insert into VM * `/metrics` - query VM system metrics * `/query` - query instant values * `/query_range` - query over a range of time * `/series` - match a certain label set * `/label/{}/values` - query a list of label values (variables mostly)' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 29 id: 52 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_http_request_errors_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job, path) > 0 format: time_series intervalFactor: 1 legendFormat: '{{`{{`}}path{{`}}`}} ({{`{{`}}job{{`}}`}})' range: true refId: A title: Requests error rate ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: bars fillOpacity: 100 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=203&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 29 id: 104 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(rate(vm_log_messages_total{job=~"$job",instance=~"$instance", level!="info"}[$__rate_interval])) by (job, level) > 0 format: time_series hide: false interval: 5m intervalFactor: 1 legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}level{{`}}`}}' range: true refId: A title: Logging rate type: timeseries - collapsed: true gridPos: h: 1 w: 24 x: 0 'y': 37 id: 46 panels: - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Percentage of used RSS memory (resident). The RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache). The application''s performance will significantly degrade when memory usage is close to 100%. Click on the line and choose Drilldown to show memory usage per instance' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2293 id: 66 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( max_over_time(process_resident_memory_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job", instance=~"$instance"} ) by(job) interval: '' legendFormat: __auto range: true refId: A title: RSS memory % usage ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2293 id: 64 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( rate(process_cpu_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job", instance=~"$instance"} ) by(job) format: time_series interval: '' intervalFactor: 1 legendFormat: __auto range: true refId: A title: CPU ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed. Safe memory usage % considered to be below 80% Click on the line and choose Drilldown to show memory usage per instance' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2301 id: 138 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( max_over_time(process_resident_memory_anon_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job", instance=~"$instance"} ) by(job) interval: '' legendFormat: __auto range: true refId: A title: RSS anonymous memory % usage ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). The lower the better.' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: s overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2301 id: 219 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(rate(process_pressure_cpu_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job) format: time_series interval: '' intervalFactor: 2 legendFormat: '{{`{{`}}job{{`}}`}} - waiting' range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(rate(process_pressure_cpu_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job) format: time_series hide: false interval: '' intervalFactor: 2 legendFormat: '{{`{{`}}job{{`}}`}} - stalled' range: true refId: B title: CPU pressure type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). The lower the better.' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: s overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2309 id: 220 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(rate(process_pressure_memory_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job) format: time_series interval: '' intervalFactor: 2 legendFormat: '{{`{{`}}job{{`}}`}} - waiting' range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(rate(process_pressure_memory_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job) format: time_series hide: false interval: '' intervalFactor: 2 legendFormat: '{{`{{`}}job{{`}}`}} - stalled' range: true refId: B title: Memory pressure type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the number of bytes read/write from the storage layer. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: Bps overrides: - matcher: id: byRegexp options: /read .*/ properties: - id: custom.transform value: negative-Y gridPos: h: 8 w: 12 x: 12 'y': 2309 id: 122 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(rate(process_io_storage_read_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0 format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: read {{`{{`}}job{{`}}`}} range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(process_io_storage_written_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0 format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: write {{`{{`}}job{{`}}`}} range: true refId: B title: Disk writes/reads ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the percentage of open file descriptors compared to the limit set in the OS. Reaching the limit of open files can cause various issues and must be prevented. See how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 2 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: percentunit overrides: - matcher: id: byRegexp options: /max.*/ properties: - id: color value: fixedColor: '#C4162A' mode: fixed gridPos: h: 8 w: 12 x: 0 'y': 2317 id: 117 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( max_over_time(process_open_fds{job=~"$job", instance=~"$instance"}[$__rate_interval]) / process_max_fds{job=~"$job", instance=~"$instance"} ) by(job) format: time_series interval: '' intervalFactor: 2 legendFormat: '{{`{{`}}job{{`}}`}}' range: true refId: A title: Open FDs usage % ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the number of read/write syscalls such as read, pread, write, pwrite. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: - matcher: id: byRegexp options: /read .*/ properties: - id: custom.transform value: negative-Y gridPos: h: 8 w: 12 x: 12 'y': 2317 id: 204 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(rate(process_io_read_syscalls_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0 format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: read {{`{{`}}job{{`}}`}} range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(process_io_write_syscalls_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0 format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: write {{`{{`}}job{{`}}`}} range: true refId: B title: Disk write/read calls ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2325 id: 68 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(max_over_time(go_goroutines{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) format: time_series interval: '' intervalFactor: 2 legendFormat: __auto range: true refId: A title: Goroutines ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). The lower the better.' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: s overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2325 id: 221 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(rate(process_pressure_io_waiting_seconds_total{job=~"$job"}[$__rate_interval])) by (job) format: time_series interval: '' intervalFactor: 2 legendFormat: '{{`{{`}}job{{`}}`}} - waiting' range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(rate(process_pressure_io_stalled_seconds_total{job=~"$job"}[$__rate_interval])) by (job) format: time_series hide: false interval: '' intervalFactor: 2 legendFormat: '{{`{{`}}job{{`}}`}} - stalled' range: true refId: B title: IO pressure type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2333 id: 70 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(max_over_time(process_num_threads{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) format: time_series interval: '' intervalFactor: 2 legendFormat: __auto range: true refId: A title: Threads ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2333 id: 119 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(max_over_time(vm_tcplistener_conns{job=~"$job", instance=~"$instance"}[$__interval])) by(job) interval: '' legendFormat: '{{`{{`}}job{{`}}`}}' range: true refId: A title: TCP connections ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the percent of CPU spent on garbage collection. If % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage. Try searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ ' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2341 id: 210 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)" format: time_series interval: '' intervalFactor: 2 legendFormat: __auto range: true refId: A title: CPU spent on GC ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2341 id: 120 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(rate(vm_tcplistener_accepts_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) interval: '' legendFormat: __auto range: true refId: A title: TCP connections rate ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: bytes overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2349 id: 218 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(rate(go_memstats_alloc_bytes_total{job=~"$job"}[$__rate_interval])) by (job) format: time_series interval: '' intervalFactor: 2 legendFormat: __auto range: true refId: A title: Memory allocations rate type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing." fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 0.1 unit: s overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2349 id: 213 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (job, instance, le))) by(job) format: time_series interval: '' intervalFactor: 2 legendFormat: __auto range: true refId: A title: Go scheduling latency type: timeseries title: Resource usage ($job) type: row - collapsed: true gridPos: h: 1 w: 24 x: 0 'y': 38 id: 106 panels: - description: '' fieldConfig: defaults: {} overrides: [] gridPos: h: 2 w: 24 x: 0 'y': 1519 id: 211 options: code: language: plaintext showLineNumbers: false showMiniMap: false content: See [Troubleshooting](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) docs. mode: markdown pluginVersion: 11.5.0 title: '' transparent: true type: text - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the rate and total number of new series created over last 24h. High [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM''s or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#cardinality) "explosions". The higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible. To investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format Good references to read: * https://www.robustperception.io/cardinality-is-key * https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: - matcher: id: byName options: new series over 24h properties: - id: custom.axisPlacement value: right - id: custom.axisSoftMin value: 0 gridPos: h: 8 w: 12 x: 0 'y': 1521 id: 102 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds exemplar: true expr: sum(rate(vm_new_timeseries_created_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) interval: '' legendFormat: churn rate refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds exemplar: true expr: sum(increase(vm_new_timeseries_created_total{job=~"$job_storage", instance=~"$instance"}[24h])) hide: false interval: '' legendFormat: new series over 24h refId: B title: Churn rate ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: "The percentage of [slow inserts](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \n\nThe lower the better. \n\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \n* Ingestion of completely new, not seen before time series;\n* [Re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\n* Not enough memory to maintain big enough caches for the current workload.\n\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series). \n\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details." fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line+area links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: transparent value: null - color: red value: 0.1 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 1521 id: 108 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: "max(\n rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval]) \n / rate(vm_rows_added_to_storage_total{job=~\"$job_storage\"}[$__rate_interval])\n)" interval: '' legendFormat: slow inserts range: true refId: A title: Slow inserts type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: bars fillOpacity: 100 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 1529 id: 170 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(increase(vm_assisted_merges_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(type) > 0 format: time_series interval: 5m intervalFactor: 1 legendFormat: __auto range: true refId: A title: Assisted merges ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default. The less value is better.' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto axisSoftMin: 0 barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 1529 id: 107 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: |- sum(rate(vm_slow_queries_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) / sum(rate(vm_http_requests_total{job=~"$job_select", instance=~"$instance", path=~"/select/.*"}[$__rate_interval])) interval: '' legendFormat: slow queries % range: true refId: A title: Slow queries % ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized." fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' mappings: [] thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: percentunit overrides: [] gridPos: h: 9 w: 12 x: 0 'y': 1537 id: 144 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: "max(\n vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} \n /\n vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(type)" interval: '' legendFormat: __auto range: true refId: A title: Cache usage % by type ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows cache miss ratio. Lower is better. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] max: 1 min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: percentunit overrides: [] gridPos: h: 9 w: 12 x: 12 'y': 1537 id: 58 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( rate(vm_cache_misses_total{job=~"$job", instance=~"$instance"}[$__rate_interval]) / rate(vm_cache_requests_total{job=~"$job", instance=~"$instance"}[$__rate_interval]) ) by(type) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: __auto range: true refId: A title: Cache miss ratio ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 1546 id: 142 links: - targetBlank: true title: Readonly mode url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(vm_rpc_vmstorage_is_read_only{job=~"$job_insert", instance=~"$instance"}) by(instance, addr) > 0 format: time_series interval: '' intervalFactor: 1 legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}addr{{`}}`}}' range: true refId: A title: Storage in readonly status for vminsert ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`. Should be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn''t set. Deduplication happens in two places: * `type="merge"` - during [background merges](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage) by vmstorages. Deduplication during merges is permanent. * `type="select"` - during [read queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data) by vmselects. Deduplication happens in-flight and has no permanent effect. The more samples need to be deduplicated, the higher will be resource usage on vmselects or vmstorages.' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 1546 id: 215 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(rate(vm_deduplicated_samples_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, type) >0 format: time_series interval: '' intervalFactor: 1 legendFormat: __auto range: true refId: A title: Deduplication rate ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows how many samples were ignored or dropped on insertion due to various reasons: * timestamp out of retention period or timestamp in future; * invalid metric name; * exceeding limit for labels length or number; * dropped by [relabeling configuration](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#relabeling).' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' mappings: [] thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 1554 id: 135 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(increase(vm_rows_ignored_total{job=~"$job", instance=~"$instance"}[1h])) by (reason) hide: false interval: '' legendFormat: __auto range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(increase(vm_relabel_metrics_dropped_total{job=~"$job", instance=~"$instance"}[1h])) hide: false instant: false legendFormat: relabeling range: true refId: B title: Samples dropped for last 1h ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the number of partial results served by `vmselects`. `vmselect` continues serving queries if at least a single vmstorage nodes is available. It marks responses as `partial` for queries served from the remaining healthy vmstorage nodes. If you prefer consistency over availability then run vmselect nodes with `-search.denyPartialResponse` cmd-line flag. In this case vmselect returns an error if at least a single vmstorage node is unavailable. See more at [cluster availability](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability).' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto axisSoftMin: 0 barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: stepAfter lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: none overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 1554 id: 217 options: legend: calcs: - lastNotNull displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(increase(vm_partial_results_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) > 0 format: time_series instant: false legendFormat: partial results refId: A title: Partial query results ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: thresholds custom: align: auto cellOptions: type: auto inspect: false mappings: [] thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 overrides: - matcher: id: byName options: Value properties: - id: custom.hidden value: true - matcher: id: byName options: Time properties: - id: custom.hidden value: true gridPos: h: 8 w: 12 x: 0 'y': 1562 id: 183 options: cellHeight: sm footer: countRows: false fields: '' reducer: - sum show: false showHeader: true sortBy: - desc: true displayName: job pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: false expr: sum(flag{is_set="true", job=~"$job", instance=~"$instance"}) by(job, instance, name, value) format: table instant: true legendFormat: __auto range: false refId: A title: Non-default flags transformations: - id: groupBy options: fields: instance: aggregations: - uniqueValues operation: aggregate job: aggregations: [] operation: groupby name: aggregations: [] operation: groupby value: aggregations: [] operation: groupby type: table - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn''t restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto axisSoftMin: 0 barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: stepAfter lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: none overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 1562 id: 214 options: legend: calcs: - lastNotNull displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(changes(vm_app_start_timestamp{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job) > 0 format: time_series instant: false legendFormat: '{{`{{`}}job{{`}}`}}' refId: A title: Restarts ($job) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: ${ds} description: 'Shows the amount of data scheduled for [downsampling](https://docs.victoriametrics.com/#downsampling) or [retention filters](https://docs.victoriametrics.com/#retention-filters). Applying downsampling or retention filters happens in background. `vmstorage` schedules this process periodically when finds data partitions that satisfy configured retention or downsampling rules. This process could consume additional CPU and Disk IO resources.' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto axisSoftMin: 0 barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: auto spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' mappings: [] thresholds: mode: absolute steps: - color: green value: null - color: red value: 80 unit: bytes overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 1570 id: 222 options: legend: calcs: [] displayMode: list placement: bottom showLegend: true tooltip: hideZeros: false mode: single sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: ${ds} editorMode: code expr: sum(vm_downsampling_partitions_scheduled_size_bytes{job=~"$job", instance=~"$instance"}) > 0 legendFormat: downsampling range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: ${ds} editorMode: code expr: sum(vm_retention_partitions_scheduled_size_bytes{job=~"$job", instance=~"$instance"}) > 0 hide: false instant: false legendFormat: retention filters range: true refId: B title: Partitions scheduled for re-processing type: timeseries title: Troubleshooting type: row - collapsed: true gridPos: h: 1 w: 24 x: 0 'y': 39 id: 48 panels: - datasource: type: {{ $defaultDatasource }} uid: $ds description: "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor." fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 9 w: 12 x: 0 'y': 4941 id: 76 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_rpc_rows_pushed_total{job=~"$job",instance=~"$instance"}[$__rate_interval])) format: time_series intervalFactor: 1 legendFormat: Pushed range: true refId: B - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_rpc_rows_sent_total{job=~"$job",instance=~"$instance"}[$__rate_interval])) format: time_series intervalFactor: 1 legendFormat: Sent range: true refId: E title: Rows ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Rate of RPC errors by type: * `Connection` - the number of connection errors to vmstorage node * `Dial` - the number of dial errors to vmstorage node. * `Handshake` - the number of handshake errors to vmstorage node * `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 9 w: 12 x: 12 'y': 4941 id: 86 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_rpc_connection_errors_total{job=~"$job",instance=~"$instance"}[$__rate_interval])) format: time_series intervalFactor: 1 legendFormat: Connection range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds expr: sum(rate(vm_rpc_dial_errors_total{job=~"$job",instance=~"$instance"}[$__rate_interval])) format: time_series intervalFactor: 1 legendFormat: Dial refId: B - datasource: type: {{ $defaultDatasource }} uid: $ds expr: sum(rate(vm_rpc_handshake_errors_total{job=~"$job",instance=~"$instance"}[$__rate_interval])) format: time_series intervalFactor: 1 legendFormat: Handshake refId: E title: RPC errors ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: rps overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 4950 id: 80 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(addr) > 0 format: time_series interval: '' intervalFactor: 1 legendFormat: __auto range: true refId: A title: 'Rows ($instance) rerouted to ' type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: rps overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 4950 id: 78 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds exemplar: true expr: sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(addr) > 0 format: time_series interval: '' intervalFactor: 1 legendFormat: '{{`{{`}}addr{{`}}`}}' refId: A title: Rows ($instance) rerouted from type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: The number of rows or bytes that vminsert internal buffer contains at the moment. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: - matcher: id: byName options: bytes properties: - id: unit value: bytes gridPos: h: 8 w: 12 x: 0 'y': 4958 id: 82 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds expr: sum(vm_rpc_buf_pending_bytes{job=~"$job", instance=~"$instance"}) legendFormat: bytes refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds expr: sum(vm_rpc_rows_pending{job=~"$job", instance=~"$instance"}) legendFormat: rows refId: B title: Pending type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Network usage by internal VictoriaMetrics RPC protocol fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: bps overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 4958 id: 74 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcpdialer_written_bytes_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) * 8 legendFormat: network usage range: true refId: A title: RPC network usage ($instance) type: timeseries title: Interconnection ($job) type: row - collapsed: true gridPos: h: 1 w: 24 x: 0 'y': 40 id: 24 panels: - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the number of datapoints ingested into storage nodes per second. This metric doesn't show all stored datapoints since some of them may be dropped because of wrong timestamps or decode errors. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: normal thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2387 id: 100 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_vminsert_metrics_read_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) format: time_series intervalFactor: 1 legendFormat: ingestion rate range: true refId: A title: Ingestion rate ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed. Safe memory usage % considered to be below 80% Click on the line and choose Drilldown to show memory usage per instance' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 0.9 unit: percentunit overrides: [] gridPos: h: 7 w: 12 x: 12 'y': 2387 id: 167 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( max_over_time(process_resident_memory_anon_bytes{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job_storage", instance=~"$instance"} ) format: time_series interval: '' intervalFactor: 1 legendFormat: max range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- min( max_over_time(process_resident_memory_anon_bytes{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job_storage", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: min range: true refId: B - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- quantile(0.5, max_over_time(process_resident_memory_anon_bytes{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job_storage", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: median range: true refId: C title: Memory (anon) usage % ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the number of concurrently executed read requests, where: * `max` - equal to `-search.maxConcurrentRequest` cmd-line flag; * `current` - current number of concurrent select requests executed by storage. Each concurrent select request requires RAM, CPU and disk IO resources during processing. Too many concurrent requests could result into OOM exceptions. When `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row). See more at https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#resource-usage-limits' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: - matcher: id: byName options: max properties: - id: color value: fixedColor: '#C4162A' mode: fixed gridPos: h: 8 w: 12 x: 12 'y': 2394 id: 133 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: "max(\n max_over_time(vm_vmselect_concurrent_requests_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)" interval: '' legendFormat: current range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: min(vm_vmselect_concurrent_requests_capacity{job=~"$job_storage", instance=~"$instance"}) hide: false interval: '' legendFormat: max range: true refId: B title: Concurrent selects ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: '' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 0.9 unit: percentunit overrides: [] gridPos: h: 7 w: 12 x: 0 'y': 2395 id: 151 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: false expr: |- max( rate(process_cpu_seconds_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job_storage", instance=~"$instance"} ) format: time_series interval: '' intervalFactor: 1 legendFormat: max range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: false expr: |- min( rate(process_cpu_seconds_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job_storage", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: min range: true refId: B - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: false expr: |- quantile(0.5, rate(process_cpu_seconds_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job_storage", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: median range: true refId: C title: CPU usage % ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row)." fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: - matcher: id: byName options: max properties: - id: color value: fixedColor: '#C4162A' mode: fixed gridPos: h: 8 w: 12 x: 0 'y': 2402 id: 212 links: - targetBlank: true title: Related discussion url: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: "max(\n max_over_time(vm_concurrent_insert_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)" interval: '' legendFormat: current range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: min(vm_concurrent_insert_capacity{job=~"$job_storage", instance=~"$instance"}) hide: false interval: '' legendFormat: max range: true refId: B title: Concurrent flushes on disk ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: The number of rows merged per second by storage nodes. Merge speed depends on available CPU and disk IO bandwidth. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2402 id: 55 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_rows_merged_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(type) legendFormat: __auto range: true refId: A title: Merge speed type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: "The max number of on-going merges across storage nodes.\n The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.\n It is expected to have high numbers for `storage/small` metric." fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2410 id: 54 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(max_over_time(vm_active_merges{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(type) legendFormat: __auto range: true refId: A title: Active merges ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage). Increase in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization. * `indexdb` - inverted index * `storage/small` - recently added parts of data ingested into storage (hot data) * `storage/big` - small parts gradually merged into bigger parts (cold data)' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2410 id: 22 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(vm_parts{job=~"$job_storage", instance=~"$instance", type=~"indexdb.*"}) by(type) format: time_series interval: '' intervalFactor: 1 legendFormat: __auto range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(vm_last_partition_parts{job=~"$job_storage", instance=~"$instance"}) by(type) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: __auto range: true refId: B title: LSM parts max by type ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=200&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2418 id: 20 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)" format: time_series intervalFactor: 1 legendFormat: max range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: "min(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)" format: time_series hide: false intervalFactor: 1 legendFormat: min range: true refId: B - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: "quantile(0.5,\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)" format: time_series hide: false intervalFactor: 1 legendFormat: median range: true refId: C title: Disk space usage % ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: - matcher: id: byName options: pending index entries properties: - id: unit value: none - id: decimals value: 3 gridPos: h: 8 w: 12 x: 12 'y': 2418 id: 14 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: none pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(max_over_time(vm_pending_rows{job=~"$job_storage", instance=~"$instance", type="storage"}[$__rate_interval])) format: time_series hide: false intervalFactor: 1 legendFormat: pending datapoints range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(max_over_time(vm_pending_rows{job=~"$job_storage", instance=~"$instance", type="indexdb"}[$__rate_interval])) format: time_series hide: false intervalFactor: 1 legendFormat: pending index entries range: true refId: B title: Pending datapoints ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly. The sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=201&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 2426 id: 202 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)" format: time_series intervalFactor: 1 legendFormat: indexdb range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)" format: time_series hide: false intervalFactor: 1 legendFormat: datapoints range: true refId: B title: Disk space usage % by type ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Value > 0 means vmstorage is in readonly mode. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2426 id: 141 links: - targetBlank: true title: Readonly mode url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode options: legend: calcs: [] displayMode: list placement: bottom showLegend: false tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: vm_storage_is_read_only{job=~"$job_storage", instance=~"$instance"} > 0 interval: '' legendFormat: '{{`{{`}} instance {{`}}`}}' range: true refId: A title: Readonly mode type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows network usage by vmstorage services. * Writes show traffic sent to clients. * Reads show traffic received from clients.' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: bps overrides: - matcher: id: byRegexp options: /read.*/ properties: - id: custom.transform value: negative-Y gridPos: h: 8 w: 12 x: 0 'y': 2434 id: 206 options: legend: calcs: - mean - lastNotNull displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcplistener_read_bytes_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(name) * 8 > 0 format: time_series intervalFactor: 1 legendFormat: read from {{`{{`}}name{{`}}`}} range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcplistener_written_bytes_total{job=~"$job_storage", instance=~"$instance"}[$__rate_interval])) by(name) * 8 > 0 format: time_series hide: false intervalFactor: 1 legendFormat: write to {{`{{`}}name{{`}}`}} range: true refId: B title: Network usage ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the max number of existing [snapshots](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots) across vmstorages. Recently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space. It is recommended deleting old snapshots when they are no longer needed to free up disk space.' fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 2434 id: 216 options: legend: calcs: [] displayMode: list placement: bottom showLegend: false tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: max(vm_snapshots{job=~"$job_storage", instance=~"$instance"}) interval: '' legendFormat: snapshots range: true refId: A title: Number of snapshots type: timeseries title: vmstorage ($instance) type: row - collapsed: true gridPos: h: 1 w: 24 x: 0 'y': 41 id: 42 panels: - datasource: type: {{ $defaultDatasource }} uid: $ds description: Request rate accepted by vmselect nodes fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 4954 id: 92 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_http_requests_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (path) > 0 format: time_series intervalFactor: 1 legendFormat: __auto range: true refId: A title: Requests rate ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the max number of concurrent selects across instances. * `max` limit can be configured via `search.maxConcurrentRequests` flag * `current` shows the current number of goroutines busy with processing requests When `current` hits `max` constantly, it means one or more vmselect nodes are overloaded with number of requests. If you observe that CPU for vmselects is saturated, consider adding more vmselect replicas or increase CPU resources. If CPU and Memory panels show a plenty of free resources - try increasing `-search.maxConcurrentRequests`. Please note, the higher is `-search.maxConcurrentRequests`, the higher could be [peak memory usage](https://docs.victoriametrics.com/victoriametrics/troubleshooting/#out-of-memory-errors).' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: - matcher: id: byName options: max properties: - id: color value: fixedColor: '#C4162A' mode: fixed - id: custom.fillOpacity value: 0 gridPos: h: 8 w: 12 x: 12 'y': 4954 id: 95 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(max_over_time(vm_concurrent_select_current{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) format: time_series interval: '' intervalFactor: 1 legendFormat: current range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds expr: min(vm_concurrent_select_capacity{job=~"$job_select", instance=~"$instance"}) format: time_series intervalFactor: 1 legendFormat: max refId: B title: Concurrent selects ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 0.9 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 4962 id: 163 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( rate(process_cpu_seconds_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job_select", instance=~"$instance"} ) format: time_series interval: '' intervalFactor: 1 legendFormat: max range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- min( rate(process_cpu_seconds_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job_select", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: min range: true refId: B - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- quantile(0.5, rate(process_cpu_seconds_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job_select", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: median range: true refId: C title: CPU usage % ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed. Safe memory usage % considered to be below 80% Click on the line and choose Drilldown to show memory usage per instance' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 0.9 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 4962 id: 165 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( max_over_time(process_resident_memory_anon_bytes{job=~"$job_select", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job_select", instance=~"$instance"} ) format: time_series interval: '' intervalFactor: 1 legendFormat: max range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- min( max_over_time(process_resident_memory_anon_bytes{job=~"$job_select", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job_select", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: min range: true refId: B - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- quantile(0.5, max_over_time(process_resident_memory_anon_bytes{job=~"$job_select", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job_select", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: median range: true refId: C title: Memory (anon) usage % ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 99th percentile of the number of time series read per query. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 2 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 4970 id: 178 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange))) format: time_series interval: '' intervalFactor: 1 legendFormat: series range: true refId: A title: Series read per query ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per queried time series. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 2 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 4970 id: 180 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange))) format: time_series interval: '' intervalFactor: 1 legendFormat: datapoints range: true refId: A title: Datapoints read per series ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per query. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 2 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 4978 id: 179 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange))) format: time_series interval: '' intervalFactor: 1 legendFormat: datapoints range: true refId: A title: Datapoints read per query ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: '99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scanner per query. This number can exceed number of DatapointsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 2 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 4978 id: 181 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange))) format: time_series interval: '' intervalFactor: 1 legendFormat: datapoints range: true refId: A title: Datapoints scanned per query ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: bps overrides: - matcher: id: byRegexp options: /read.*/ properties: - id: custom.transform value: negative-Y gridPos: h: 8 w: 12 x: 0 'y': 4986 id: 93 options: legend: calcs: - mean - lastNotNull displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcplistener_read_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0 format: time_series intervalFactor: 1 legendFormat: read from client range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcplistener_written_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0 format: time_series hide: false intervalFactor: 1 legendFormat: write to client range: true refId: B title: 'Network usage: clients ($instance)' type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows network usage between vmselects and vmstorages. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: bps overrides: - matcher: id: byRegexp options: /read.*/ properties: - id: custom.transform value: negative-Y gridPos: h: 8 w: 12 x: 12 'y': 4986 id: 207 options: legend: calcs: - mean - lastNotNull displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcpdialer_read_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0 format: time_series intervalFactor: 1 legendFormat: read from vmstorage range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcpdialer_written_bytes_total{job=~"$job_select", instance=~"$instance"}[$__rate_interval])) * 8 > 0 format: time_series hide: false intervalFactor: 1 legendFormat: write to vmstorage range: true refId: B title: 'Network usage: vmstorage ($instance)' type: timeseries title: vmselect ($instance) type: row - collapsed: true gridPos: h: 1 w: 24 x: 0 'y': 42 id: 40 panels: - datasource: type: {{ $defaultDatasource }} uid: $ds description: '* `*` - unsupported query path * `/write` - insert into VM * `/metrics` - query VM system metrics * `/query` - query instant values * `/query_range` - query over a range of time * `/series` - match a certain label set * `/label/{}/values` - query a list of label values (variables mostly)' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 4955 id: 97 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_http_requests_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) by (path) > 0 format: time_series intervalFactor: 1 legendFormat: __auto range: true refId: A title: Requests rate ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the max number of ongoing insertions. * `max` - equal to number of CPU * 2 by default. May be configured with `maxConcurrentInserts` flag; * `current` - current number of goroutines busy with processing requests. `-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate. When `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 10 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: - matcher: id: byName options: max properties: - id: color value: fixedColor: '#C4162A' mode: fixed - id: custom.fillOpacity value: 0 gridPos: h: 8 w: 12 x: 12 'y': 4955 id: 99 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(max_over_time(vm_concurrent_insert_current{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) format: time_series interval: '' intervalFactor: 1 legendFormat: current range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds exemplar: true expr: min(vm_concurrent_insert_capacity{job=~"$job_insert", instance=~"$instance"}) format: time_series interval: '' intervalFactor: 1 legendFormat: max refId: B title: Concurrent inserts ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 0.9 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 4963 id: 185 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( rate(process_cpu_seconds_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job_insert", instance=~"$instance"} ) format: time_series interval: '' intervalFactor: 1 legendFormat: max range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- min( rate(process_cpu_seconds_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job_insert", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: min range: true refId: B - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- quantile(0.5, rate(process_cpu_seconds_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval]) / process_cpu_cores_available{job=~"$job_insert", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: median range: true refId: C title: CPU usage % ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed. Safe memory usage % considered to be below 80% Click on the line and choose Drilldown to show memory usage per instance' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: - targetBlank: true title: Drilldown url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range} mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 0.9 unit: percentunit overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 4963 id: 187 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- max( max_over_time(process_resident_memory_anon_bytes{job=~"$job_insert", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job_insert", instance=~"$instance"} ) format: time_series interval: '' intervalFactor: 1 legendFormat: max range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- min( max_over_time(process_resident_memory_anon_bytes{job=~"$job_insert", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job_insert", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: min range: true refId: B - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: |- quantile(0.5, max_over_time(process_resident_memory_anon_bytes{job=~"$job_insert", instance=~"$instance"}[$__rate_interval]) / vm_available_memory_bytes{job=~"$job_insert", instance=~"$instance"} ) format: time_series hide: false interval: '' intervalFactor: 1 legendFormat: median range: true refId: C title: Memory (anon) usage % ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: "Shows the saturation level of connection between vminsert and vmstorage components. \n\nIf the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that either vminsert or vmstorage nodes are struggling with the load. Verify CPU/mem saturation of both components and network saturation between them.\nIf vminsert resources are saturated - consider adding more resources or scale vminserts horizontally.\n\nIf vminsert resources and network are fine, check vmstorage metrics for anomalies." fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line+area decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: transparent - color: red value: 0.9 unit: s overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 4971 id: 139 options: legend: calcs: - mean - lastNotNull displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: max(rate(vm_rpc_send_duration_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(addr) format: time_series interval: '' intervalFactor: 1 legendFormat: __auto range: true refId: A title: Storage connection saturation ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows when vmstorage node is unreachable for vminsert. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 0 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 4971 id: 114 options: legend: calcs: - mean - lastNotNull displayMode: table placement: bottom showLegend: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: vm_rpc_vmstorage_is_reachable{job=~"$job", instance=~"$instance"} != 1 format: time_series interval: '' intervalFactor: 1 legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}addr{{`}}`}}' range: true refId: A title: Storage reachability ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows network usage between vminserts and clients, such as vmagent, Prometheus, or any other client pushing metrics to vminsert. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: bps overrides: - matcher: id: byRegexp options: /read.*/ properties: - id: custom.transform value: negative-Y gridPos: h: 8 w: 12 x: 0 'y': 4979 id: 208 options: legend: calcs: - mean - lastNotNull displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcplistener_read_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0 format: time_series intervalFactor: 1 legendFormat: read from client range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcplistener_written_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0 format: time_series hide: false intervalFactor: 1 legendFormat: write to client range: true refId: B title: 'Network usage: clients ($instance)' type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows network usage between vminserts and vmstorages. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: bps overrides: - matcher: id: byRegexp options: /read.*/ properties: - id: custom.transform value: negative-Y gridPos: h: 8 w: 12 x: 12 'y': 4979 id: 209 options: legend: calcs: - mean - lastNotNull displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcpdialer_read_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0 format: time_series intervalFactor: 1 legendFormat: read from vmstorage range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(rate(vm_tcpdialer_written_bytes_total{job=~"$job_insert", instance=~"$instance"}[$__rate_interval])) * 8 > 0 format: time_series hide: false intervalFactor: 1 legendFormat: write to vmstorage range: true refId: B title: 'Network usage: vmstorage ($instance)' type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: '' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' decimals: 2 links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 4987 id: 88 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~"$job", instance=~"$instance"}[$__rate_interval])) by (instance, vmrange))) format: time_series interval: '' intervalFactor: 1 legendFormat: max range: true refId: A title: Rows per insert ($instance) type: timeseries title: vminsert ($instance) type: row - collapsed: true gridPos: h: 1 w: 24 x: 0 'y': 43 id: 194 panels: - fieldConfig: defaults: {} overrides: [] gridPos: h: 2 w: 24 x: 0 'y': 5004 id: 198 options: code: language: plaintext showLineNumbers: false showMiniMap: false content: Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance. mode: markdown pluginVersion: 9.2.7 title: '' transparent: true type: text - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the used memory (resident). The application''s performance will significantly degrade when memory usage is close to 100%.' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: decbytes overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 5006 id: 189 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: max_over_time(process_resident_memory_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval]) interval: '' legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})' range: true refId: A title: RSS memory usage ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: decbytes overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 5006 id: 190 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: max_over_time(process_resident_memory_anon_bytes{job=~"$job", instance=~"$instance"}[$__rate_interval]) interval: '' legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})' range: true refId: A title: RSS anonymous memory usage ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: '' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: short overrides: [] gridPos: h: 7 w: 12 x: 0 'y': 5014 id: 192 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: false expr: sum(rate(process_cpu_seconds_total{job=~"$job", instance=~"$instance"}[$__rate_interval])) by(job, instance) format: time_series interval: '' intervalFactor: 1 legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})' range: true refId: A title: CPU usage ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: 'Shows the approx time needed to reach 100% of disk capacity based on the following params: * free disk space (after -storage.minFreeDiskSpaceBytes); * row ingestion rate; * compression.' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: s overrides: [] gridPos: h: 7 w: 12 x: 12 'y': 5014 id: 196 options: legend: calcs: - mean - lastNotNull - min displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: none pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: "(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n (rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) - \n sum(rate(vm_deduplicated_samples_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])) without (type)) * \n (\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type) /\n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type)\n )\n +\n rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) * \n (\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"}) /\n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"})\n )\n) > 0" format: time_series interval: '' intervalFactor: 1 legendFormat: '{{`{{`}}instance{{`}}`}}' range: true refId: A title: Storage full ETA ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance. fieldConfig: defaults: color: mode: palette-classic custom: axisBorderShow: false axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 barWidthFactor: 0.6 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false insertNulls: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: decbytes overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 5021 id: 200 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: hideZeros: false mode: multi sort: desc pluginVersion: 11.5.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(vm_data_size_bytes{job=~"$job", instance=~"$instance"}) by(job, instance) format: time_series intervalFactor: 1 legendFormat: '' range: true refId: A title: Disk space usage ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: '' fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: line fillOpacity: 0 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: line links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green unit: decbytes overrides: [] gridPos: h: 8 w: 12 x: 12 'y': 5021 id: 201 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(vm_data_size_bytes{job=~"$job", instance=~"$instance", type=~"indexdb.*"}) by(job, instance) format: time_series intervalFactor: 1 legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (indexdb)' range: true refId: A - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code expr: sum(vm_data_size_bytes{job=~"$job", instance=~"$instance", type!~"indexdb.*"}) by(job, instance) format: time_series hide: false intervalFactor: 1 legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (datapoints)' range: true refId: B title: Disk space usage by type ($instance) type: timeseries - datasource: type: {{ $defaultDatasource }} uid: $ds description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs. fieldConfig: defaults: color: mode: palette-classic custom: axisCenteredZero: false axisColorMode: text axisLabel: '' axisPlacement: auto barAlignment: 0 drawStyle: bars fillOpacity: 100 gradientMode: none hideFrom: legend: false tooltip: false viz: false lineInterpolation: linear lineWidth: 1 pointSize: 5 scaleDistribution: type: linear showPoints: never spanNulls: false stacking: group: A mode: none thresholdsStyle: mode: 'off' links: [] mappings: [] min: 0 thresholds: mode: absolute steps: - color: green - color: red value: 80 unit: short overrides: [] gridPos: h: 8 w: 12 x: 0 'y': 5029 id: 203 options: legend: calcs: - mean - lastNotNull - max displayMode: table placement: bottom showLegend: true sortBy: Last * sortDesc: true tooltip: mode: multi sort: desc pluginVersion: 9.1.0 targets: - datasource: type: {{ $defaultDatasource }} uid: $ds editorMode: code exemplar: true expr: sum(rate(vm_log_messages_total{job=~"$job",instance=~"$instance", level!="info"}[$__rate_interval])) by (job, instance, level, location) > 0 format: time_series hide: false interval: 5m intervalFactor: 1 legendFormat: __auto range: true refId: A title: Logging rate type: timeseries title: Drilldown type: row preload: false refresh: '' schemaVersion: 40 tags: - victoriametrics - vm-k8s-stack templating: list: - current: text: VictoriaMetrics - cluster value: PAF93674D0B4E9963 includeAll: false name: ds options: [] query: {{ $defaultDatasource }} refresh: 1 regex: '' type: datasource - current: {} datasource: type: prometheus uid: $ds definition: label_values(vm_app_version{version=~"^vm(insert|select|storage).*"}, job) includeAll: true multi: true name: job options: [] query: query: label_values(vm_app_version{version=~"^vm(insert|select|storage).*"}, job) refId: VictoriaMetrics-job-Variable-Query refresh: 1 regex: '' type: query - current: {} datasource: type: prometheus uid: $ds definition: label_values(vm_app_version{job=~"$job", version=~"^vminsert.*"}, job) hide: 2 includeAll: true multi: true name: job_insert options: [] query: query: label_values(vm_app_version{job=~"$job", version=~"^vminsert.*"}, job) refId: StandardVariableQuery refresh: 1 regex: '' type: query - current: {} datasource: type: prometheus uid: $ds definition: label_values(vm_app_version{job=~"$job", version=~"^vmselect.*"}, job) hide: 2 includeAll: true multi: true name: job_select options: [] query: query: label_values(vm_app_version{job=~"$job", version=~"^vmselect.*"}, job) refId: StandardVariableQuery refresh: 1 regex: '' type: query - current: {} datasource: type: prometheus uid: $ds definition: label_values(vm_app_version{job=~"$job", version=~"^vmstorage.*"}, job) hide: 2 includeAll: true multi: true name: job_storage options: [] query: query: label_values(vm_app_version{job=~"$job", version=~"^vmstorage.*"}, job) refId: StandardVariableQuery refresh: 1 regex: '' type: query - allValue: .* current: {} datasource: type: prometheus uid: $ds definition: label_values(vm_app_version{job=~"$job"}, instance) includeAll: true multi: true name: instance options: [] query: query: label_values(vm_app_version{job=~"$job"}, instance) refId: VictoriaMetrics-instance-Variable-Query refresh: 1 regex: '' type: query - baseFilters: [] datasource: type: prometheus uid: $ds filters: [] name: adhoc type: adhoc time: from: now-3h to: now timepicker: refresh_intervals: - 10s - 30s - 1m - 5m - 15m - 30m - 1h - 2h - 1d timezone: {{ default "utc" ($Values.defaultDashboards).defaultTimezone }} title: VictoriaMetrics - cluster uid: oS7Bi_0Wz version: 1 weekStart: ''