diff --git a/dashboards/victoriatraces-cluster.json b/dashboards/victoriatraces-cluster.json index 7b711831c..bbd7188e8 100644 --- a/dashboards/victoriatraces-cluster.json +++ b/dashboards/victoriatraces-cluster.json @@ -4341,6 +4341,110 @@ ], "title": "Trace spans dropped for last 1h", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 454 + }, + "id": 159, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": true, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Fsync avg duration ($instance)", + "type": "timeseries" } ], "title": "Troubleshooting", diff --git a/dashboards/victoriatraces.json b/dashboards/victoriatraces.json index b3970fd2b..8a8803c97 100644 --- a/dashboards/victoriatraces.json +++ b/dashboards/victoriatraces.json @@ -4015,6 +4015,110 @@ ], "title": "Trace spans dropped for last 1h", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 454 + }, + "id": 159, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": true, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Fsync avg duration ($instance)", + "type": "timeseries" } ], "title": "Troubleshooting", diff --git a/dashboards/vm/victoriatraces-cluster.json b/dashboards/vm/victoriatraces-cluster.json index 1d3bb2040..bcbb9ad04 100644 --- a/dashboards/vm/victoriatraces-cluster.json +++ b/dashboards/vm/victoriatraces-cluster.json @@ -4342,6 +4342,110 @@ ], "title": "Trace spans dropped for last 1h", "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "$ds" + }, + "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 454 + }, + "id": 159, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": true, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Fsync avg duration ($instance)", + "type": "timeseries" } ], "title": "Troubleshooting", diff --git a/dashboards/vm/victoriatraces.json b/dashboards/vm/victoriatraces.json index bc996b5f7..eaef78b7d 100644 --- a/dashboards/vm/victoriatraces.json +++ b/dashboards/vm/victoriatraces.json @@ -4016,6 +4016,110 @@ ], "title": "Trace spans dropped for last 1h", "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "$ds" + }, + "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 454 + }, + "id": 159, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": true, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Fsync avg duration ($instance)", + "type": "timeseries" } ], "title": "Troubleshooting", diff --git a/docs/victoriatraces/changelog/CHANGELOG.md b/docs/victoriatraces/changelog/CHANGELOG.md index 616753a90..6ba402725 100644 --- a/docs/victoriatraces/changelog/CHANGELOG.md +++ b/docs/victoriatraces/changelog/CHANGELOG.md @@ -12,6 +12,8 @@ The following `tip` changes can be tested by building VictoriaTraces components ## tip +* FEATURE: [dashboards/single-node](https://grafana.com/grafana/dashboards/24136), [dashboards/cluster](https://grafana.com/grafana/dashboards/24134): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, and cluster dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [VictoriaMetrics#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432). + ## [v0.10.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.10.0) Released at 2026-07-22