{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "PROMETHEUS", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "7.4.5" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": false, "gnetId": null, "graphTooltip": 0, "id": null, "iteration": 1625069717503, "links": [ { "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "vendor=crunchydata" ], "type": "dashboards" } ], "panels": [ { "aliasColors": { "% Throttled": "yellow", "% Used": "blue", "Limit": "red", "Process count": "blue" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "PROMETHEUS", "fieldConfig": { "defaults": { "color": {}, "custom": {}, "thresholds": { "mode": "absolute", "steps": [] }, "unit": "short" }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, "hiddenSeries": false, "id": 11, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.5", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Limit", "dashes": true } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg((idelta(ccp_nodemx_cpuacct_usage{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}[30s])/1000000000)/idelta(ccp_nodemx_cpuacct_usage_ts{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}[30s])*100) without(instance,ip,role)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "% Usage", "refId": "A" }, { "expr": "avg((ccp_nodemx_cpucfs_quota_us{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}*100/ccp_nodemx_cpucfs_period_us{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"})) without (instance,ip,role)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Limit", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Percent", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "Process count", "logBase": 1, "max": null, "min": "0", "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "% Throttled": "yellow", "% Used": "blue", "Limit": "red", "Process count": "blue" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "PROMETHEUS", "fieldConfig": { "defaults": { "color": {}, "custom": {}, "thresholds": { "mode": "absolute", "steps": [] }, "unit": "short" }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, "hiddenSeries": false, "id": 15, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.5", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Limit", "dashes": true }, { "alias": "Process count", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(ccp_nodemx_process_count{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Process count", "refId": "B" }, { "expr": "avg((idelta(ccp_nodemx_cpustat_throttled_time{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}[30s])/1000000000)/idelta(ccp_nodemx_cpustat_snap_ts{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}[30s])*100) without (instance,ip,role)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "% Throttled", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Throttle", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "Process count", "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Inactive anon": "super-light-purple", "Limit": "red", "Mem free": "green", "Request": "blue" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "PROMETHEUS", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, "hiddenSeries": false, "id": 6, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.5", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "Limit", "dashes": true }, { "alias": "Request", "dashes": true } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(ccp_nodemx_mem_limit{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Limit", "refId": "A" }, { "expr": "avg(ccp_nodemx_mem_request{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Request", "refId": "B" }, { "expr": "avg(ccp_nodemx_mem_usage_in_bytes{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Usage", "refId": "J" }, { "expr": "avg(clamp_min((ccp_nodemx_mem_limit{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"} - ccp_nodemx_mem_usage_in_bytes{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}),0)) without (instance,ip,role)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Free", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "PROMETHEUS", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 6 }, "hiddenSeries": false, "id": 8, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.5", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/tx bytes/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(rate(ccp_nodemx_network_rx_bytes{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\", interface!=\"tunl0\"}[1m])) without(instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{interface}} - rx bytes", "refId": "A" }, { "expr": "avg(rate(ccp_nodemx_network_tx_bytes{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\", interface!=\"tunl0\"}[1m])) without(instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{interface}} - tx bytes", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Network", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "PROMETHEUS", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 6 }, "hiddenSeries": false, "id": 10, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.5", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg((ccp_nodemx_data_disk_total_bytes{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\"}-ccp_nodemx_data_disk_available_bytes{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\"})*100/ccp_nodemx_data_disk_total_bytes{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{mount_point}}", "refId": "A" }, { "expr": "avg((ccp_nodemx_data_disk_total_file_nodes{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\"}-ccp_nodemx_data_disk_free_file_nodes{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\"})*100/ccp_nodemx_data_disk_total_file_nodes{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\"}) without(instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{mount_point}} - Inodes", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": "100", "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "PROMETHEUS", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 6 }, "hiddenSeries": false, "id": 12, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.5", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Reads/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(rate(ccp_nodemx_disk_activity_sectors_read{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}[1m])*512) without(instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{mount_point}} - Reads", "refId": "A" }, { "expr": "avg(rate(ccp_nodemx_disk_activity_sectors_written{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}[1m])*512) without(instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{mount_point}} - Writes ", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Activity", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Inactive anon": "super-light-purple", "Limit": "red", "Request": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "PROMETHEUS", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 14, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 150, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.5", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(ccp_nodemx_mem_cache{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Cached", "refId": "C" }, { "expr": "avg(ccp_nodemx_mem_dirty{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Dirty", "refId": "D" }, { "expr": "avg(ccp_nodemx_mem_shmem{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "shared mem", "refId": "E" }, { "expr": "avg(ccp_nodemx_mem_rss{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "RSS", "refId": "F" }, { "expr": "avg(ccp_nodemx_mem_mapped_file{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Mapped file", "refId": "G" }, { "expr": "avg(ccp_nodemx_mem_kmem_usage_in_bytes{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "kmem", "refId": "H" }, { "expr": "avg(ccp_nodemx_mem_inactive_anon{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Inactive anon", "refId": "I" }, { "expr": "avg(ccp_nodemx_mem_active_file{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Active file", "refId": "J" }, { "expr": "avg(ccp_nodemx_mem_inactive_file{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Inactive file", "refId": "K" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Breakdown", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "CPU limit": "red", "CPU request": "blue", "Memory limit": "dark-red", "Memory request": "dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "PROMETHEUS", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.5", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "CPU request", "yaxis": 2 }, { "alias": "CPU limit", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(ccp_nodemx_cpu_limit{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "CPU limit", "refId": "A" }, { "expr": "avg(ccp_nodemx_cpu_request{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "CPU request", "refId": "B" }, { "expr": "avg(ccp_nodemx_mem_limit{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Memory limit", "refId": "C" }, { "expr": "avg(ccp_nodemx_mem_request{pg_cluster=\"[[cluster]]\",pod=\"[[pod]]\"}) without (instance,ip,role)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Memory request", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Container resources", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "CPU (millicores)", "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "5s", "schemaVersion": 27, "style": "dark", "tags": [ "vendor=crunchydata" ], "templating": { "list": [ { "allValue": null, "current": {}, "datasource": "PROMETHEUS", "definition": "label_values(pg_cluster)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "cluster", "multi": false, "name": "cluster", "options": [], "query": { "query": "label_values(pg_cluster)", "refId": "PROMETHEUS-cluster-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "PROMETHEUS", "definition": "label_values({pg_cluster=\"[[cluster]]\"},pod)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "pod", "multi": false, "name": "pod", "options": [], "query": { "query": "label_values({pg_cluster=\"[[cluster]]\"},pod)", "refId": "PROMETHEUS-pod-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": { "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "POD Details", "uid": "4auP6Mk7k", "version": 1 }