4 "name": "DS_PROMETHEUS",
6 "description": "A prometheus server with prometheus server metrics",
8 "pluginId": "prometheus",
9 "pluginName": "Prometheus"
17 "version": "4.5.0-beta1"
47 "datasource": "${DS_PROMETHEUS}",
49 "expr": "sum(changes(prometheus_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}[10m])) by (instance)",
51 "iconColor": "rgb(0, 96, 19)",
59 "datasource": "${DS_PROMETHEUS}",
61 "expr": "count(sum(up{instance=\"$instance\"}) by (instance) < 1)",
63 "iconColor": "rgba(255, 96, 96, 1)",
72 "description": "Get started faster with Grafana Cloud then easily build these dashboards. https://grafana.com/products/cloud/\nOverview of metrics from Prometheus 2.0. \nUseful for using prometheus to monitor your prometheus.\nRevisions welcome!",
76 "hideControls": false,
87 "colorBackground": false,
90 "rgba(245, 54, 54, 0.9)",
91 "rgba(237, 129, 40, 0.89)",
92 "rgba(50, 172, 45, 0.97)"
94 "datasource": "${DS_PROMETHEUS}",
96 "description": "Percentage of uptime during the most recent $interval period. Change the period with the 'interval' dropdown above.",
102 "thresholdLabels": false,
103 "thresholdMarkers": false
111 "name": "value to text",
115 "name": "range to text",
119 "maxDataPoints": 100,
120 "nullPointMode": "connected",
123 "postfixFontSize": "100%",
125 "prefixFontSize": "50%",
135 "fillColor": "rgba(31, 118, 189, 0.18)",
137 "lineColor": "rgb(31, 120, 193)",
143 "expr": "avg(avg_over_time(up{instance=~\"$instance\",job=~\"$job\"}[$interval]) * 100)",
144 "format": "time_series",
151 "thresholds": "90, 99",
152 "title": "Uptime [$interval]",
153 "type": "singlestat",
154 "valueFontSize": "100%",
162 "valueName": "current"
166 "datasource": "${DS_PROMETHEUS}",
167 "description": "Servers which are DOWN RIGHT NOW! \nFIX THEM!!",
169 "hideTimeOverride": true,
183 "dateFormat": "YYYY-MM-DD HH:mm:ss",
191 "rgba(245, 54, 54, 0.9)",
192 "rgba(237, 129, 40, 0.89)",
193 "rgba(50, 172, 45, 0.97)"
195 "dateFormat": "YYYY-MM-DD HH:mm:ss",
197 "pattern": "/__name__|job|Value/",
206 "rgba(255, 0, 0, 0.9)",
207 "rgba(237, 129, 40, 0.89)",
208 "rgba(255, 0, 0, 0.97)"
210 "dateFormat": "YYYY-MM-DD HH:mm:ss",
213 "pattern": "instance",
225 "expr": "up{instance=~\"$instance\",job=~\"$job\"} < 1",
233 "title": "Currently Down",
234 "transform": "table",
238 "cacheTimeout": null,
239 "colorBackground": false,
242 "rgba(50, 172, 45, 0.97)",
243 "rgba(237, 129, 40, 0.89)",
244 "rgba(245, 54, 54, 0.9)"
246 "datasource": "${DS_PROMETHEUS}",
247 "description": "Total number of time series in prometheus",
253 "thresholdLabels": false,
254 "thresholdMarkers": true
262 "name": "value to text",
266 "name": "range to text",
270 "maxDataPoints": 100,
271 "nullPointMode": "connected",
274 "postfixFontSize": "50%",
276 "prefixFontSize": "50%",
286 "fillColor": "rgba(31, 118, 189, 0.18)",
288 "lineColor": "rgb(31, 120, 193)",
294 "expr": "sum(prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"})",
295 "format": "time_series",
301 "thresholds": "1000000,2000000",
302 "title": "Total Series",
303 "type": "singlestat",
304 "valueFontSize": "100%",
312 "valueName": "current"
315 "cacheTimeout": null,
316 "colorBackground": false,
319 "rgba(245, 54, 54, 0.9)",
320 "rgba(237, 129, 40, 0.89)",
321 "rgba(50, 172, 45, 0.97)"
323 "datasource": "${DS_PROMETHEUS}",
329 "thresholdLabels": false,
330 "thresholdMarkers": true
338 "name": "value to text",
342 "name": "range to text",
346 "maxDataPoints": 100,
347 "nullPointMode": "connected",
350 "postfixFontSize": "50%",
352 "prefixFontSize": "50%",
362 "fillColor": "rgba(31, 118, 189, 0.18)",
364 "lineColor": "rgb(31, 120, 193)",
370 "expr": "sum(prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"})",
371 "format": "time_series",
378 "title": "Memory Chunks",
379 "type": "singlestat",
380 "valueFontSize": "100%",
388 "valueName": "current"
392 "repeatIteration": null,
395 "title": "at a glance",
403 "cacheTimeout": null,
404 "colorBackground": false,
407 "rgba(50, 172, 45, 0.97)",
408 "rgba(237, 129, 40, 0.89)",
409 "rgba(245, 54, 54, 0.9)"
411 "datasource": "${DS_PROMETHEUS}",
412 "description": "The total number of rule group evaluations missed due to slow rule group evaluation.",
418 "thresholdLabels": false,
419 "thresholdMarkers": true
427 "name": "value to text",
431 "name": "range to text",
435 "maxDataPoints": 100,
436 "nullPointMode": "connected",
439 "postfixFontSize": "50%",
441 "prefixFontSize": "50%",
451 "fillColor": "rgba(31, 118, 189, 0.18)",
453 "lineColor": "rgb(31, 120, 193)",
459 "expr": "sum(sum_over_time(prometheus_evaluator_iterations_missed_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))",
460 "format": "time_series",
466 "thresholds": "1,10",
467 "title": "Missed Iterations [$interval]",
468 "type": "singlestat",
469 "valueFontSize": "100%",
477 "valueName": "current"
480 "cacheTimeout": null,
481 "colorBackground": false,
484 "rgba(50, 172, 45, 0.97)",
485 "rgba(237, 129, 40, 0.89)",
486 "rgba(245, 54, 54, 0.9)"
488 "datasource": "${DS_PROMETHEUS}",
489 "description": "The total number of rule group evaluations skipped due to throttled metric storage.",
495 "thresholdLabels": false,
496 "thresholdMarkers": true
504 "name": "value to text",
508 "name": "range to text",
512 "maxDataPoints": 100,
513 "nullPointMode": "connected",
516 "postfixFontSize": "50%",
518 "prefixFontSize": "50%",
528 "fillColor": "rgba(31, 118, 189, 0.18)",
530 "lineColor": "rgb(31, 120, 193)",
536 "expr": "sum(sum_over_time(prometheus_evaluator_iterations_skipped_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))",
537 "format": "time_series",
543 "thresholds": "1,10",
544 "title": "Skipped Iterations [$interval]",
545 "type": "singlestat",
546 "valueFontSize": "100%",
554 "valueName": "current"
557 "cacheTimeout": null,
558 "colorBackground": false,
561 "rgba(50, 172, 45, 0.97)",
562 "rgba(237, 129, 40, 0.89)",
563 "rgba(245, 54, 54, 0.9)"
565 "datasource": "${DS_PROMETHEUS}",
566 "description": "Total number of scrapes that hit the sample limit and were rejected.",
572 "thresholdLabels": false,
573 "thresholdMarkers": true
581 "name": "value to text",
585 "name": "range to text",
589 "maxDataPoints": 100,
590 "nullPointMode": "connected",
593 "postfixFontSize": "50%",
595 "prefixFontSize": "50%",
605 "fillColor": "rgba(31, 118, 189, 0.18)",
607 "lineColor": "rgb(31, 120, 193)",
613 "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))",
614 "format": "time_series",
620 "thresholds": "1,10",
621 "title": "Tardy Scrapes [$interval]",
622 "type": "singlestat",
623 "valueFontSize": "100%",
631 "valueName": "current"
634 "cacheTimeout": null,
635 "colorBackground": false,
638 "rgba(50, 172, 45, 0.97)",
639 "rgba(237, 129, 40, 0.89)",
640 "rgba(245, 54, 54, 0.9)"
642 "datasource": "${DS_PROMETHEUS}",
643 "description": "Number of times the database failed to reload block data from disk.",
649 "thresholdLabels": false,
650 "thresholdMarkers": true
658 "name": "value to text",
662 "name": "range to text",
666 "maxDataPoints": 100,
667 "nullPointMode": "connected",
670 "postfixFontSize": "50%",
672 "prefixFontSize": "50%",
682 "fillColor": "rgba(31, 118, 189, 0.18)",
684 "lineColor": "rgb(31, 120, 193)",
690 "expr": "sum(sum_over_time(prometheus_tsdb_reloads_failures_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))",
691 "format": "time_series",
697 "thresholds": "1,10",
698 "title": "Reload Failures [$interval]",
699 "type": "singlestat",
700 "valueFontSize": "100%",
708 "valueName": "current"
711 "cacheTimeout": null,
712 "colorBackground": false,
715 "rgba(50, 172, 45, 0.97)",
716 "rgba(237, 129, 40, 0.89)",
717 "rgba(245, 54, 54, 0.9)"
719 "datasource": "${DS_PROMETHEUS}",
720 "description": "Sum of all skipped scrapes",
726 "thresholdLabels": false,
727 "thresholdMarkers": true
735 "name": "value to text",
739 "name": "range to text",
743 "maxDataPoints": 100,
744 "nullPointMode": "connected",
747 "postfixFontSize": "50%",
749 "prefixFontSize": "50%",
759 "fillColor": "rgba(31, 118, 189, 0.18)",
761 "lineColor": "rgb(31, 120, 193)",
767 "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) ",
768 "format": "time_series",
774 "thresholds": "1,10",
775 "title": "Skipped Scrapes [$interval]",
776 "type": "singlestat",
777 "valueFontSize": "100%",
785 "valueName": "current"
789 "repeatIteration": null,
792 "title": "quick numbers",
804 "datasource": "${DS_PROMETHEUS}",
805 "description": "All non-zero failures and errors",
820 "nullPointMode": "null",
825 "seriesOverrides": [],
829 "steppedLine": false,
832 "expr": "sum(increase(net_conntrack_dialer_conn_failed_total{instance=~\"$instance\"}[5m])) > 0",
833 "format": "time_series",
835 "legendFormat": "Failed Connections",
840 "expr": "sum(increase(prometheus_evaluator_iterations_missed_total{instance=~\"$instance\"}[5m])) > 0",
841 "format": "time_series",
843 "legendFormat": "Missed Iterations",
848 "expr": "sum(increase(prometheus_evaluator_iterations_skipped_total{instance=~\"$instance\"}[5m])) > 0",
849 "format": "time_series",
851 "legendFormat": "Skipped Iterations",
856 "expr": "sum(increase(prometheus_rule_evaluation_failures_total{instance=~\"$instance\"}[5m])) > 0",
857 "format": "time_series",
859 "legendFormat": "Evaluation",
864 "expr": "sum(increase(prometheus_sd_azure_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
865 "format": "time_series",
867 "legendFormat": "Azure Refresh",
872 "expr": "sum(increase(prometheus_sd_consul_rpc_failures_total{instance=~\"$instance\"}[5m])) > 0",
873 "format": "time_series",
875 "legendFormat": "Consul RPC",
880 "expr": "sum(increase(prometheus_sd_dns_lookup_failures_total{instance=~\"$instance\"}[5m])) > 0",
881 "format": "time_series",
883 "legendFormat": "DNS Lookup",
888 "expr": "sum(increase(prometheus_sd_ec2_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
889 "format": "time_series",
891 "legendFormat": "EC2 Refresh",
896 "expr": "sum(increase(prometheus_sd_gce_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
897 "format": "time_series",
899 "legendFormat": "GCE Refresh",
904 "expr": "sum(increase(prometheus_sd_marathon_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
905 "format": "time_series",
907 "legendFormat": "Marathon Refresh",
912 "expr": "sum(increase(prometheus_sd_openstack_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
913 "format": "time_series",
915 "legendFormat": "Openstack Refresh",
920 "expr": "sum(increase(prometheus_sd_triton_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
921 "format": "time_series",
923 "legendFormat": "Triton Refresh",
928 "expr": "sum(increase(prometheus_target_scrapes_exceeded_sample_limit_total{instance=~\"$instance\"}[5m])) > 0",
929 "format": "time_series",
931 "legendFormat": "Sample Limit",
936 "expr": "sum(increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{instance=~\"$instance\"}[5m])) > 0",
937 "format": "time_series",
939 "legendFormat": "Duplicate Timestamp",
944 "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_bounds_total{instance=~\"$instance\"}[5m])) > 0",
945 "format": "time_series",
947 "legendFormat": "Timestamp Out of Bounds",
952 "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_order_total{instance=~\"$instance\"}[5m])) > 0",
953 "format": "time_series",
955 "legendFormat": "Sample Out of Order",
960 "expr": "sum(increase(prometheus_treecache_zookeeper_failures_total{instance=~\"$instance\"}[5m])) > 0",
961 "format": "time_series",
963 "legendFormat": "Zookeeper",
968 "expr": "sum(increase(prometheus_tsdb_compactions_failed_total{instance=~\"$instance\"}[5m])) > 0",
969 "format": "time_series",
971 "legendFormat": "TSDB Compactions",
976 "expr": "sum(increase(prometheus_tsdb_head_series_not_found{instance=~\"$instance\"}[5m])) > 0",
977 "format": "time_series",
979 "legendFormat": "Series Not Found",
984 "expr": "sum(increase(prometheus_tsdb_reloads_failures_total{instance=~\"$instance\"}[5m])) > 0",
985 "format": "time_series",
987 "legendFormat": "Reload",
995 "title": "Failures and Errors",
999 "value_type": "individual"
1030 "repeatIteration": null,
1031 "repeatRowId": null,
1045 "datasource": "${DS_PROMETHEUS}",
1060 "nullPointMode": "null",
1061 "percentage": false,
1065 "seriesOverrides": [],
1069 "steppedLine": false,
1072 "expr": "up{instance=~\"$instance\",job=~\"$job\"}",
1073 "format": "time_series",
1075 "intervalFactor": 1,
1076 "legendFormat": "{{instance}}",
1084 "title": "Upness (stacked)",
1088 "value_type": "individual"
1123 "datasource": "${DS_PROMETHEUS}",
1138 "nullPointMode": "null",
1139 "percentage": false,
1143 "seriesOverrides": [],
1147 "steppedLine": false,
1150 "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}",
1151 "format": "time_series",
1152 "intervalFactor": 2,
1153 "legendFormat": "{{instance}}",
1161 "title": "Storage Memory Chunks",
1165 "value_type": "individual"
1196 "repeatIteration": null,
1197 "repeatRowId": null,
1211 "datasource": "${DS_PROMETHEUS}",
1226 "nullPointMode": "null",
1227 "percentage": false,
1231 "seriesOverrides": [],
1235 "steppedLine": false,
1238 "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}",
1239 "format": "time_series",
1240 "intervalFactor": 2,
1241 "legendFormat": "{{instance}}",
1249 "title": "Series Count",
1253 "value_type": "individual"
1287 "datasource": "${DS_PROMETHEUS}",
1302 "nullPointMode": "null",
1303 "percentage": false,
1307 "seriesOverrides": [
1310 "transform": "negative-Y"
1316 "steppedLine": false,
1319 "expr": "sum( increase(prometheus_tsdb_head_series_created_total{instance=~\"$instance\"}[5m]) )",
1320 "format": "time_series",
1321 "intervalFactor": 2,
1322 "legendFormat": "created",
1327 "expr": "sum( increase(prometheus_tsdb_head_series_removed_total{instance=~\"$instance\"}[5m]) )",
1328 "format": "time_series",
1329 "intervalFactor": 2,
1330 "legendFormat": "removed",
1338 "title": "Series Created / Removed",
1342 "value_type": "individual"
1355 "label": "Series Count",
1373 "repeatIteration": null,
1374 "repeatRowId": null,
1385 "10.58.3.10:80": "#BA43A9"
1390 "datasource": "${DS_PROMETHEUS}",
1391 "description": "Rate of total number of appended samples",
1406 "nullPointMode": "null",
1407 "percentage": false,
1411 "seriesOverrides": [],
1415 "steppedLine": false,
1418 "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[1m])",
1419 "format": "time_series",
1420 "intervalFactor": 2,
1421 "legendFormat": "{{instance}}",
1429 "title": "Appended Samples per Second",
1433 "value_type": "individual"
1446 "label": "Samples / Second",
1464 "repeatIteration": null,
1465 "repeatRowId": null,
1467 "title": "appended samples",
1479 "datasource": "${DS_PROMETHEUS}",
1480 "description": "Total number of syncs that were executed on a scrape pool.",
1495 "nullPointMode": "null",
1496 "percentage": false,
1500 "seriesOverrides": [],
1504 "steppedLine": false,
1507 "expr": "sum(prometheus_target_scrape_pool_sync_total{job=~\"$job\",instance=~\"$instance\"}) by (scrape_job)",
1508 "format": "time_series",
1510 "intervalFactor": 2,
1511 "legendFormat": "{{scrape_job}}",
1519 "title": "Scrape Sync Total",
1523 "value_type": "individual"
1557 "datasource": "${DS_PROMETHEUS}",
1558 "description": "Actual interval to sync the scrape pool.",
1573 "nullPointMode": "null",
1574 "percentage": false,
1578 "seriesOverrides": [],
1582 "steppedLine": false,
1585 "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[2m])) by (scrape_job) * 1000",
1586 "format": "time_series",
1587 "intervalFactor": 2,
1588 "legendFormat": "{{scrape_job}}",
1596 "title": "Target Sync",
1600 "value_type": "individual"
1613 "label": "Milliseconds",
1631 "repeatIteration": null,
1632 "repeatRowId": null,
1646 "datasource": "${DS_PROMETHEUS}",
1661 "nullPointMode": "null",
1662 "percentage": false,
1666 "seriesOverrides": [],
1670 "steppedLine": false,
1673 "expr": "scrape_duration_seconds{instance=~\"$instance\"}",
1674 "format": "time_series",
1675 "intervalFactor": 2,
1676 "legendFormat": "{{instance}}",
1684 "title": "Scrape Duration",
1688 "value_type": "individual"
1722 "datasource": "${DS_PROMETHEUS}",
1723 "description": "Total number of rejected scrapes",
1738 "nullPointMode": "null",
1739 "percentage": false,
1743 "seriesOverrides": [],
1747 "steppedLine": false,
1750 "expr": "sum(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"})",
1751 "format": "time_series",
1752 "intervalFactor": 2,
1753 "legendFormat": "exceeded sample limit",
1758 "expr": "sum(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"})",
1759 "format": "time_series",
1761 "intervalFactor": 2,
1762 "legendFormat": "duplicate timestamp",
1767 "expr": "sum(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"})",
1768 "format": "time_series",
1770 "intervalFactor": 2,
1771 "legendFormat": "out of bounds",
1776 "expr": "sum(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}) ",
1777 "format": "time_series",
1779 "intervalFactor": 2,
1780 "legendFormat": "out of order",
1788 "title": "Rejected Scrapes",
1792 "value_type": "individual"
1824 "repeatIteration": null,
1825 "repeatRowId": null,
1839 "datasource": "${DS_PROMETHEUS}",
1840 "description": "The duration of rule group evaluations",
1855 "nullPointMode": "null",
1856 "percentage": false,
1860 "seriesOverrides": [],
1864 "steppedLine": false,
1867 "expr": "1000 * rate(prometheus_evaluator_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[5m]) / rate(prometheus_evaluator_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[5m])",
1868 "format": "time_series",
1869 "intervalFactor": 2,
1870 "legendFormat": "{{instance}}",
1878 "title": "Average Rule Evaluation Duration",
1882 "value_type": "individual"
1895 "label": "Milliseconds",
1916 "datasource": "${DS_PROMETHEUS}",
1931 "nullPointMode": "null",
1932 "percentage": false,
1936 "seriesOverrides": [],
1940 "steppedLine": false,
1943 "expr": "sum(rate(http_request_duration_microseconds_count{job=~\"$job\",instance=~\"$instance\"}[1m])) by (handler) > 0",
1944 "format": "time_series",
1945 "intervalFactor": 2,
1946 "legendFormat": "{{handler}}",
1954 "title": "HTTP Request Duration",
1958 "value_type": "individual"
1971 "label": "Microseconds",
1992 "datasource": "${DS_PROMETHEUS}",
2007 "nullPointMode": "null",
2008 "percentage": false,
2012 "seriesOverrides": [],
2016 "steppedLine": false,
2019 "expr": "sum(prometheus_engine_query_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"}) by (slice)",
2020 "format": "time_series",
2021 "intervalFactor": 2,
2022 "legendFormat": "{{slice}}",
2030 "title": "Prometheus Engine Query Duration Seconds",
2034 "value_type": "individual"
2068 "datasource": "${DS_PROMETHEUS}",
2069 "description": "Rule-group evaluations \n - total\n - missed due to slow rule group evaluation\n - skipped due to throttled metric storage",
2084 "nullPointMode": "null",
2085 "percentage": false,
2089 "seriesOverrides": [],
2093 "steppedLine": false,
2096 "expr": "sum(rate(prometheus_evaluator_iterations_total{job=~\"$job\", instance=~\"$instance\"}[5m]))",
2097 "format": "time_series",
2098 "intervalFactor": 2,
2099 "legendFormat": "Total",
2104 "expr": "sum(rate(prometheus_evaluator_iterations_missed_total{job=~\"$job\", instance=~\"$instance\"}[5m]))",
2105 "format": "time_series",
2106 "intervalFactor": 2,
2107 "legendFormat": "Missed",
2112 "expr": "sum(rate(prometheus_evaluator_iterations_skipped_total{job=~\"$job\", instance=~\"$instance\"}[5m]))",
2113 "format": "time_series",
2114 "intervalFactor": 2,
2115 "legendFormat": "Skipped",
2123 "title": "Rule Evaluator Iterations",
2127 "value_type": "individual"
2140 "label": "iterations",
2158 "repeatIteration": null,
2159 "repeatRowId": null,
2161 "title": "durations",
2173 "datasource": "${DS_PROMETHEUS}",
2188 "nullPointMode": "null",
2189 "percentage": false,
2193 "seriesOverrides": [],
2197 "steppedLine": false,
2200 "expr": "rate(prometheus_notifications_sent_total[5m])",
2201 "format": "time_series",
2202 "intervalFactor": 2,
2203 "legendFormat": "{{instance}}",
2211 "title": "Notifications Sent",
2215 "value_type": "individual"
2228 "label": "Notifications",
2246 "repeatIteration": null,
2247 "repeatRowId": null,
2249 "title": "notifications",
2261 "datasource": "${DS_PROMETHEUS}",
2276 "nullPointMode": "null",
2277 "percentage": false,
2281 "seriesOverrides": [],
2285 "steppedLine": false,
2288 "expr": "(time() - prometheus_config_last_reload_success_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"}) / 60",
2289 "format": "time_series",
2290 "intervalFactor": 2,
2291 "legendFormat": "{{instance}}",
2299 "title": "Minutes Since Successful Config Reload",
2303 "value_type": "individual"
2337 "datasource": "${DS_PROMETHEUS}",
2352 "nullPointMode": "null",
2353 "percentage": false,
2357 "seriesOverrides": [],
2361 "steppedLine": false,
2364 "expr": "prometheus_config_last_reload_successful{job=~\"$job\",instance=~\"$instance\"}",
2365 "format": "time_series",
2366 "intervalFactor": 2,
2367 "legendFormat": "{{instance}}",
2375 "title": "Successful Config Reload",
2379 "value_type": "individual"
2411 "repeatIteration": null,
2412 "repeatRowId": null,
2426 "datasource": "${DS_PROMETHEUS}",
2427 "description": "GC invocation durations",
2442 "nullPointMode": "null",
2443 "percentage": false,
2447 "seriesOverrides": [],
2451 "steppedLine": false,
2454 "expr": "sum(rate(go_gc_duration_seconds_sum{instance=~\"$instance\",job=~\"$job\"}[2m])) by (instance)",
2455 "format": "time_series",
2456 "intervalFactor": 2,
2457 "legendFormat": "{{instance}}",
2465 "title": "GC Rate / 2m",
2469 "value_type": "individual"
2500 "repeatIteration": null,
2501 "repeatRowId": null,
2503 "title": "garbage collection",
2515 "datasource": "${DS_PROMETHEUS}",
2516 "description": "This is probably wrong! Please help.",
2531 "nullPointMode": "null",
2532 "percentage": false,
2536 "seriesOverrides": [
2538 "alias": "allocated",
2545 "steppedLine": false,
2548 "expr": "sum(go_memstats_alloc_bytes_total{job=~\"$job\", instance=~\"$instance\"})",
2549 "format": "time_series",
2551 "intervalFactor": 2,
2552 "legendFormat": "alloc_bytes_total",
2557 "expr": "sum(go_memstats_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})",
2558 "format": "time_series",
2560 "intervalFactor": 2,
2561 "legendFormat": "allocated",
2566 "expr": "sum(go_memstats_buck_hash_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2567 "format": "time_series",
2568 "intervalFactor": 2,
2569 "legendFormat": "profiling bucket hash table",
2574 "expr": "sum(go_memstats_gc_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2575 "format": "time_series",
2576 "intervalFactor": 2,
2577 "legendFormat": "GC metadata",
2582 "expr": "sum(go_memstats_heap_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})",
2583 "format": "time_series",
2584 "intervalFactor": 2,
2585 "legendFormat": "heap in-use",
2590 "expr": "sum(go_memstats_heap_idle_bytes{job=~\"$job\", instance=~\"$instance\"})",
2591 "format": "time_series",
2592 "intervalFactor": 2,
2593 "legendFormat": "heap idle",
2598 "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
2599 "format": "time_series",
2600 "intervalFactor": 2,
2601 "legendFormat": "heap in use",
2606 "expr": "sum(go_memstats_heap_released_bytes{job=~\"$job\", instance=~\"$instance\"})",
2607 "format": "time_series",
2608 "intervalFactor": 2,
2609 "legendFormat": "heap released",
2614 "expr": "sum(go_memstats_heap_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2615 "format": "time_series",
2616 "intervalFactor": 2,
2617 "legendFormat": "heap system",
2622 "expr": "sum(go_memstats_mcache_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
2623 "format": "time_series",
2624 "intervalFactor": 2,
2625 "legendFormat": "mcache in use",
2630 "expr": "sum(go_memstats_mcache_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2631 "format": "time_series",
2632 "intervalFactor": 2,
2633 "legendFormat": "mcache sys",
2638 "expr": "sum(go_memstats_mspan_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
2639 "format": "time_series",
2640 "intervalFactor": 2,
2641 "legendFormat": "mspan in use",
2646 "expr": "sum(go_memstats_mspan_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2647 "format": "time_series",
2648 "intervalFactor": 2,
2649 "legendFormat": "mspan sys",
2654 "expr": "sum(go_memstats_next_gc_bytes{job=~\"$job\", instance=~\"$instance\"})",
2655 "format": "time_series",
2656 "intervalFactor": 2,
2657 "legendFormat": "heap next gc",
2662 "expr": "sum(go_memstats_other_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2663 "format": "time_series",
2664 "intervalFactor": 2,
2665 "legendFormat": "other sys",
2670 "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
2671 "format": "time_series",
2672 "intervalFactor": 2,
2673 "legendFormat": "stack in use",
2678 "expr": "sum(go_memstats_stack_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2679 "format": "time_series",
2680 "intervalFactor": 2,
2681 "legendFormat": "stack sys",
2686 "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2687 "format": "time_series",
2688 "intervalFactor": 2,
2689 "legendFormat": "sys",
2697 "title": "Go Memory Usage (FIXME)",
2701 "value_type": "individual"
2735 "datasource": "${DS_PROMETHEUS}",
2750 "nullPointMode": "null",
2751 "percentage": false,
2755 "seriesOverrides": [],
2759 "steppedLine": false,
2762 "expr": "prometheus_target_interval_length_seconds{instance=~\"$instance\", job=~\"$job\"}",
2763 "format": "time_series",
2764 "intervalFactor": 2,
2765 "legendFormat": "{{quantile}} {{interval}}",
2773 "title": "Scrape Duration",
2777 "value_type": "individual"
2811 "datasource": "${DS_PROMETHEUS}",
2826 "nullPointMode": "null",
2827 "percentage": false,
2831 "seriesOverrides": [],
2835 "steppedLine": false,
2838 "expr": "sum(rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m])) by (interval)",
2839 "format": "time_series",
2840 "intervalFactor": 2,
2841 "legendFormat": "{{interval}}",
2849 "title": "Target Scrapes / 5m",
2853 "value_type": "individual"
2884 "repeatIteration": null,
2885 "repeatRowId": null,
2887 "title": "Broken, ignore",
2891 "schemaVersion": 14,
2903 "includeAll": false,
2904 "label": "datasource",
2906 "name": "DS_PROMETHEUS",
2908 "query": "prometheus",
2911 "skipUrlSync": false,
2912 "type": "datasource"
2917 "datasource": "${DS_PROMETHEUS}",
2924 "query": "query_result(prometheus_tsdb_head_samples_appended_total)",
2926 "regex": "/.*job=\"([^\"]+)/",
2928 "tagValuesQuery": "",
2937 "datasource": "${DS_PROMETHEUS}",
2944 "query": "query_result(up{job=~\"$job\"})",
2946 "regex": "/.*instance=\"([^\"]+).*/",
2948 "tagValuesQuery": "",
2962 "includeAll": false,
3018 "query": "1h, 3h, 6h, 12h, 1d, 2d, 7d, 30d, 90d, 180d",
3028 "refresh_intervals": [
3052 "timezone": "browser",
3053 "title": "Prometheus",