fdio.infra.terraform/1n_nmd/grafana/conf/prometheus.json

   1 {
   2   "__inputs": [
   3     {
   4       "name": "DS_PROMETHEUS",
   5       "label": "prometheus",
   6       "description": "A prometheus server with prometheus server metrics",
   7       "type": "datasource",
   8       "pluginId": "prometheus",
   9       "pluginName": "Prometheus"
  10     }
  11   ],
  12   "__requires": [
  13     {
  14       "type": "grafana",
  15       "id": "grafana",
  16       "name": "Grafana",
  17       "version": "4.5.0-beta1"
  18     },
  19     {
  20       "type": "panel",
  21       "id": "graph",
  22       "name": "Graph",
  23       "version": ""
  24     },
  25     {
  26       "type": "datasource",
  27       "id": "prometheus",
  28       "name": "Prometheus",
  29       "version": "1.0.0"
  30     },
  31     {
  32       "type": "panel",
  33       "id": "singlestat",
  34       "name": "Singlestat",
  35       "version": ""
  36     },
  37     {
  38       "type": "panel",
  39       "id": "table",
  40       "name": "Table",
  41       "version": ""
  42     }
  43   ],
  44   "annotations": {
  45     "list": [
  46       {
  47         "datasource": "${DS_PROMETHEUS}",
  48         "enable": true,
  49         "expr": "sum(changes(prometheus_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}[10m])) by (instance)",
  50         "hide": false,
  51         "iconColor": "rgb(0, 96, 19)",
  52         "limit": 100,
  53         "name": "reloads",
  54         "showIn": 0,
  55         "step": "5m",
  56         "type": "alert"
  57       },
  58       {
  59         "datasource": "${DS_PROMETHEUS}",
  60         "enable": true,
  61         "expr": "count(sum(up{instance=\"$instance\"}) by (instance) < 1)",
  62         "hide": false,
  63         "iconColor": "rgba(255, 96, 96, 1)",
  64         "limit": 100,
  65         "name": "down",
  66         "showIn": 0,
  67         "step": "5m",
  68         "type": "alert"
  69       }
  70     ]
  71   },
  72   "description": "Get started faster with Grafana Cloud then easily build these dashboards. https://grafana.com/products/cloud/\nOverview of metrics from Prometheus 2.0.  \nUseful for using prometheus to monitor your prometheus.\nRevisions welcome!",
  73   "editable": true,
  74   "gnetId": 3662,
  75   "graphTooltip": 0,
  76   "hideControls": false,
  77   "id": null,
  78   "links": [],
  79   "refresh": "30s",
  80   "rows": [
  81     {
  82       "collapse": false,
  83       "height": 250,
  84       "panels": [
  85         {
  86           "cacheTimeout": null,
  87           "colorBackground": false,
  88           "colorValue": true,
  89           "colors": [
  90             "rgba(245, 54, 54, 0.9)",
  91             "rgba(237, 129, 40, 0.89)",
  92             "rgba(50, 172, 45, 0.97)"
  93           ],
  94           "datasource": "${DS_PROMETHEUS}",
  95           "decimals": 3,
  96           "description": "Percentage of uptime during the most recent $interval period.  Change the period with the 'interval' dropdown above.",
  97           "format": "none",
  98           "gauge": {
  99             "maxValue": 100,
 100             "minValue": 0,
 101             "show": false,
 102             "thresholdLabels": false,
 103             "thresholdMarkers": false
 104           },
 105           "id": 2,
 106           "interval": null,
 107           "links": [],
 108           "mappingType": 1,
 109           "mappingTypes": [
 110             {
 111               "name": "value to text",
 112               "value": 1
 113             },
 114             {
 115               "name": "range to text",
 116               "value": 2
 117             }
 118           ],
 119           "maxDataPoints": 100,
 120           "nullPointMode": "connected",
 121           "nullText": null,
 122           "postfix": "%",
 123           "postfixFontSize": "100%",
 124           "prefix": "",
 125           "prefixFontSize": "50%",
 126           "rangeMaps": [
 127             {
 128               "from": "null",
 129               "text": "N/A",
 130               "to": "null"
 131             }
 132           ],
 133           "span": 3,
 134           "sparkline": {
 135             "fillColor": "rgba(31, 118, 189, 0.18)",
 136             "full": true,
 137             "lineColor": "rgb(31, 120, 193)",
 138             "show": false
 139           },
 140           "tableColumn": "",
 141           "targets": [
 142             {
 143               "expr": "avg(avg_over_time(up{instance=~\"$instance\",job=~\"$job\"}[$interval]) * 100)",
 144               "format": "time_series",
 145               "intervalFactor": 2,
 146               "legendFormat": "",
 147               "refId": "A",
 148               "step": 40
 149             }
 150           ],
 151           "thresholds": "90, 99",
 152           "title": "Uptime [$interval]",
 153           "type": "singlestat",
 154           "valueFontSize": "100%",
 155           "valueMaps": [
 156             {
 157               "op": "=",
 158               "text": "N/A",
 159               "value": "null"
 160             }
 161           ],
 162           "valueName": "current"
 163         },
 164         {
 165           "columns": [],
 166           "datasource": "${DS_PROMETHEUS}",
 167           "description": "Servers which are DOWN RIGHT NOW! \nFIX THEM!!",
 168           "fontSize": "100%",
 169           "hideTimeOverride": true,
 170           "id": 25,
 171           "links": [],
 172           "pageSize": null,
 173           "scroll": true,
 174           "showHeader": true,
 175           "sort": {
 176             "col": 0,
 177             "desc": true
 178           },
 179           "span": 3,
 180           "styles": [
 181             {
 182               "alias": "Time",
 183               "dateFormat": "YYYY-MM-DD HH:mm:ss",
 184               "pattern": "Time",
 185               "type": "hidden"
 186             },
 187             {
 188               "alias": "",
 189               "colorMode": null,
 190               "colors": [
 191                 "rgba(245, 54, 54, 0.9)",
 192                 "rgba(237, 129, 40, 0.89)",
 193                 "rgba(50, 172, 45, 0.97)"
 194               ],
 195               "dateFormat": "YYYY-MM-DD HH:mm:ss",
 196               "decimals": 2,
 197               "pattern": "/__name__|job|Value/",
 198               "thresholds": [],
 199               "type": "hidden",
 200               "unit": "short"
 201             },
 202             {
 203               "alias": "   ",
 204               "colorMode": "cell",
 205               "colors": [
 206                 "rgba(255, 0, 0, 0.9)",
 207                 "rgba(237, 129, 40, 0.89)",
 208                 "rgba(255, 0, 0, 0.97)"
 209               ],
 210               "dateFormat": "YYYY-MM-DD HH:mm:ss",
 211               "decimals": 2,
 212               "link": false,
 213               "pattern": "instance",
 214               "thresholds": [
 215                 "",
 216                 "",
 217                 ""
 218               ],
 219               "type": "string",
 220               "unit": "short"
 221             }
 222           ],
 223           "targets": [
 224             {
 225               "expr": "up{instance=~\"$instance\",job=~\"$job\"} < 1",
 226               "format": "table",
 227               "intervalFactor": 2,
 228               "refId": "A",
 229               "step": 2
 230             }
 231           ],
 232           "timeFrom": "1s",
 233           "title": "Currently Down",
 234           "transform": "table",
 235           "type": "table"
 236         },
 237         {
 238           "cacheTimeout": null,
 239           "colorBackground": false,
 240           "colorValue": true,
 241           "colors": [
 242             "rgba(50, 172, 45, 0.97)",
 243             "rgba(237, 129, 40, 0.89)",
 244             "rgba(245, 54, 54, 0.9)"
 245           ],
 246           "datasource": "${DS_PROMETHEUS}",
 247           "description": "Total number of time series in prometheus",
 248           "format": "none",
 249           "gauge": {
 250             "maxValue": 100,
 251             "minValue": 0,
 252             "show": false,
 253             "thresholdLabels": false,
 254             "thresholdMarkers": true
 255           },
 256           "id": 12,
 257           "interval": null,
 258           "links": [],
 259           "mappingType": 1,
 260           "mappingTypes": [
 261             {
 262               "name": "value to text",
 263               "value": 1
 264             },
 265             {
 266               "name": "range to text",
 267               "value": 2
 268             }
 269           ],
 270           "maxDataPoints": 100,
 271           "nullPointMode": "connected",
 272           "nullText": null,
 273           "postfix": "",
 274           "postfixFontSize": "50%",
 275           "prefix": "",
 276           "prefixFontSize": "50%",
 277           "rangeMaps": [
 278             {
 279               "from": "null",
 280               "text": "N/A",
 281               "to": "null"
 282             }
 283           ],
 284           "span": 3,
 285           "sparkline": {
 286             "fillColor": "rgba(31, 118, 189, 0.18)",
 287             "full": true,
 288             "lineColor": "rgb(31, 120, 193)",
 289             "show": true
 290           },
 291           "tableColumn": "",
 292           "targets": [
 293             {
 294               "expr": "sum(prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"})",
 295               "format": "time_series",
 296               "intervalFactor": 2,
 297               "refId": "B",
 298               "step": 40
 299             }
 300           ],
 301           "thresholds": "1000000,2000000",
 302           "title": "Total Series",
 303           "type": "singlestat",
 304           "valueFontSize": "100%",
 305           "valueMaps": [
 306             {
 307               "op": "=",
 308               "text": "N/A",
 309               "value": "null"
 310             }
 311           ],
 312           "valueName": "current"
 313         },
 314         {
 315           "cacheTimeout": null,
 316           "colorBackground": false,
 317           "colorValue": false,
 318           "colors": [
 319             "rgba(245, 54, 54, 0.9)",
 320             "rgba(237, 129, 40, 0.89)",
 321             "rgba(50, 172, 45, 0.97)"
 322           ],
 323           "datasource": "${DS_PROMETHEUS}",
 324           "format": "none",
 325           "gauge": {
 326             "maxValue": 100,
 327             "minValue": 0,
 328             "show": false,
 329             "thresholdLabels": false,
 330             "thresholdMarkers": true
 331           },
 332           "id": 14,
 333           "interval": null,
 334           "links": [],
 335           "mappingType": 1,
 336           "mappingTypes": [
 337             {
 338               "name": "value to text",
 339               "value": 1
 340             },
 341             {
 342               "name": "range to text",
 343               "value": 2
 344             }
 345           ],
 346           "maxDataPoints": 100,
 347           "nullPointMode": "connected",
 348           "nullText": null,
 349           "postfix": "",
 350           "postfixFontSize": "50%",
 351           "prefix": "",
 352           "prefixFontSize": "50%",
 353           "rangeMaps": [
 354             {
 355               "from": "null",
 356               "text": "N/A",
 357               "to": "null"
 358             }
 359           ],
 360           "span": 3,
 361           "sparkline": {
 362             "fillColor": "rgba(31, 118, 189, 0.18)",
 363             "full": true,
 364             "lineColor": "rgb(31, 120, 193)",
 365             "show": true
 366           },
 367           "tableColumn": "",
 368           "targets": [
 369             {
 370               "expr": "sum(prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"})",
 371               "format": "time_series",
 372               "intervalFactor": 2,
 373               "refId": "B",
 374               "step": 40
 375             }
 376           ],
 377           "thresholds": "",
 378           "title": "Memory Chunks",
 379           "type": "singlestat",
 380           "valueFontSize": "100%",
 381           "valueMaps": [
 382             {
 383               "op": "=",
 384               "text": "N/A",
 385               "value": "null"
 386             }
 387           ],
 388           "valueName": "current"
 389         }
 390       ],
 391       "repeat": null,
 392       "repeatIteration": null,
 393       "repeatRowId": null,
 394       "showTitle": false,
 395       "title": "at a glance",
 396       "titleSize": "h6"
 397     },
 398     {
 399       "collapse": false,
 400       "height": 236,
 401       "panels": [
 402         {
 403           "cacheTimeout": null,
 404           "colorBackground": false,
 405           "colorValue": true,
 406           "colors": [
 407             "rgba(50, 172, 45, 0.97)",
 408             "rgba(237, 129, 40, 0.89)",
 409             "rgba(245, 54, 54, 0.9)"
 410           ],
 411           "datasource": "${DS_PROMETHEUS}",
 412           "description": "The total number of rule group evaluations missed due to slow rule group evaluation.",
 413           "format": "none",
 414           "gauge": {
 415             "maxValue": 100,
 416             "minValue": 0,
 417             "show": false,
 418             "thresholdLabels": false,
 419             "thresholdMarkers": true
 420           },
 421           "id": 16,
 422           "interval": null,
 423           "links": [],
 424           "mappingType": 1,
 425           "mappingTypes": [
 426             {
 427               "name": "value to text",
 428               "value": 1
 429             },
 430             {
 431               "name": "range to text",
 432               "value": 2
 433             }
 434           ],
 435           "maxDataPoints": 100,
 436           "nullPointMode": "connected",
 437           "nullText": null,
 438           "postfix": "",
 439           "postfixFontSize": "50%",
 440           "prefix": "",
 441           "prefixFontSize": "50%",
 442           "rangeMaps": [
 443             {
 444               "from": "null",
 445               "text": "N/A",
 446               "to": "null"
 447             }
 448           ],
 449           "span": 2,
 450           "sparkline": {
 451             "fillColor": "rgba(31, 118, 189, 0.18)",
 452             "full": false,
 453             "lineColor": "rgb(31, 120, 193)",
 454             "show": false
 455           },
 456           "tableColumn": "",
 457           "targets": [
 458             {
 459               "expr": "sum(sum_over_time(prometheus_evaluator_iterations_missed_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))",
 460               "format": "time_series",
 461               "intervalFactor": 2,
 462               "refId": "A",
 463               "step": 40
 464             }
 465           ],
 466           "thresholds": "1,10",
 467           "title": "Missed Iterations [$interval]",
 468           "type": "singlestat",
 469           "valueFontSize": "100%",
 470           "valueMaps": [
 471             {
 472               "op": "=",
 473               "text": "N/A",
 474               "value": "null"
 475             }
 476           ],
 477           "valueName": "current"
 478         },
 479         {
 480           "cacheTimeout": null,
 481           "colorBackground": false,
 482           "colorValue": true,
 483           "colors": [
 484             "rgba(50, 172, 45, 0.97)",
 485             "rgba(237, 129, 40, 0.89)",
 486             "rgba(245, 54, 54, 0.9)"
 487           ],
 488           "datasource": "${DS_PROMETHEUS}",
 489           "description": "The total number of rule group evaluations skipped due to throttled metric storage.",
 490           "format": "none",
 491           "gauge": {
 492             "maxValue": 100,
 493             "minValue": 0,
 494             "show": false,
 495             "thresholdLabels": false,
 496             "thresholdMarkers": true
 497           },
 498           "id": 18,
 499           "interval": null,
 500           "links": [],
 501           "mappingType": 1,
 502           "mappingTypes": [
 503             {
 504               "name": "value to text",
 505               "value": 1
 506             },
 507             {
 508               "name": "range to text",
 509               "value": 2
 510             }
 511           ],
 512           "maxDataPoints": 100,
 513           "nullPointMode": "connected",
 514           "nullText": null,
 515           "postfix": "",
 516           "postfixFontSize": "50%",
 517           "prefix": "",
 518           "prefixFontSize": "50%",
 519           "rangeMaps": [
 520             {
 521               "from": "null",
 522               "text": "N/A",
 523               "to": "null"
 524             }
 525           ],
 526           "span": 2,
 527           "sparkline": {
 528             "fillColor": "rgba(31, 118, 189, 0.18)",
 529             "full": false,
 530             "lineColor": "rgb(31, 120, 193)",
 531             "show": false
 532           },
 533           "tableColumn": "",
 534           "targets": [
 535             {
 536               "expr": "sum(sum_over_time(prometheus_evaluator_iterations_skipped_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))",
 537               "format": "time_series",
 538               "intervalFactor": 2,
 539               "refId": "A",
 540               "step": 40
 541             }
 542           ],
 543           "thresholds": "1,10",
 544           "title": "Skipped Iterations [$interval]",
 545           "type": "singlestat",
 546           "valueFontSize": "100%",
 547           "valueMaps": [
 548             {
 549               "op": "=",
 550               "text": "N/A",
 551               "value": "null"
 552             }
 553           ],
 554           "valueName": "current"
 555         },
 556         {
 557           "cacheTimeout": null,
 558           "colorBackground": false,
 559           "colorValue": true,
 560           "colors": [
 561             "rgba(50, 172, 45, 0.97)",
 562             "rgba(237, 129, 40, 0.89)",
 563             "rgba(245, 54, 54, 0.9)"
 564           ],
 565           "datasource": "${DS_PROMETHEUS}",
 566           "description": "Total number of scrapes that hit the sample limit and were rejected.",
 567           "format": "none",
 568           "gauge": {
 569             "maxValue": 100,
 570             "minValue": 0,
 571             "show": false,
 572             "thresholdLabels": false,
 573             "thresholdMarkers": true
 574           },
 575           "id": 19,
 576           "interval": null,
 577           "links": [],
 578           "mappingType": 1,
 579           "mappingTypes": [
 580             {
 581               "name": "value to text",
 582               "value": 1
 583             },
 584             {
 585               "name": "range to text",
 586               "value": 2
 587             }
 588           ],
 589           "maxDataPoints": 100,
 590           "nullPointMode": "connected",
 591           "nullText": null,
 592           "postfix": "",
 593           "postfixFontSize": "50%",
 594           "prefix": "",
 595           "prefixFontSize": "50%",
 596           "rangeMaps": [
 597             {
 598               "from": "null",
 599               "text": "N/A",
 600               "to": "null"
 601             }
 602           ],
 603           "span": 2,
 604           "sparkline": {
 605             "fillColor": "rgba(31, 118, 189, 0.18)",
 606             "full": false,
 607             "lineColor": "rgb(31, 120, 193)",
 608             "show": false
 609           },
 610           "tableColumn": "",
 611           "targets": [
 612             {
 613               "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))",
 614               "format": "time_series",
 615               "intervalFactor": 2,
 616               "refId": "A",
 617               "step": 40
 618             }
 619           ],
 620           "thresholds": "1,10",
 621           "title": "Tardy Scrapes [$interval]",
 622           "type": "singlestat",
 623           "valueFontSize": "100%",
 624           "valueMaps": [
 625             {
 626               "op": "=",
 627               "text": "N/A",
 628               "value": "null"
 629             }
 630           ],
 631           "valueName": "current"
 632         },
 633         {
 634           "cacheTimeout": null,
 635           "colorBackground": false,
 636           "colorValue": true,
 637           "colors": [
 638             "rgba(50, 172, 45, 0.97)",
 639             "rgba(237, 129, 40, 0.89)",
 640             "rgba(245, 54, 54, 0.9)"
 641           ],
 642           "datasource": "${DS_PROMETHEUS}",
 643           "description": "Number of times the database failed to reload block data from disk.",
 644           "format": "none",
 645           "gauge": {
 646             "maxValue": 100,
 647             "minValue": 0,
 648             "show": false,
 649             "thresholdLabels": false,
 650             "thresholdMarkers": true
 651           },
 652           "id": 13,
 653           "interval": null,
 654           "links": [],
 655           "mappingType": 1,
 656           "mappingTypes": [
 657             {
 658               "name": "value to text",
 659               "value": 1
 660             },
 661             {
 662               "name": "range to text",
 663               "value": 2
 664             }
 665           ],
 666           "maxDataPoints": 100,
 667           "nullPointMode": "connected",
 668           "nullText": null,
 669           "postfix": "",
 670           "postfixFontSize": "50%",
 671           "prefix": "",
 672           "prefixFontSize": "50%",
 673           "rangeMaps": [
 674             {
 675               "from": "null",
 676               "text": "N/A",
 677               "to": "null"
 678             }
 679           ],
 680           "span": 2,
 681           "sparkline": {
 682             "fillColor": "rgba(31, 118, 189, 0.18)",
 683             "full": false,
 684             "lineColor": "rgb(31, 120, 193)",
 685             "show": false
 686           },
 687           "tableColumn": "",
 688           "targets": [
 689             {
 690               "expr": "sum(sum_over_time(prometheus_tsdb_reloads_failures_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))",
 691               "format": "time_series",
 692               "intervalFactor": 2,
 693               "refId": "A",
 694               "step": 40
 695             }
 696           ],
 697           "thresholds": "1,10",
 698           "title": "Reload Failures [$interval]",
 699           "type": "singlestat",
 700           "valueFontSize": "100%",
 701           "valueMaps": [
 702             {
 703               "op": "=",
 704               "text": "N/A",
 705               "value": "null"
 706             }
 707           ],
 708           "valueName": "current"
 709         },
 710         {
 711           "cacheTimeout": null,
 712           "colorBackground": false,
 713           "colorValue": true,
 714           "colors": [
 715             "rgba(50, 172, 45, 0.97)",
 716             "rgba(237, 129, 40, 0.89)",
 717             "rgba(245, 54, 54, 0.9)"
 718           ],
 719           "datasource": "${DS_PROMETHEUS}",
 720           "description": "Sum of all skipped scrapes",
 721           "format": "none",
 722           "gauge": {
 723             "maxValue": 100,
 724             "minValue": 0,
 725             "show": false,
 726             "thresholdLabels": false,
 727             "thresholdMarkers": true
 728           },
 729           "id": 20,
 730           "interval": null,
 731           "links": [],
 732           "mappingType": 1,
 733           "mappingTypes": [
 734             {
 735               "name": "value to text",
 736               "value": 1
 737             },
 738             {
 739               "name": "range to text",
 740               "value": 2
 741             }
 742           ],
 743           "maxDataPoints": 100,
 744           "nullPointMode": "connected",
 745           "nullText": null,
 746           "postfix": "",
 747           "postfixFontSize": "50%",
 748           "prefix": "",
 749           "prefixFontSize": "50%",
 750           "rangeMaps": [
 751             {
 752               "from": "null",
 753               "text": "N/A",
 754               "to": "null"
 755             }
 756           ],
 757           "span": 4,
 758           "sparkline": {
 759             "fillColor": "rgba(31, 118, 189, 0.18)",
 760             "full": false,
 761             "lineColor": "rgb(31, 120, 193)",
 762             "show": false
 763           },
 764           "tableColumn": "",
 765           "targets": [
 766             {
 767               "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) ",
 768               "format": "time_series",
 769               "intervalFactor": 2,
 770               "refId": "A",
 771               "step": 40
 772             }
 773           ],
 774           "thresholds": "1,10",
 775           "title": "Skipped Scrapes [$interval]",
 776           "type": "singlestat",
 777           "valueFontSize": "100%",
 778           "valueMaps": [
 779             {
 780               "op": "=",
 781               "text": "N/A",
 782               "value": "null"
 783             }
 784           ],
 785           "valueName": "current"
 786         }
 787       ],
 788       "repeat": null,
 789       "repeatIteration": null,
 790       "repeatRowId": null,
 791       "showTitle": false,
 792       "title": "quick numbers",
 793       "titleSize": "h6"
 794     },
 795     {
 796       "collapse": false,
 797       "height": 250,
 798       "panels": [
 799         {
 800           "aliasColors": {},
 801           "bars": false,
 802           "dashLength": 10,
 803           "dashes": false,
 804           "datasource": "${DS_PROMETHEUS}",
 805           "description": "All non-zero failures and errors",
 806           "fill": 1,
 807           "id": 33,
 808           "legend": {
 809             "avg": false,
 810             "current": false,
 811             "max": false,
 812             "min": false,
 813             "show": true,
 814             "total": false,
 815             "values": false
 816           },
 817           "lines": true,
 818           "linewidth": 1,
 819           "links": [],
 820           "nullPointMode": "null",
 821           "percentage": false,
 822           "pointradius": 5,
 823           "points": false,
 824           "renderer": "flot",
 825           "seriesOverrides": [],
 826           "spaceLength": 10,
 827           "span": 12,
 828           "stack": false,
 829           "steppedLine": false,
 830           "targets": [
 831             {
 832               "expr": "sum(increase(net_conntrack_dialer_conn_failed_total{instance=~\"$instance\"}[5m])) > 0",
 833               "format": "time_series",
 834               "intervalFactor": 2,
 835               "legendFormat": "Failed Connections",
 836               "refId": "A",
 837               "step": 2
 838             },
 839             {
 840               "expr": "sum(increase(prometheus_evaluator_iterations_missed_total{instance=~\"$instance\"}[5m])) > 0",
 841               "format": "time_series",
 842               "intervalFactor": 2,
 843               "legendFormat": "Missed Iterations",
 844               "refId": "B",
 845               "step": 2
 846             },
 847             {
 848               "expr": "sum(increase(prometheus_evaluator_iterations_skipped_total{instance=~\"$instance\"}[5m])) > 0",
 849               "format": "time_series",
 850               "intervalFactor": 2,
 851               "legendFormat": "Skipped Iterations",
 852               "refId": "C",
 853               "step": 2
 854             },
 855             {
 856               "expr": "sum(increase(prometheus_rule_evaluation_failures_total{instance=~\"$instance\"}[5m])) > 0",
 857               "format": "time_series",
 858               "intervalFactor": 2,
 859               "legendFormat": "Evaluation",
 860               "refId": "D",
 861               "step": 2
 862             },
 863             {
 864               "expr": "sum(increase(prometheus_sd_azure_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
 865               "format": "time_series",
 866               "intervalFactor": 2,
 867               "legendFormat": "Azure Refresh",
 868               "refId": "E",
 869               "step": 2
 870             },
 871             {
 872               "expr": "sum(increase(prometheus_sd_consul_rpc_failures_total{instance=~\"$instance\"}[5m])) > 0",
 873               "format": "time_series",
 874               "intervalFactor": 2,
 875               "legendFormat": "Consul RPC",
 876               "refId": "F",
 877               "step": 2
 878             },
 879             {
 880               "expr": "sum(increase(prometheus_sd_dns_lookup_failures_total{instance=~\"$instance\"}[5m])) > 0",
 881               "format": "time_series",
 882               "intervalFactor": 2,
 883               "legendFormat": "DNS Lookup",
 884               "refId": "G",
 885               "step": 2
 886             },
 887             {
 888               "expr": "sum(increase(prometheus_sd_ec2_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
 889               "format": "time_series",
 890               "intervalFactor": 2,
 891               "legendFormat": "EC2 Refresh",
 892               "refId": "H",
 893               "step": 2
 894             },
 895             {
 896               "expr": "sum(increase(prometheus_sd_gce_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
 897               "format": "time_series",
 898               "intervalFactor": 2,
 899               "legendFormat": "GCE Refresh",
 900               "refId": "I",
 901               "step": 2
 902             },
 903             {
 904               "expr": "sum(increase(prometheus_sd_marathon_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
 905               "format": "time_series",
 906               "intervalFactor": 2,
 907               "legendFormat": "Marathon Refresh",
 908               "refId": "J",
 909               "step": 2
 910             },
 911             {
 912               "expr": "sum(increase(prometheus_sd_openstack_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
 913               "format": "time_series",
 914               "intervalFactor": 2,
 915               "legendFormat": "Openstack Refresh",
 916               "refId": "K",
 917               "step": 2
 918             },
 919             {
 920               "expr": "sum(increase(prometheus_sd_triton_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0",
 921               "format": "time_series",
 922               "intervalFactor": 2,
 923               "legendFormat": "Triton Refresh",
 924               "refId": "L",
 925               "step": 2
 926             },
 927             {
 928               "expr": "sum(increase(prometheus_target_scrapes_exceeded_sample_limit_total{instance=~\"$instance\"}[5m])) > 0",
 929               "format": "time_series",
 930               "intervalFactor": 2,
 931               "legendFormat": "Sample Limit",
 932               "refId": "M",
 933               "step": 2
 934             },
 935             {
 936               "expr": "sum(increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{instance=~\"$instance\"}[5m])) > 0",
 937               "format": "time_series",
 938               "intervalFactor": 2,
 939               "legendFormat": "Duplicate Timestamp",
 940               "refId": "N",
 941               "step": 2
 942             },
 943             {
 944               "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_bounds_total{instance=~\"$instance\"}[5m])) > 0",
 945               "format": "time_series",
 946               "intervalFactor": 2,
 947               "legendFormat": "Timestamp Out of Bounds",
 948               "refId": "O",
 949               "step": 2
 950             },
 951             {
 952               "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_order_total{instance=~\"$instance\"}[5m])) > 0",
 953               "format": "time_series",
 954               "intervalFactor": 2,
 955               "legendFormat": "Sample Out of Order",
 956               "refId": "P",
 957               "step": 2
 958             },
 959             {
 960               "expr": "sum(increase(prometheus_treecache_zookeeper_failures_total{instance=~\"$instance\"}[5m])) > 0",
 961               "format": "time_series",
 962               "intervalFactor": 2,
 963               "legendFormat": "Zookeeper",
 964               "refId": "Q",
 965               "step": 2
 966             },
 967             {
 968               "expr": "sum(increase(prometheus_tsdb_compactions_failed_total{instance=~\"$instance\"}[5m])) > 0",
 969               "format": "time_series",
 970               "intervalFactor": 2,
 971               "legendFormat": "TSDB Compactions",
 972               "refId": "R",
 973               "step": 2
 974             },
 975             {
 976               "expr": "sum(increase(prometheus_tsdb_head_series_not_found{instance=~\"$instance\"}[5m])) > 0",
 977               "format": "time_series",
 978               "intervalFactor": 2,
 979               "legendFormat": "Series Not Found",
 980               "refId": "S",
 981               "step": 2
 982             },
 983             {
 984               "expr": "sum(increase(prometheus_tsdb_reloads_failures_total{instance=~\"$instance\"}[5m])) > 0",
 985               "format": "time_series",
 986               "intervalFactor": 2,
 987               "legendFormat": "Reload",
 988               "refId": "T",
 989               "step": 2
 990             }
 991           ],
 992           "thresholds": [],
 993           "timeFrom": null,
 994           "timeShift": null,
 995           "title": "Failures and Errors",
 996           "tooltip": {
 997             "shared": true,
 998             "sort": 0,
 999             "value_type": "individual"
1000           },
1001           "type": "graph",
1002           "xaxis": {
1003             "buckets": null,
1004             "mode": "time",
1005             "name": null,
1006             "show": true,
1007             "values": []
1008           },
1009           "yaxes": [
1010             {
1011               "format": "short",
1012               "label": "Errors",
1013               "logBase": 1,
1014               "max": null,
1015               "min": "0",
1016               "show": true
1017             },
1018             {
1019               "format": "short",
1020               "label": null,
1021               "logBase": 1,
1022               "max": null,
1023               "min": null,
1024               "show": true
1025             }
1026           ]
1027         }
1028       ],
1029       "repeat": null,
1030       "repeatIteration": null,
1031       "repeatRowId": null,
1032       "showTitle": false,
1033       "title": "errors",
1034       "titleSize": "h6"
1035     },
1036     {
1037       "collapse": false,
1038       "height": "250px",
1039       "panels": [
1040         {
1041           "aliasColors": {},
1042           "bars": false,
1043           "dashLength": 10,
1044           "dashes": false,
1045           "datasource": "${DS_PROMETHEUS}",
1046           "fill": 1,
1047           "id": 1,
1048           "legend": {
1049             "avg": false,
1050             "current": false,
1051             "max": false,
1052             "min": false,
1053             "show": true,
1054             "total": false,
1055             "values": false
1056           },
1057           "lines": true,
1058           "linewidth": 1,
1059           "links": [],
1060           "nullPointMode": "null",
1061           "percentage": false,
1062           "pointradius": 5,
1063           "points": false,
1064           "renderer": "flot",
1065           "seriesOverrides": [],
1066           "spaceLength": 10,
1067           "span": 6,
1068           "stack": true,
1069           "steppedLine": false,
1070           "targets": [
1071             {
1072               "expr": "up{instance=~\"$instance\",job=~\"$job\"}",
1073               "format": "time_series",
1074               "interval": "",
1075               "intervalFactor": 1,
1076               "legendFormat": "{{instance}}",
1077               "refId": "A",
1078               "step": 2
1079             }
1080           ],
1081           "thresholds": [],
1082           "timeFrom": null,
1083           "timeShift": null,
1084           "title": "Upness (stacked)",
1085           "tooltip": {
1086             "shared": true,
1087             "sort": 0,
1088             "value_type": "individual"
1089           },
1090           "type": "graph",
1091           "xaxis": {
1092             "buckets": null,
1093             "mode": "time",
1094             "name": null,
1095             "show": true,
1096             "values": []
1097           },
1098           "yaxes": [
1099             {
1100               "decimals": 0,
1101               "format": "none",
1102               "label": "Up",
1103               "logBase": 1,
1104               "max": null,
1105               "min": "0",
1106               "show": true
1107             },
1108             {
1109               "format": "short",
1110               "label": null,
1111               "logBase": 1,
1112               "max": null,
1113               "min": null,
1114               "show": false
1115             }
1116           ]
1117         },
1118         {
1119           "aliasColors": {},
1120           "bars": false,
1121           "dashLength": 10,
1122           "dashes": false,
1123           "datasource": "${DS_PROMETHEUS}",
1124           "fill": 1,
1125           "id": 5,
1126           "legend": {
1127             "avg": false,
1128             "current": false,
1129             "max": false,
1130             "min": false,
1131             "show": true,
1132             "total": false,
1133             "values": false
1134           },
1135           "lines": true,
1136           "linewidth": 1,
1137           "links": [],
1138           "nullPointMode": "null",
1139           "percentage": false,
1140           "pointradius": 5,
1141           "points": false,
1142           "renderer": "flot",
1143           "seriesOverrides": [],
1144           "spaceLength": 10,
1145           "span": 6,
1146           "stack": false,
1147           "steppedLine": false,
1148           "targets": [
1149             {
1150               "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}",
1151               "format": "time_series",
1152               "intervalFactor": 2,
1153               "legendFormat": "{{instance}}",
1154               "refId": "A",
1155               "step": 4
1156             }
1157           ],
1158           "thresholds": [],
1159           "timeFrom": null,
1160           "timeShift": null,
1161           "title": "Storage Memory Chunks",
1162           "tooltip": {
1163             "shared": true,
1164             "sort": 0,
1165             "value_type": "individual"
1166           },
1167           "type": "graph",
1168           "xaxis": {
1169             "buckets": null,
1170             "mode": "time",
1171             "name": null,
1172             "show": true,
1173             "values": []
1174           },
1175           "yaxes": [
1176             {
1177               "format": "short",
1178               "label": "Chunks",
1179               "logBase": 1,
1180               "max": null,
1181               "min": "0",
1182               "show": true
1183             },
1184             {
1185               "format": "short",
1186               "label": null,
1187               "logBase": 1,
1188               "max": null,
1189               "min": null,
1190               "show": true
1191             }
1192           ]
1193         }
1194       ],
1195       "repeat": null,
1196       "repeatIteration": null,
1197       "repeatRowId": null,
1198       "showTitle": false,
1199       "title": "up",
1200       "titleSize": "h6"
1201     },
1202     {
1203       "collapse": false,
1204       "height": 250,
1205       "panels": [
1206         {
1207           "aliasColors": {},
1208           "bars": false,
1209           "dashLength": 10,
1210           "dashes": false,
1211           "datasource": "${DS_PROMETHEUS}",
1212           "fill": 1,
1213           "id": 3,
1214           "legend": {
1215             "avg": false,
1216             "current": false,
1217             "max": false,
1218             "min": false,
1219             "show": true,
1220             "total": false,
1221             "values": false
1222           },
1223           "lines": true,
1224           "linewidth": 1,
1225           "links": [],
1226           "nullPointMode": "null",
1227           "percentage": false,
1228           "pointradius": 5,
1229           "points": false,
1230           "renderer": "flot",
1231           "seriesOverrides": [],
1232           "spaceLength": 10,
1233           "span": 6,
1234           "stack": false,
1235           "steppedLine": false,
1236           "targets": [
1237             {
1238               "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}",
1239               "format": "time_series",
1240               "intervalFactor": 2,
1241               "legendFormat": "{{instance}}",
1242               "refId": "A",
1243               "step": 4
1244             }
1245           ],
1246           "thresholds": [],
1247           "timeFrom": null,
1248           "timeShift": null,
1249           "title": "Series Count",
1250           "tooltip": {
1251             "shared": true,
1252             "sort": 0,
1253             "value_type": "individual"
1254           },
1255           "type": "graph",
1256           "xaxis": {
1257             "buckets": null,
1258             "mode": "time",
1259             "name": null,
1260             "show": true,
1261             "values": []
1262           },
1263           "yaxes": [
1264             {
1265               "format": "short",
1266               "label": "Series",
1267               "logBase": 1,
1268               "max": null,
1269               "min": "0",
1270               "show": true
1271             },
1272             {
1273               "format": "short",
1274               "label": null,
1275               "logBase": 1,
1276               "max": null,
1277               "min": null,
1278               "show": true
1279             }
1280           ]
1281         },
1282         {
1283           "aliasColors": {},
1284           "bars": false,
1285           "dashLength": 10,
1286           "dashes": false,
1287           "datasource": "${DS_PROMETHEUS}",
1288           "fill": 1,
1289           "id": 32,
1290           "legend": {
1291             "avg": false,
1292             "current": false,
1293             "max": false,
1294             "min": false,
1295             "show": true,
1296             "total": false,
1297             "values": false
1298           },
1299           "lines": true,
1300           "linewidth": 1,
1301           "links": [],
1302           "nullPointMode": "null",
1303           "percentage": false,
1304           "pointradius": 5,
1305           "points": false,
1306           "renderer": "flot",
1307           "seriesOverrides": [
1308             {
1309               "alias": "removed",
1310               "transform": "negative-Y"
1311             }
1312           ],
1313           "spaceLength": 10,
1314           "span": 6,
1315           "stack": false,
1316           "steppedLine": false,
1317           "targets": [
1318             {
1319               "expr": "sum( increase(prometheus_tsdb_head_series_created_total{instance=~\"$instance\"}[5m]) )",
1320               "format": "time_series",
1321               "intervalFactor": 2,
1322               "legendFormat": "created",
1323               "refId": "A",
1324               "step": 4
1325             },
1326             {
1327               "expr": "sum( increase(prometheus_tsdb_head_series_removed_total{instance=~\"$instance\"}[5m]) )",
1328               "format": "time_series",
1329               "intervalFactor": 2,
1330               "legendFormat": "removed",
1331               "refId": "B",
1332               "step": 4
1333             }
1334           ],
1335           "thresholds": [],
1336           "timeFrom": null,
1337           "timeShift": null,
1338           "title": "Series Created / Removed",
1339           "tooltip": {
1340             "shared": true,
1341             "sort": 0,
1342             "value_type": "individual"
1343           },
1344           "type": "graph",
1345           "xaxis": {
1346             "buckets": null,
1347             "mode": "time",
1348             "name": null,
1349             "show": true,
1350             "values": []
1351           },
1352           "yaxes": [
1353             {
1354               "format": "short",
1355               "label": "Series Count",
1356               "logBase": 1,
1357               "max": null,
1358               "min": null,
1359               "show": true
1360             },
1361             {
1362               "format": "short",
1363               "label": null,
1364               "logBase": 1,
1365               "max": null,
1366               "min": null,
1367               "show": true
1368             }
1369           ]
1370         }
1371       ],
1372       "repeat": null,
1373       "repeatIteration": null,
1374       "repeatRowId": null,
1375       "showTitle": false,
1376       "title": "series",
1377       "titleSize": "h6"
1378     },
1379     {
1380       "collapse": false,
1381       "height": 250,
1382       "panels": [
1383         {
1384           "aliasColors": {
1385             "10.58.3.10:80": "#BA43A9"
1386           },
1387           "bars": false,
1388           "dashLength": 10,
1389           "dashes": false,
1390           "datasource": "${DS_PROMETHEUS}",
1391           "description": "Rate of total number of appended samples",
1392           "fill": 1,
1393           "id": 4,
1394           "legend": {
1395             "avg": false,
1396             "current": false,
1397             "max": false,
1398             "min": false,
1399             "show": true,
1400             "total": false,
1401             "values": false
1402           },
1403           "lines": true,
1404           "linewidth": 1,
1405           "links": [],
1406           "nullPointMode": "null",
1407           "percentage": false,
1408           "pointradius": 5,
1409           "points": false,
1410           "renderer": "flot",
1411           "seriesOverrides": [],
1412           "spaceLength": 10,
1413           "span": 12,
1414           "stack": false,
1415           "steppedLine": false,
1416           "targets": [
1417             {
1418               "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[1m])",
1419               "format": "time_series",
1420               "intervalFactor": 2,
1421               "legendFormat": "{{instance}}",
1422               "refId": "A",
1423               "step": 2
1424             }
1425           ],
1426           "thresholds": [],
1427           "timeFrom": null,
1428           "timeShift": null,
1429           "title": "Appended Samples per Second",
1430           "tooltip": {
1431             "shared": true,
1432             "sort": 0,
1433             "value_type": "individual"
1434           },
1435           "type": "graph",
1436           "xaxis": {
1437             "buckets": null,
1438             "mode": "time",
1439             "name": null,
1440             "show": true,
1441             "values": []
1442           },
1443           "yaxes": [
1444             {
1445               "format": "short",
1446               "label": "Samples / Second",
1447               "logBase": 1,
1448               "max": null,
1449               "min": "0",
1450               "show": true
1451             },
1452             {
1453               "format": "short",
1454               "label": null,
1455               "logBase": 1,
1456               "max": null,
1457               "min": null,
1458               "show": true
1459             }
1460           ]
1461         }
1462       ],
1463       "repeat": null,
1464       "repeatIteration": null,
1465       "repeatRowId": null,
1466       "showTitle": false,
1467       "title": "appended samples",
1468       "titleSize": "h6"
1469     },
1470     {
1471       "collapse": false,
1472       "height": 250,
1473       "panels": [
1474         {
1475           "aliasColors": {},
1476           "bars": false,
1477           "dashLength": 10,
1478           "dashes": false,
1479           "datasource": "${DS_PROMETHEUS}",
1480           "description": "Total number of syncs that were executed on a scrape pool.",
1481           "fill": 1,
1482           "id": 6,
1483           "legend": {
1484             "avg": false,
1485             "current": false,
1486             "max": false,
1487             "min": false,
1488             "show": true,
1489             "total": false,
1490             "values": false
1491           },
1492           "lines": true,
1493           "linewidth": 1,
1494           "links": [],
1495           "nullPointMode": "null",
1496           "percentage": false,
1497           "pointradius": 5,
1498           "points": false,
1499           "renderer": "flot",
1500           "seriesOverrides": [],
1501           "spaceLength": 10,
1502           "span": 6,
1503           "stack": false,
1504           "steppedLine": false,
1505           "targets": [
1506             {
1507               "expr": "sum(prometheus_target_scrape_pool_sync_total{job=~\"$job\",instance=~\"$instance\"}) by (scrape_job)",
1508               "format": "time_series",
1509               "hide": false,
1510               "intervalFactor": 2,
1511               "legendFormat": "{{scrape_job}}",
1512               "refId": "B",
1513               "step": 4
1514             }
1515           ],
1516           "thresholds": [],
1517           "timeFrom": null,
1518           "timeShift": null,
1519           "title": "Scrape Sync Total",
1520           "tooltip": {
1521             "shared": true,
1522             "sort": 0,
1523             "value_type": "individual"
1524           },
1525           "type": "graph",
1526           "xaxis": {
1527             "buckets": null,
1528             "mode": "time",
1529             "name": null,
1530             "show": true,
1531             "values": []
1532           },
1533           "yaxes": [
1534             {
1535               "format": "short",
1536               "label": "Syncs",
1537               "logBase": 1,
1538               "max": null,
1539               "min": "0",
1540               "show": true
1541             },
1542             {
1543               "format": "short",
1544               "label": null,
1545               "logBase": 1,
1546               "max": null,
1547               "min": null,
1548               "show": true
1549             }
1550           ]
1551         },
1552         {
1553           "aliasColors": {},
1554           "bars": false,
1555           "dashLength": 10,
1556           "dashes": false,
1557           "datasource": "${DS_PROMETHEUS}",
1558           "description": "Actual interval to sync the scrape pool.",
1559           "fill": 1,
1560           "id": 21,
1561           "legend": {
1562             "avg": false,
1563             "current": false,
1564             "max": false,
1565             "min": false,
1566             "show": true,
1567             "total": false,
1568             "values": false
1569           },
1570           "lines": true,
1571           "linewidth": 1,
1572           "links": [],
1573           "nullPointMode": "null",
1574           "percentage": false,
1575           "pointradius": 5,
1576           "points": false,
1577           "renderer": "flot",
1578           "seriesOverrides": [],
1579           "spaceLength": 10,
1580           "span": 6,
1581           "stack": false,
1582           "steppedLine": false,
1583           "targets": [
1584             {
1585               "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[2m])) by (scrape_job) * 1000",
1586               "format": "time_series",
1587               "intervalFactor": 2,
1588               "legendFormat": "{{scrape_job}}",
1589               "refId": "A",
1590               "step": 4
1591             }
1592           ],
1593           "thresholds": [],
1594           "timeFrom": null,
1595           "timeShift": null,
1596           "title": "Target Sync",
1597           "tooltip": {
1598             "shared": true,
1599             "sort": 0,
1600             "value_type": "individual"
1601           },
1602           "type": "graph",
1603           "xaxis": {
1604             "buckets": null,
1605             "mode": "time",
1606             "name": null,
1607             "show": true,
1608             "values": []
1609           },
1610           "yaxes": [
1611             {
1612               "format": "short",
1613               "label": "Milliseconds",
1614               "logBase": 1,
1615               "max": null,
1616               "min": "0",
1617               "show": true
1618             },
1619             {
1620               "format": "short",
1621               "label": null,
1622               "logBase": 1,
1623               "max": null,
1624               "min": null,
1625               "show": true
1626             }
1627           ]
1628         }
1629       ],
1630       "repeat": null,
1631       "repeatIteration": null,
1632       "repeatRowId": null,
1633       "showTitle": false,
1634       "title": "sync",
1635       "titleSize": "h6"
1636     },
1637     {
1638       "collapse": false,
1639       "height": 250,
1640       "panels": [
1641         {
1642           "aliasColors": {},
1643           "bars": false,
1644           "dashLength": 10,
1645           "dashes": false,
1646           "datasource": "${DS_PROMETHEUS}",
1647           "fill": 1,
1648           "id": 29,
1649           "legend": {
1650             "avg": false,
1651             "current": false,
1652             "max": false,
1653             "min": false,
1654             "show": true,
1655             "total": false,
1656             "values": false
1657           },
1658           "lines": true,
1659           "linewidth": 1,
1660           "links": [],
1661           "nullPointMode": "null",
1662           "percentage": false,
1663           "pointradius": 5,
1664           "points": false,
1665           "renderer": "flot",
1666           "seriesOverrides": [],
1667           "spaceLength": 10,
1668           "span": 6,
1669           "stack": false,
1670           "steppedLine": false,
1671           "targets": [
1672             {
1673               "expr": "scrape_duration_seconds{instance=~\"$instance\"}",
1674               "format": "time_series",
1675               "intervalFactor": 2,
1676               "legendFormat": "{{instance}}",
1677               "refId": "A",
1678               "step": 4
1679             }
1680           ],
1681           "thresholds": [],
1682           "timeFrom": null,
1683           "timeShift": null,
1684           "title": "Scrape Duration",
1685           "tooltip": {
1686             "shared": true,
1687             "sort": 0,
1688             "value_type": "individual"
1689           },
1690           "type": "graph",
1691           "xaxis": {
1692             "buckets": null,
1693             "mode": "time",
1694             "name": null,
1695             "show": true,
1696             "values": []
1697           },
1698           "yaxes": [
1699             {
1700               "format": "short",
1701               "label": "Seconds",
1702               "logBase": 1,
1703               "max": null,
1704               "min": null,
1705               "show": true
1706             },
1707             {
1708               "format": "short",
1709               "label": null,
1710               "logBase": 1,
1711               "max": null,
1712               "min": null,
1713               "show": true
1714             }
1715           ]
1716         },
1717         {
1718           "aliasColors": {},
1719           "bars": false,
1720           "dashLength": 10,
1721           "dashes": false,
1722           "datasource": "${DS_PROMETHEUS}",
1723           "description": "Total number of rejected scrapes",
1724           "fill": 1,
1725           "id": 30,
1726           "legend": {
1727             "avg": false,
1728             "current": false,
1729             "max": false,
1730             "min": false,
1731             "show": true,
1732             "total": false,
1733             "values": false
1734           },
1735           "lines": true,
1736           "linewidth": 1,
1737           "links": [],
1738           "nullPointMode": "null",
1739           "percentage": false,
1740           "pointradius": 5,
1741           "points": false,
1742           "renderer": "flot",
1743           "seriesOverrides": [],
1744           "spaceLength": 10,
1745           "span": 6,
1746           "stack": false,
1747           "steppedLine": false,
1748           "targets": [
1749             {
1750               "expr": "sum(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"})",
1751               "format": "time_series",
1752               "intervalFactor": 2,
1753               "legendFormat": "exceeded sample limit",
1754               "refId": "A",
1755               "step": 4
1756             },
1757             {
1758               "expr": "sum(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"})",
1759               "format": "time_series",
1760               "hide": false,
1761               "intervalFactor": 2,
1762               "legendFormat": "duplicate timestamp",
1763               "refId": "B",
1764               "step": 4
1765             },
1766             {
1767               "expr": "sum(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"})",
1768               "format": "time_series",
1769               "hide": false,
1770               "intervalFactor": 2,
1771               "legendFormat": "out of bounds",
1772               "refId": "C",
1773               "step": 4
1774             },
1775             {
1776               "expr": "sum(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}) ",
1777               "format": "time_series",
1778               "hide": false,
1779               "intervalFactor": 2,
1780               "legendFormat": "out of order",
1781               "refId": "D",
1782               "step": 4
1783             }
1784           ],
1785           "thresholds": [],
1786           "timeFrom": null,
1787           "timeShift": null,
1788           "title": "Rejected Scrapes",
1789           "tooltip": {
1790             "shared": true,
1791             "sort": 0,
1792             "value_type": "individual"
1793           },
1794           "type": "graph",
1795           "xaxis": {
1796             "buckets": null,
1797             "mode": "time",
1798             "name": null,
1799             "show": true,
1800             "values": []
1801           },
1802           "yaxes": [
1803             {
1804               "decimals": 0,
1805               "format": "short",
1806               "label": "Scrapes",
1807               "logBase": 1,
1808               "max": null,
1809               "min": "0",
1810               "show": true
1811             },
1812             {
1813               "format": "short",
1814               "label": null,
1815               "logBase": 1,
1816               "max": null,
1817               "min": null,
1818               "show": true
1819             }
1820           ]
1821         }
1822       ],
1823       "repeat": null,
1824       "repeatIteration": null,
1825       "repeatRowId": null,
1826       "showTitle": false,
1827       "title": "scrapes",
1828       "titleSize": "h6"
1829     },
1830     {
1831       "collapse": false,
1832       "height": 250,
1833       "panels": [
1834         {
1835           "aliasColors": {},
1836           "bars": false,
1837           "dashLength": 10,
1838           "dashes": false,
1839           "datasource": "${DS_PROMETHEUS}",
1840           "description": "The duration of rule group evaluations",
1841           "fill": 1,
1842           "id": 10,
1843           "legend": {
1844             "avg": false,
1845             "current": false,
1846             "max": false,
1847             "min": false,
1848             "show": true,
1849             "total": false,
1850             "values": false
1851           },
1852           "lines": true,
1853           "linewidth": 1,
1854           "links": [],
1855           "nullPointMode": "null",
1856           "percentage": false,
1857           "pointradius": 5,
1858           "points": false,
1859           "renderer": "flot",
1860           "seriesOverrides": [],
1861           "spaceLength": 10,
1862           "span": 6,
1863           "stack": false,
1864           "steppedLine": false,
1865           "targets": [
1866             {
1867               "expr": "1000 * rate(prometheus_evaluator_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[5m]) / rate(prometheus_evaluator_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[5m])",
1868               "format": "time_series",
1869               "intervalFactor": 2,
1870               "legendFormat": "{{instance}}",
1871               "refId": "E",
1872               "step": 4
1873             }
1874           ],
1875           "thresholds": [],
1876           "timeFrom": null,
1877           "timeShift": null,
1878           "title": "Average Rule Evaluation Duration",
1879           "tooltip": {
1880             "shared": true,
1881             "sort": 0,
1882             "value_type": "individual"
1883           },
1884           "type": "graph",
1885           "xaxis": {
1886             "buckets": null,
1887             "mode": "time",
1888             "name": null,
1889             "show": true,
1890             "values": []
1891           },
1892           "yaxes": [
1893             {
1894               "format": "short",
1895               "label": "Milliseconds",
1896               "logBase": 1,
1897               "max": null,
1898               "min": "0",
1899               "show": true
1900             },
1901             {
1902               "format": "short",
1903               "label": null,
1904               "logBase": 1,
1905               "max": null,
1906               "min": null,
1907               "show": true
1908             }
1909           ]
1910         },
1911         {
1912           "aliasColors": {},
1913           "bars": false,
1914           "dashLength": 10,
1915           "dashes": false,
1916           "datasource": "${DS_PROMETHEUS}",
1917           "fill": 1,
1918           "id": 11,
1919           "legend": {
1920             "avg": false,
1921             "current": false,
1922             "max": false,
1923             "min": false,
1924             "show": true,
1925             "total": false,
1926             "values": false
1927           },
1928           "lines": true,
1929           "linewidth": 1,
1930           "links": [],
1931           "nullPointMode": "null",
1932           "percentage": false,
1933           "pointradius": 5,
1934           "points": false,
1935           "renderer": "flot",
1936           "seriesOverrides": [],
1937           "spaceLength": 10,
1938           "span": 6,
1939           "stack": false,
1940           "steppedLine": false,
1941           "targets": [
1942             {
1943               "expr": "sum(rate(http_request_duration_microseconds_count{job=~\"$job\",instance=~\"$instance\"}[1m])) by (handler) > 0",
1944               "format": "time_series",
1945               "intervalFactor": 2,
1946               "legendFormat": "{{handler}}",
1947               "refId": "A",
1948               "step": 4
1949             }
1950           ],
1951           "thresholds": [],
1952           "timeFrom": null,
1953           "timeShift": null,
1954           "title": "HTTP Request Duration",
1955           "tooltip": {
1956             "shared": true,
1957             "sort": 0,
1958             "value_type": "individual"
1959           },
1960           "type": "graph",
1961           "xaxis": {
1962             "buckets": null,
1963             "mode": "time",
1964             "name": null,
1965             "show": true,
1966             "values": []
1967           },
1968           "yaxes": [
1969             {
1970               "format": "short",
1971               "label": "Microseconds",
1972               "logBase": 1,
1973               "max": null,
1974               "min": null,
1975               "show": true
1976             },
1977             {
1978               "format": "short",
1979               "label": null,
1980               "logBase": 1,
1981               "max": null,
1982               "min": null,
1983               "show": true
1984             }
1985           ]
1986         },
1987         {
1988           "aliasColors": {},
1989           "bars": false,
1990           "dashLength": 10,
1991           "dashes": false,
1992           "datasource": "${DS_PROMETHEUS}",
1993           "fill": 1,
1994           "id": 15,
1995           "legend": {
1996             "avg": false,
1997             "current": false,
1998             "max": false,
1999             "min": false,
2000             "show": true,
2001             "total": false,
2002             "values": false
2003           },
2004           "lines": true,
2005           "linewidth": 1,
2006           "links": [],
2007           "nullPointMode": "null",
2008           "percentage": false,
2009           "pointradius": 5,
2010           "points": false,
2011           "renderer": "flot",
2012           "seriesOverrides": [],
2013           "spaceLength": 10,
2014           "span": 6,
2015           "stack": false,
2016           "steppedLine": false,
2017           "targets": [
2018             {
2019               "expr": "sum(prometheus_engine_query_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"}) by (slice)",
2020               "format": "time_series",
2021               "intervalFactor": 2,
2022               "legendFormat": "{{slice}}",
2023               "refId": "A",
2024               "step": 4
2025             }
2026           ],
2027           "thresholds": [],
2028           "timeFrom": null,
2029           "timeShift": null,
2030           "title": "Prometheus Engine Query Duration Seconds",
2031           "tooltip": {
2032             "shared": true,
2033             "sort": 0,
2034             "value_type": "individual"
2035           },
2036           "type": "graph",
2037           "xaxis": {
2038             "buckets": null,
2039             "mode": "time",
2040             "name": null,
2041             "show": true,
2042             "values": []
2043           },
2044           "yaxes": [
2045             {
2046               "format": "short",
2047               "label": "Seconds",
2048               "logBase": 1,
2049               "max": null,
2050               "min": "0",
2051               "show": true
2052             },
2053             {
2054               "format": "short",
2055               "label": null,
2056               "logBase": 1,
2057               "max": null,
2058               "min": null,
2059               "show": true
2060             }
2061           ]
2062         },
2063         {
2064           "aliasColors": {},
2065           "bars": false,
2066           "dashLength": 10,
2067           "dashes": false,
2068           "datasource": "${DS_PROMETHEUS}",
2069           "description": "Rule-group evaluations \n - total\n - missed due to slow rule group evaluation\n - skipped due to throttled metric storage",
2070           "fill": 1,
2071           "id": 31,
2072           "legend": {
2073             "avg": false,
2074             "current": false,
2075             "max": false,
2076             "min": false,
2077             "show": true,
2078             "total": false,
2079             "values": false
2080           },
2081           "lines": true,
2082           "linewidth": 1,
2083           "links": [],
2084           "nullPointMode": "null",
2085           "percentage": false,
2086           "pointradius": 5,
2087           "points": false,
2088           "renderer": "flot",
2089           "seriesOverrides": [],
2090           "spaceLength": 10,
2091           "span": 6,
2092           "stack": false,
2093           "steppedLine": false,
2094           "targets": [
2095             {
2096               "expr": "sum(rate(prometheus_evaluator_iterations_total{job=~\"$job\", instance=~\"$instance\"}[5m]))",
2097               "format": "time_series",
2098               "intervalFactor": 2,
2099               "legendFormat": "Total",
2100               "refId": "B",
2101               "step": 4
2102             },
2103             {
2104               "expr": "sum(rate(prometheus_evaluator_iterations_missed_total{job=~\"$job\", instance=~\"$instance\"}[5m]))",
2105               "format": "time_series",
2106               "intervalFactor": 2,
2107               "legendFormat": "Missed",
2108               "refId": "A",
2109               "step": 4
2110             },
2111             {
2112               "expr": "sum(rate(prometheus_evaluator_iterations_skipped_total{job=~\"$job\", instance=~\"$instance\"}[5m]))",
2113               "format": "time_series",
2114               "intervalFactor": 2,
2115               "legendFormat": "Skipped",
2116               "refId": "C",
2117               "step": 4
2118             }
2119           ],
2120           "thresholds": [],
2121           "timeFrom": null,
2122           "timeShift": null,
2123           "title": "Rule Evaluator Iterations",
2124           "tooltip": {
2125             "shared": true,
2126             "sort": 0,
2127             "value_type": "individual"
2128           },
2129           "type": "graph",
2130           "xaxis": {
2131             "buckets": null,
2132             "mode": "time",
2133             "name": null,
2134             "show": true,
2135             "values": []
2136           },
2137           "yaxes": [
2138             {
2139               "format": "short",
2140               "label": "iterations",
2141               "logBase": 1,
2142               "max": null,
2143               "min": "0",
2144               "show": true
2145             },
2146             {
2147               "format": "short",
2148               "label": null,
2149               "logBase": 1,
2150               "max": null,
2151               "min": null,
2152               "show": true
2153             }
2154           ]
2155         }
2156       ],
2157       "repeat": null,
2158       "repeatIteration": null,
2159       "repeatRowId": null,
2160       "showTitle": false,
2161       "title": "durations",
2162       "titleSize": "h6"
2163     },
2164     {
2165       "collapse": false,
2166       "height": 250,
2167       "panels": [
2168         {
2169           "aliasColors": {},
2170           "bars": false,
2171           "dashLength": 10,
2172           "dashes": false,
2173           "datasource": "${DS_PROMETHEUS}",
2174           "fill": 1,
2175           "id": 22,
2176           "legend": {
2177             "avg": false,
2178             "current": false,
2179             "max": false,
2180             "min": false,
2181             "show": true,
2182             "total": false,
2183             "values": false
2184           },
2185           "lines": true,
2186           "linewidth": 1,
2187           "links": [],
2188           "nullPointMode": "null",
2189           "percentage": false,
2190           "pointradius": 5,
2191           "points": false,
2192           "renderer": "flot",
2193           "seriesOverrides": [],
2194           "spaceLength": 10,
2195           "span": 12,
2196           "stack": false,
2197           "steppedLine": false,
2198           "targets": [
2199             {
2200               "expr": "rate(prometheus_notifications_sent_total[5m])",
2201               "format": "time_series",
2202               "intervalFactor": 2,
2203               "legendFormat": "{{instance}}",
2204               "refId": "A",
2205               "step": 2
2206             }
2207           ],
2208           "thresholds": [],
2209           "timeFrom": null,
2210           "timeShift": null,
2211           "title": "Notifications Sent",
2212           "tooltip": {
2213             "shared": true,
2214             "sort": 0,
2215             "value_type": "individual"
2216           },
2217           "type": "graph",
2218           "xaxis": {
2219             "buckets": null,
2220             "mode": "time",
2221             "name": null,
2222             "show": true,
2223             "values": []
2224           },
2225           "yaxes": [
2226             {
2227               "format": "short",
2228               "label": "Notifications",
2229               "logBase": 1,
2230               "max": null,
2231               "min": "0",
2232               "show": true
2233             },
2234             {
2235               "format": "short",
2236               "label": null,
2237               "logBase": 1,
2238               "max": null,
2239               "min": null,
2240               "show": true
2241             }
2242           ]
2243         }
2244       ],
2245       "repeat": null,
2246       "repeatIteration": null,
2247       "repeatRowId": null,
2248       "showTitle": false,
2249       "title": "notifications",
2250       "titleSize": "h6"
2251     },
2252     {
2253       "collapse": false,
2254       "height": 250,
2255       "panels": [
2256         {
2257           "aliasColors": {},
2258           "bars": false,
2259           "dashLength": 10,
2260           "dashes": false,
2261           "datasource": "${DS_PROMETHEUS}",
2262           "fill": 1,
2263           "id": 23,
2264           "legend": {
2265             "avg": false,
2266             "current": false,
2267             "max": false,
2268             "min": false,
2269             "show": true,
2270             "total": false,
2271             "values": false
2272           },
2273           "lines": true,
2274           "linewidth": 1,
2275           "links": [],
2276           "nullPointMode": "null",
2277           "percentage": false,
2278           "pointradius": 5,
2279           "points": false,
2280           "renderer": "flot",
2281           "seriesOverrides": [],
2282           "spaceLength": 10,
2283           "span": 6,
2284           "stack": false,
2285           "steppedLine": false,
2286           "targets": [
2287             {
2288               "expr": "(time() - prometheus_config_last_reload_success_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"}) / 60",
2289               "format": "time_series",
2290               "intervalFactor": 2,
2291               "legendFormat": "{{instance}}",
2292               "refId": "A",
2293               "step": 4
2294             }
2295           ],
2296           "thresholds": [],
2297           "timeFrom": null,
2298           "timeShift": null,
2299           "title": "Minutes Since Successful Config Reload",
2300           "tooltip": {
2301             "shared": true,
2302             "sort": 0,
2303             "value_type": "individual"
2304           },
2305           "type": "graph",
2306           "xaxis": {
2307             "buckets": null,
2308             "mode": "time",
2309             "name": null,
2310             "show": true,
2311             "values": []
2312           },
2313           "yaxes": [
2314             {
2315               "format": "short",
2316               "label": "Minutes",
2317               "logBase": 1,
2318               "max": null,
2319               "min": "0",
2320               "show": true
2321             },
2322             {
2323               "format": "short",
2324               "label": null,
2325               "logBase": 1,
2326               "max": null,
2327               "min": null,
2328               "show": true
2329             }
2330           ]
2331         },
2332         {
2333           "aliasColors": {},
2334           "bars": false,
2335           "dashLength": 10,
2336           "dashes": false,
2337           "datasource": "${DS_PROMETHEUS}",
2338           "fill": 1,
2339           "id": 24,
2340           "legend": {
2341             "avg": false,
2342             "current": false,
2343             "max": false,
2344             "min": false,
2345             "show": true,
2346             "total": false,
2347             "values": false
2348           },
2349           "lines": true,
2350           "linewidth": 1,
2351           "links": [],
2352           "nullPointMode": "null",
2353           "percentage": false,
2354           "pointradius": 5,
2355           "points": false,
2356           "renderer": "flot",
2357           "seriesOverrides": [],
2358           "spaceLength": 10,
2359           "span": 6,
2360           "stack": false,
2361           "steppedLine": false,
2362           "targets": [
2363             {
2364               "expr": "prometheus_config_last_reload_successful{job=~\"$job\",instance=~\"$instance\"}",
2365               "format": "time_series",
2366               "intervalFactor": 2,
2367               "legendFormat": "{{instance}}",
2368               "refId": "A",
2369               "step": 4
2370             }
2371           ],
2372           "thresholds": [],
2373           "timeFrom": null,
2374           "timeShift": null,
2375           "title": "Successful Config Reload",
2376           "tooltip": {
2377             "shared": true,
2378             "sort": 0,
2379             "value_type": "individual"
2380           },
2381           "type": "graph",
2382           "xaxis": {
2383             "buckets": null,
2384             "mode": "time",
2385             "name": null,
2386             "show": true,
2387             "values": []
2388           },
2389           "yaxes": [
2390             {
2391               "decimals": 0,
2392               "format": "short",
2393               "label": "Success",
2394               "logBase": 1,
2395               "max": "1",
2396               "min": "0",
2397               "show": true
2398             },
2399             {
2400               "format": "short",
2401               "label": null,
2402               "logBase": 1,
2403               "max": null,
2404               "min": null,
2405               "show": true
2406             }
2407           ]
2408         }
2409       ],
2410       "repeat": null,
2411       "repeatIteration": null,
2412       "repeatRowId": null,
2413       "showTitle": false,
2414       "title": "config",
2415       "titleSize": "h6"
2416     },
2417     {
2418       "collapse": false,
2419       "height": 250,
2420       "panels": [
2421         {
2422           "aliasColors": {},
2423           "bars": false,
2424           "dashLength": 10,
2425           "dashes": false,
2426           "datasource": "${DS_PROMETHEUS}",
2427           "description": "GC invocation durations",
2428           "fill": 1,
2429           "id": 28,
2430           "legend": {
2431             "avg": false,
2432             "current": false,
2433             "max": false,
2434             "min": false,
2435             "show": true,
2436             "total": false,
2437             "values": false
2438           },
2439           "lines": true,
2440           "linewidth": 1,
2441           "links": [],
2442           "nullPointMode": "null",
2443           "percentage": false,
2444           "pointradius": 5,
2445           "points": false,
2446           "renderer": "flot",
2447           "seriesOverrides": [],
2448           "spaceLength": 10,
2449           "span": 12,
2450           "stack": false,
2451           "steppedLine": false,
2452           "targets": [
2453             {
2454               "expr": "sum(rate(go_gc_duration_seconds_sum{instance=~\"$instance\",job=~\"$job\"}[2m])) by (instance)",
2455               "format": "time_series",
2456               "intervalFactor": 2,
2457               "legendFormat": "{{instance}}",
2458               "refId": "A",
2459               "step": 2
2460             }
2461           ],
2462           "thresholds": [],
2463           "timeFrom": null,
2464           "timeShift": null,
2465           "title": "GC Rate / 2m",
2466           "tooltip": {
2467             "shared": true,
2468             "sort": 0,
2469             "value_type": "individual"
2470           },
2471           "type": "graph",
2472           "xaxis": {
2473             "buckets": null,
2474             "mode": "time",
2475             "name": null,
2476             "show": true,
2477             "values": []
2478           },
2479           "yaxes": [
2480             {
2481               "format": "short",
2482               "label": null,
2483               "logBase": 1,
2484               "max": null,
2485               "min": null,
2486               "show": true
2487             },
2488             {
2489               "format": "short",
2490               "label": null,
2491               "logBase": 1,
2492               "max": null,
2493               "min": null,
2494               "show": true
2495             }
2496           ]
2497         }
2498       ],
2499       "repeat": null,
2500       "repeatIteration": null,
2501       "repeatRowId": null,
2502       "showTitle": false,
2503       "title": "garbage collection",
2504       "titleSize": "h6"
2505     },
2506     {
2507       "collapse": true,
2508       "height": 250,
2509       "panels": [
2510         {
2511           "aliasColors": {},
2512           "bars": false,
2513           "dashLength": 10,
2514           "dashes": false,
2515           "datasource": "${DS_PROMETHEUS}",
2516           "description": "This is probably wrong!  Please help.",
2517           "fill": 1,
2518           "id": 26,
2519           "legend": {
2520             "avg": false,
2521             "current": false,
2522             "max": false,
2523             "min": false,
2524             "show": true,
2525             "total": false,
2526             "values": false
2527           },
2528           "lines": true,
2529           "linewidth": 1,
2530           "links": [],
2531           "nullPointMode": "null",
2532           "percentage": false,
2533           "pointradius": 5,
2534           "points": false,
2535           "renderer": "flot",
2536           "seriesOverrides": [
2537             {
2538               "alias": "allocated",
2539               "stack": false
2540             }
2541           ],
2542           "spaceLength": 10,
2543           "span": 6,
2544           "stack": true,
2545           "steppedLine": false,
2546           "targets": [
2547             {
2548               "expr": "sum(go_memstats_alloc_bytes_total{job=~\"$job\", instance=~\"$instance\"})",
2549               "format": "time_series",
2550               "hide": true,
2551               "intervalFactor": 2,
2552               "legendFormat": "alloc_bytes_total",
2553               "refId": "A",
2554               "step": 10
2555             },
2556             {
2557               "expr": "sum(go_memstats_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})",
2558               "format": "time_series",
2559               "hide": false,
2560               "intervalFactor": 2,
2561               "legendFormat": "allocated",
2562               "refId": "B",
2563               "step": 10
2564             },
2565             {
2566               "expr": "sum(go_memstats_buck_hash_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2567               "format": "time_series",
2568               "intervalFactor": 2,
2569               "legendFormat": "profiling bucket hash table",
2570               "refId": "C",
2571               "step": 10
2572             },
2573             {
2574               "expr": "sum(go_memstats_gc_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2575               "format": "time_series",
2576               "intervalFactor": 2,
2577               "legendFormat": "GC metadata",
2578               "refId": "D",
2579               "step": 10
2580             },
2581             {
2582               "expr": "sum(go_memstats_heap_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})",
2583               "format": "time_series",
2584               "intervalFactor": 2,
2585               "legendFormat": "heap in-use",
2586               "refId": "E",
2587               "step": 10
2588             },
2589             {
2590               "expr": "sum(go_memstats_heap_idle_bytes{job=~\"$job\", instance=~\"$instance\"})",
2591               "format": "time_series",
2592               "intervalFactor": 2,
2593               "legendFormat": "heap idle",
2594               "refId": "F",
2595               "step": 10
2596             },
2597             {
2598               "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
2599               "format": "time_series",
2600               "intervalFactor": 2,
2601               "legendFormat": "heap in use",
2602               "refId": "G",
2603               "step": 10
2604             },
2605             {
2606               "expr": "sum(go_memstats_heap_released_bytes{job=~\"$job\", instance=~\"$instance\"})",
2607               "format": "time_series",
2608               "intervalFactor": 2,
2609               "legendFormat": "heap released",
2610               "refId": "H",
2611               "step": 10
2612             },
2613             {
2614               "expr": "sum(go_memstats_heap_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2615               "format": "time_series",
2616               "intervalFactor": 2,
2617               "legendFormat": "heap system",
2618               "refId": "I",
2619               "step": 10
2620             },
2621             {
2622               "expr": "sum(go_memstats_mcache_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
2623               "format": "time_series",
2624               "intervalFactor": 2,
2625               "legendFormat": "mcache in use",
2626               "refId": "J",
2627               "step": 10
2628             },
2629             {
2630               "expr": "sum(go_memstats_mcache_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2631               "format": "time_series",
2632               "intervalFactor": 2,
2633               "legendFormat": "mcache sys",
2634               "refId": "K",
2635               "step": 10
2636             },
2637             {
2638               "expr": "sum(go_memstats_mspan_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
2639               "format": "time_series",
2640               "intervalFactor": 2,
2641               "legendFormat": "mspan in use",
2642               "refId": "L",
2643               "step": 10
2644             },
2645             {
2646               "expr": "sum(go_memstats_mspan_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2647               "format": "time_series",
2648               "intervalFactor": 2,
2649               "legendFormat": "mspan sys",
2650               "refId": "M",
2651               "step": 10
2652             },
2653             {
2654               "expr": "sum(go_memstats_next_gc_bytes{job=~\"$job\", instance=~\"$instance\"})",
2655               "format": "time_series",
2656               "intervalFactor": 2,
2657               "legendFormat": "heap next gc",
2658               "refId": "N",
2659               "step": 10
2660             },
2661             {
2662               "expr": "sum(go_memstats_other_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2663               "format": "time_series",
2664               "intervalFactor": 2,
2665               "legendFormat": "other sys",
2666               "refId": "O",
2667               "step": 10
2668             },
2669             {
2670               "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
2671               "format": "time_series",
2672               "intervalFactor": 2,
2673               "legendFormat": "stack in use",
2674               "refId": "P",
2675               "step": 10
2676             },
2677             {
2678               "expr": "sum(go_memstats_stack_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2679               "format": "time_series",
2680               "intervalFactor": 2,
2681               "legendFormat": "stack sys",
2682               "refId": "Q",
2683               "step": 10
2684             },
2685             {
2686               "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
2687               "format": "time_series",
2688               "intervalFactor": 2,
2689               "legendFormat": "sys",
2690               "refId": "R",
2691               "step": 10
2692             }
2693           ],
2694           "thresholds": [],
2695           "timeFrom": null,
2696           "timeShift": null,
2697           "title": "Go Memory Usage (FIXME)",
2698           "tooltip": {
2699             "shared": true,
2700             "sort": 0,
2701             "value_type": "individual"
2702           },
2703           "type": "graph",
2704           "xaxis": {
2705             "buckets": null,
2706             "mode": "time",
2707             "name": null,
2708             "show": true,
2709             "values": []
2710           },
2711           "yaxes": [
2712             {
2713               "format": "bytes",
2714               "label": null,
2715               "logBase": 1,
2716               "max": null,
2717               "min": "0",
2718               "show": true
2719             },
2720             {
2721               "format": "short",
2722               "label": null,
2723               "logBase": 1,
2724               "max": null,
2725               "min": null,
2726               "show": true
2727             }
2728           ]
2729         },
2730         {
2731           "aliasColors": {},
2732           "bars": false,
2733           "dashLength": 10,
2734           "dashes": false,
2735           "datasource": "${DS_PROMETHEUS}",
2736           "fill": 1,
2737           "id": 9,
2738           "legend": {
2739             "avg": false,
2740             "current": false,
2741             "max": false,
2742             "min": false,
2743             "show": true,
2744             "total": false,
2745             "values": false
2746           },
2747           "lines": true,
2748           "linewidth": 1,
2749           "links": [],
2750           "nullPointMode": "null",
2751           "percentage": false,
2752           "pointradius": 5,
2753           "points": false,
2754           "renderer": "flot",
2755           "seriesOverrides": [],
2756           "spaceLength": 10,
2757           "span": 3,
2758           "stack": false,
2759           "steppedLine": false,
2760           "targets": [
2761             {
2762               "expr": "prometheus_target_interval_length_seconds{instance=~\"$instance\", job=~\"$job\"}",
2763               "format": "time_series",
2764               "intervalFactor": 2,
2765               "legendFormat": "{{quantile}} {{interval}}",
2766               "refId": "A",
2767               "step": 20
2768             }
2769           ],
2770           "thresholds": [],
2771           "timeFrom": null,
2772           "timeShift": null,
2773           "title": "Scrape Duration",
2774           "tooltip": {
2775             "shared": true,
2776             "sort": 0,
2777             "value_type": "individual"
2778           },
2779           "type": "graph",
2780           "xaxis": {
2781             "buckets": null,
2782             "mode": "time",
2783             "name": null,
2784             "show": true,
2785             "values": []
2786           },
2787           "yaxes": [
2788             {
2789               "format": "short",
2790               "label": "Seconds",
2791               "logBase": 1,
2792               "max": null,
2793               "min": "0",
2794               "show": true
2795             },
2796             {
2797               "format": "short",
2798               "label": null,
2799               "logBase": 1,
2800               "max": null,
2801               "min": null,
2802               "show": true
2803             }
2804           ]
2805         },
2806         {
2807           "aliasColors": {},
2808           "bars": false,
2809           "dashLength": 10,
2810           "dashes": false,
2811           "datasource": "${DS_PROMETHEUS}",
2812           "fill": 1,
2813           "id": 7,
2814           "legend": {
2815             "avg": false,
2816             "current": false,
2817             "max": false,
2818             "min": false,
2819             "show": true,
2820             "total": false,
2821             "values": false
2822           },
2823           "lines": true,
2824           "linewidth": 1,
2825           "links": [],
2826           "nullPointMode": "null",
2827           "percentage": false,
2828           "pointradius": 5,
2829           "points": false,
2830           "renderer": "flot",
2831           "seriesOverrides": [],
2832           "spaceLength": 10,
2833           "span": 3,
2834           "stack": false,
2835           "steppedLine": false,
2836           "targets": [
2837             {
2838               "expr": "sum(rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m])) by (interval)",
2839               "format": "time_series",
2840               "intervalFactor": 2,
2841               "legendFormat": "{{interval}}",
2842               "refId": "A",
2843               "step": 20
2844             }
2845           ],
2846           "thresholds": [],
2847           "timeFrom": null,
2848           "timeShift": null,
2849           "title": "Target Scrapes / 5m",
2850           "tooltip": {
2851             "shared": true,
2852             "sort": 0,
2853             "value_type": "individual"
2854           },
2855           "type": "graph",
2856           "xaxis": {
2857             "buckets": null,
2858             "mode": "time",
2859             "name": null,
2860             "show": true,
2861             "values": []
2862           },
2863           "yaxes": [
2864             {
2865               "format": "short",
2866               "label": "Scrapes",
2867               "logBase": 1,
2868               "max": null,
2869               "min": "0",
2870               "show": true
2871             },
2872             {
2873               "format": "short",
2874               "label": null,
2875               "logBase": 1,
2876               "max": null,
2877               "min": null,
2878               "show": true
2879             }
2880           ]
2881         }
2882       ],
2883       "repeat": null,
2884       "repeatIteration": null,
2885       "repeatRowId": null,
2886       "showTitle": false,
2887       "title": "Broken, ignore",
2888       "titleSize": "h6"
2889     }
2890   ],
2891   "schemaVersion": 14,
2892   "style": "dark",
2893   "tags": [],
2894   "templating": {
2895     "list": [
2896       {
2897         "current": {
2898           "selected": false,
2899           "text": "default",
2900           "value": "default"
2901         },
2902         "hide": 0,
2903         "includeAll": false,
2904         "label": "datasource",
2905         "multi": false,
2906         "name": "DS_PROMETHEUS",
2907         "options": [],
2908         "query": "prometheus",
2909         "refresh": 1,
2910         "regex": "",
2911         "skipUrlSync": false,
2912         "type": "datasource"
2913       },
2914       {
2915         "allValue": null,
2916         "current": {},
2917         "datasource": "${DS_PROMETHEUS}",
2918         "hide": 0,
2919         "includeAll": true,
2920         "label": null,
2921         "multi": true,
2922         "name": "job",
2923         "options": [],
2924         "query": "query_result(prometheus_tsdb_head_samples_appended_total)",
2925         "refresh": 2,
2926         "regex": "/.*job=\"([^\"]+)/",
2927         "sort": 1,
2928         "tagValuesQuery": "",
2929         "tags": [],
2930         "tagsQuery": "",
2931         "type": "query",
2932         "useTags": false
2933       },
2934       {
2935         "allValue": null,
2936         "current": {},
2937         "datasource": "${DS_PROMETHEUS}",
2938         "hide": 0,
2939         "includeAll": true,
2940         "label": null,
2941         "multi": true,
2942         "name": "instance",
2943         "options": [],
2944         "query": "query_result(up{job=~\"$job\"})",
2945         "refresh": 2,
2946         "regex": "/.*instance=\"([^\"]+).*/",
2947         "sort": 0,
2948         "tagValuesQuery": "",
2949         "tags": [],
2950         "tagsQuery": "",
2951         "type": "query",
2952         "useTags": false
2953       },
2954       {
2955         "allValue": null,
2956         "current": {
2957           "selected": true,
2958           "text": "1h",
2959           "value": "1h"
2960         },
2961         "hide": 0,
2962         "includeAll": false,
2963         "label": null,
2964         "multi": false,
2965         "name": "interval",
2966         "options": [
2967           {
2968             "selected": true,
2969             "text": "1h",
2970             "value": "1h"
2971           },
2972           {
2973             "selected": false,
2974             "text": "3h",
2975             "value": "3h"
2976           },
2977           {
2978             "selected": false,
2979             "text": "6h",
2980             "value": "6h"
2981           },
2982           {
2983             "selected": false,
2984             "text": "12h",
2985             "value": "12h"
2986           },
2987           {
2988             "selected": false,
2989             "text": "1d",
2990             "value": "1d"
2991           },
2992           {
2993             "selected": false,
2994             "text": "2d",
2995             "value": "2d"
2996           },
2997           {
2998             "selected": false,
2999             "text": "7d",
3000             "value": "7d"
3001           },
3002           {
3003             "selected": false,
3004             "text": "30d",
3005             "value": "30d"
3006           },
3007           {
3008             "selected": false,
3009             "text": "90d",
3010             "value": "90d"
3011           },
3012           {
3013             "selected": false,
3014             "text": "180d",
3015             "value": "180d"
3016           }
3017         ],
3018         "query": "1h, 3h, 6h, 12h, 1d, 2d, 7d, 30d, 90d, 180d",
3019         "type": "custom"
3020       }
3021     ]
3022   },
3023   "time": {
3024     "from": "now-4h",
3025     "to": "now"
3026   },
3027   "timepicker": {
3028     "refresh_intervals": [
3029       "5s",
3030       "10s",
3031       "30s",
3032       "1m",
3033       "5m",
3034       "15m",
3035       "30m",
3036       "1h",
3037       "2h",
3038       "1d"
3039     ],
3040     "time_options": [
3041       "5m",
3042       "15m",
3043       "1h",
3044       "6h",
3045       "12h",
3046       "24h",
3047       "2d",
3048       "7d",
3049       "30d"
3050     ]
3051   },
3052   "timezone": "browser",
3053   "title": "Prometheus",
3054   "version": 1
3055 }