Infra: Update monitoring for new layout

[csit.git] / fdio.infra.terraform / 1n_nmd / prometheus / conf / nomad / prometheus.hcl

diff --git a/fdio.infra.terraform/1n_nmd/prometheus/conf/nomad/prometheus.hcl b/fdio.infra.terraform/1n_nmd/prometheus/conf/nomad/prometheus.hcl
index adc3031..4451d72 100644 (file)
--- a/fdio.infra.terraform/1n_nmd/prometheus/conf/nomad/prometheus.hcl
+++ b/fdio.infra.terraform/1n_nmd/prometheus/conf/nomad/prometheus.hcl
@@ -175,7 +175,7 @@ job "${job_name}" {
         args            = [
           "--config.file=secrets/prometheus.yml",
           "--storage.tsdb.path=${data_dir}prometheus/",
-          "--storage.tsdb.retention.time=15d"
+          "--storage.tsdb.retention.time=7d"
         ]
       }
 
@@ -265,14 +265,6 @@ groups:
     annotations:
       summary: "Prometheus target missing (instance {{ $labels.instance }})."
       description: "A Prometheus target has disappeared. An exporter might be crashed."
-  - alert: HostHighCpuLoad
-    expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
-    for: 0m
-    labels:
-      severity: warning
-    annotations:
-      summary: "Host high CPU load (instance {{ $labels.instance }})."
-      description: "CPU load is > 95%."
   - alert: HostOutOfMemory
     expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
     for: 2m
@@ -484,13 +476,13 @@ scrape_configs:
 
   - job_name: 'Consul Cluster'
     static_configs:
+      - targets: [ '10.30.51.22:8500' ]
+      - targets: [ '10.30.51.24:8500' ]
+      - targets: [ '10.30.51.25:8500' ]
+      - targets: [ '10.30.51.26:8500' ]
       - targets: [ '10.30.51.28:8500' ]
       - targets: [ '10.30.51.29:8500' ]
       - targets: [ '10.30.51.30:8500' ]
-      - targets: [ '10.30.51.32:8500' ]
-      - targets: [ '10.30.51.33:8500' ]
-      - targets: [ '10.30.51.34:8500' ]
-      - targets: [ '10.30.51.35:8500' ]
       - targets: [ '10.30.51.39:8500' ]
       - targets: [ '10.30.51.40:8500' ]
       - targets: [ '10.30.51.50:8500' ]
@@ -513,7 +505,7 @@ scrape_configs:
     static_configs:
       - targets: [ 'gerrit.fd.io' ]
       - targets: [ 'jenkins.fd.io' ]
-      - targets: [ '10.30.51.32' ]
+      - targets: [ '10.32.8.17' ]
     params:
       module: [ 'icmp_v4' ]
     relabel_configs:
@@ -540,33 +532,9 @@ scrape_configs:
         replacement: localhost:9115
     metrics_path: /probe
 
-  - job_name: 'cAdvisor Exporter'
-    static_configs:
-      - targets: [ '10.30.51.28:8080' ]
-      - targets: [ '10.30.51.29:8080' ]
-      - targets: [ '10.30.51.30:8080' ]
-      #- targets: [ '10.30.51.32:8080' ]
-      - targets: [ '10.30.51.33:8080' ]
-      - targets: [ '10.30.51.34:8080' ]
-      - targets: [ '10.30.51.35:8080' ]
-      - targets: [ '10.30.51.39:8080' ]
-      - targets: [ '10.30.51.40:8080' ]
-      - targets: [ '10.30.51.50:8080' ]
-      - targets: [ '10.30.51.51:8080' ]
-      - targets: [ '10.30.51.65:8080' ]
-      - targets: [ '10.30.51.66:8080' ]
-      - targets: [ '10.30.51.67:8080' ]
-      - targets: [ '10.30.51.68:8080' ]
-      - targets: [ '10.30.51.70:8080' ]
-      - targets: [ '10.30.51.71:8080' ]
-      - targets: [ '10.32.8.14:8080' ]
-      - targets: [ '10.32.8.15:8080' ]
-      - targets: [ '10.32.8.16:8080' ]
-      - targets: [ '10.32.8.17:8080' ]
-
   - job_name: 'Jenkins Job Health Exporter'
     static_configs:
-      - targets: [ '10.30.51.32:9186' ]
+      - targets: [ '10.30.51.22:9186' ]
     metric_relabel_configs:
       - source_labels: [ __name__ ]
         regex: '^(vpp.*|csit.*)_(success|failure|total|unstable|reqtime_ms)$'
@@ -580,13 +548,13 @@ scrape_configs:
 
   - job_name: 'Node Exporter'
     static_configs:
+      - targets: [ '10.30.51.22:9100' ]
+      - targets: [ '10.30.51.24:9100' ]
+      - targets: [ '10.30.51.25:9100' ]
+      - targets: [ '10.30.51.26:9100' ]
       - targets: [ '10.30.51.28:9100' ]
       - targets: [ '10.30.51.29:9100' ]
       - targets: [ '10.30.51.30:9100' ]
-      - targets: [ '10.30.51.32:9100' ]
-      - targets: [ '10.30.51.33:9100' ]
-      - targets: [ '10.30.51.34:9100' ]
-      - targets: [ '10.30.51.35:9100' ]
       - targets: [ '10.30.51.39:9100' ]
       - targets: [ '10.30.51.40:9100' ]
       - targets: [ '10.30.51.50:9100' ]