Infra: Add reschedule policy

[csit.git] / terraform-ci-infra / 1n_nmd / prometheus / conf / nomad / prometheus.hcl

diff --git a/terraform-ci-infra/1n_nmd/prometheus/conf/nomad/prometheus.hcl b/terraform-ci-infra/1n_nmd/prometheus/conf/nomad/prometheus.hcl
index 4918a5f..adc3031 100644 (file)
--- a/terraform-ci-infra/1n_nmd/prometheus/conf/nomad/prometheus.hcl
+++ b/terraform-ci-infra/1n_nmd/prometheus/conf/nomad/prometheus.hcl
@@ -71,6 +71,17 @@ job "${job_name}" {
 %{ endif }
   }
 
+  # The reschedule stanza specifies the group's rescheduling strategy. If
+  # specified at the job level, the configuration will apply to all groups
+  # within the job. If the reschedule stanza is present on both the job and the
+  # group, they are merged with the group stanza taking the highest precedence
+  # and then the job.
+  reschedule {
+    delay             = "30s"
+    delay_function    = "constant"
+    unlimited         = true
+  }
+
   # The "group" stanza defines a series of tasks that should be co-located on
   # the same Nomad client. Any task within a group will be placed on the same
   # client.
@@ -86,6 +97,18 @@ job "${job_name}" {
     # to 1.
     count               = ${group_count}
 
+    # The restart stanza configures a tasks's behavior on task failure. Restarts
+    # happen on the client that is running the task.
+    #
+    # https://www.nomadproject.io/docs/job-specification/restart
+    #
+    restart {
+      interval  = "30m"
+      attempts  = 40
+      delay     = "15s"
+      mode      = "delay"
+    }
+
     # The volume stanza allows the group to specify that it requires a given
     # volume from the cluster.
     #
@@ -188,6 +211,24 @@ job "${job_name}" {
         data            = <<EOH
 ---
 groups:
+- name: "Jenkins Job Health Exporter"
+  rules:
+  - alert: JenkinsJobHealthExporterFailures
+    expr: jenkins_job_failure{id=~".*"} > jenkins_job_success{id=~".*"}
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: "Jenkins Job Health detected high failure rate on jenkins jobs."
+      description: "Job: {{ $labels.id }}"
+  - alert: JenkinsJobHealthExporterUnstable
+    expr: jenkins_job_unstable{id=~".*"} > jenkins_job_success{id=~".*"}
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Jenkins Job Health detected high unstable rate on jenkins jobs."
+      description: "Job: {{ $labels.id }}"
 - name: "Consul"
   rules:
   - alert: ConsulServiceHealthcheckFailed
@@ -523,6 +564,20 @@ scrape_configs:
       - targets: [ '10.32.8.16:8080' ]
       - targets: [ '10.32.8.17:8080' ]
 
+  - job_name: 'Jenkins Job Health Exporter'
+    static_configs:
+      - targets: [ '10.30.51.32:9186' ]
+    metric_relabel_configs:
+      - source_labels: [ __name__ ]
+        regex: '^(vpp.*|csit.*)_(success|failure|total|unstable|reqtime_ms)$'
+        action: replace
+        replacement: '$1'
+        target_label: id
+      - source_labels: [ __name__ ]
+        regex: '^(vpp.*|csit.*)_(success|failure|total|unstable|reqtime_ms)$'
+        replacement: 'jenkins_job_$2'
+        target_label: __name__
+
   - job_name: 'Node Exporter'
     static_configs:
       - targets: [ '10.30.51.28:9100' ]