Infra: Add reschedule policy

[csit.git] / terraform-ci-infra / 1n_nmd / alertmanager / conf / nomad / alertmanager.hcl

diff --git a/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl b/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl
index fafe623..6b0d669 100644 (file)
--- a/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl
+++ b/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl
@@ -71,6 +71,17 @@ job "${job_name}" {
 %{ endif }
   }
 
+  # The reschedule stanza specifies the group's rescheduling strategy. If
+  # specified at the job level, the configuration will apply to all groups
+  # within the job. If the reschedule stanza is present on both the job and the
+  # group, they are merged with the group stanza taking the highest precedence
+  # and then the job.
+  reschedule {
+    delay             = "30s"
+    delay_function    = "constant"
+    unlimited         = true
+  }
+
   # The "group" stanza defines a series of tasks that should be co-located on
   # the same Nomad client. Any task within a group will be placed on the same
   # client.
@@ -86,6 +97,18 @@ job "${job_name}" {
     # to 1.
     count             = ${group_count}
 
+    # The restart stanza configures a tasks's behavior on task failure. Restarts
+    # happen on the client that is running the task.
+    #
+    # https://www.nomadproject.io/docs/job-specification/restart
+    #
+    restart {
+      interval  = "30m"
+      attempts  = 40
+      delay     = "15s"
+      mode      = "delay"
+    }
+
     # The constraint allows restricting the set of eligible nodes. Constraints
     # may filter on attributes or client metadata.
     #
@@ -160,10 +183,6 @@ job "${job_name}" {
         left_delimiter  = "{{{"
         right_delimiter = "}}}"
         data            = <<EOH
-global:
-  # The API URL to use for Slack notifications.
-  slack_api_url: 'https://hooks.slack.com/services/${slack_api_key}'
-
 # The directory from which notification templates are read.
 templates:
 - '/etc/alertmanager/template/*.tmpl'
@@ -185,7 +204,7 @@ templates:
 
 # The root route on which each incoming alert enters.
 route:
-  receiver: '${default_receiver}'
+  receiver: '${slack_default_receiver}'
 
   # The labels by which incoming alerts are grouped together. For example,
   # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
@@ -217,18 +236,21 @@ route:
   # overwritten on each.
   # The child route trees.
   routes:
-  # This routes performs a regular expression match on alert labels to
-  # catch alerts that are related to a list of services.
+  - match_re:
+      alertname: JenkinsJob.*
+    receiver: ${slack_jenkins_receiver}
+    routes:
+    - match:
+        severity: critical
+      receiver: '${slack_jenkins_receiver}'
+
   - match_re:
       service: .*
-    receiver: ${default_receiver}
-    # The service has a sub-route for critical alerts, any alerts
-    # that do not match, i.e. severity != critical, fall-back to the
-    # parent node and are sent to 'team-X-mails'
+    receiver: ${slack_default_receiver}
     routes:
     - match:
         severity: critical
-      receiver: '${default_receiver}'
+      receiver: '${slack_default_receiver}'
 
 # Inhibition rules allow to mute a set of alerts given that another alert is
 # firing.
@@ -239,17 +261,42 @@ inhibit_rules:
     severity: 'critical'
   target_match:
     severity: 'warning'
-  # Apply inhibition if the alertname is the same.
-  # CAUTION:
-  #   If all label names listed in `equal` are missing
-  #   from both the source and target alerts,
-  #   the inhibition rule will apply!
-  equal: ['alertname', 'cluster', 'service']
+  equal: ['alertname', 'instance']
 
 receivers:
-- name: '${default_receiver}'
+- name: '${slack_jenkins_receiver}'
+  slack_configs:
+  - api_url: 'https://hooks.slack.com/services/${slack_jenkins_api_key}'
+    channel: '#${slack_jenkins_channel}'
+    send_resolved: true
+    icon_url: https://avatars3.githubusercontent.com/u/3380462
+    title: |-
+     [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}
+     {{- if gt (len .CommonLabels) (len .GroupLabels) -}}
+       {{" "}}(
+       {{- with .CommonLabels.Remove .GroupLabels.Names }}
+         {{- range $index, $label := .SortedPairs -}}
+           {{ if $index }}, {{ end }}
+           {{- $label.Name }}="{{ $label.Value -}}"
+         {{- end }}
+       {{- end -}}
+       )
+     {{- end }}
+    text: >-
+     {{ range .Alerts -}}
+     *Alert:* {{ .Annotations.summary }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
+
+     *Description:* {{ .Annotations.description }}
+
+     *Details:*
+       {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
+       {{ end }}
+     {{ end }}
+
+- name: '${slack_default_receiver}'
   slack_configs:
-  - channel: '#${slack_channel}'
+  - api_url: 'https://hooks.slack.com/services/${slack_default_api_key}'
+    channel: '#${slack_default_channel}'
     send_resolved: true
     icon_url: https://avatars3.githubusercontent.com/u/3380462
     title: |-