From: pmikus Date: Fri, 27 Aug 2021 14:02:21 +0000 (+0000) Subject: Infra: Cleanup Nomad configs X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=4bf3efc45c708370b5d8bc30ae0fb64c671a3877 Infra: Cleanup Nomad configs Signed-off-by: pmikus Change-Id: Ia5c9f0902551de1a63144e6f56dfa6db2895b0b2 --- diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml index 76a7def96b..bff2e04863 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml @@ -26,7 +26,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml index bca8cf77b3..ceb27c8705 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml @@ -26,7 +26,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml index 4d871980f0..c839243a26 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml @@ -26,7 +26,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml index 99438bb2b7..4b28db6efd 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml @@ -26,7 +26,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml index 1a6db1419f..30341b5401 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml @@ -26,7 +26,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml index 0a4abedaa1..05328242f0 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml @@ -26,7 +26,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml index b36e195e77..8d35857d8b 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml @@ -26,7 +26,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.39.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.39.yaml index 3a283c14ef..1cc7b68fb1 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.39.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.39.yaml @@ -27,7 +27,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.40.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.40.yaml index a50cdef9d5..7d9db1f9e6 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.40.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.40.yaml @@ -27,7 +27,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.50.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.50.yaml index 51bb5e91aa..3bbae834f5 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.50.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.50.yaml @@ -31,7 +31,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "csit" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.51.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.51.yaml index ba536632d1..020ff03082 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.51.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.51.yaml @@ -31,7 +31,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "csit" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.65.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.65.yaml index 544b6627d6..7ba4fd4816 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.65.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.65.yaml @@ -27,7 +27,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.66.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.66.yaml index e0f5c2a52f..5dee333c60 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.66.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.66.yaml @@ -27,7 +27,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.67.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.67.yaml index 4d99871e81..ab478c840c 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.67.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.67.yaml @@ -27,7 +27,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.68.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.68.yaml index 5ea1d8147a..9451451624 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.68.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.68.yaml @@ -27,7 +27,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.70.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.70.yaml index a71ddba9a9..2bb0bc294b 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.70.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.70.yaml @@ -32,7 +32,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "csitarm" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.71.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.71.yaml index 79733c4160..fc6050b861 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.71.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.71.yaml @@ -32,7 +32,7 @@ nomad_certificates: - src: "{{ file_nomad_client_key_pem }}" dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "client" nomad_node_class: "csitarm" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.14.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.14.yaml index 06e147eef7..2fe0d71547 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.14.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.14.yaml @@ -27,7 +27,7 @@ nomad_certificates: dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" nomad_encrypt: "Y4T+5JGx1C3l2NFBBvkTWQ==" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "both" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.15.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.15.yaml index 90d6cce2a3..5d13377ac1 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.15.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.15.yaml @@ -27,7 +27,7 @@ nomad_certificates: dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" nomad_encrypt: "Y4T+5JGx1C3l2NFBBvkTWQ==" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "both" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.16.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.16.yaml index 8c4fa7e42c..32320170a4 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.16.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.16.yaml @@ -27,7 +27,7 @@ nomad_certificates: dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" nomad_encrypt: "Y4T+5JGx1C3l2NFBBvkTWQ==" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "both" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.17.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.17.yaml index f7a5626dc4..1f10670a2d 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.17.yaml +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.32.8.17.yaml @@ -27,7 +27,7 @@ nomad_certificates: dest: "{{ nomad_key_file }}" nomad_datacenter: "yul1" nomad_encrypt: "Y4T+5JGx1C3l2NFBBvkTWQ==" -nomad_name: "{{ hostname }}-{{ ansible_architecture }}" +nomad_node_name: "{{ hostname }}-{{ ansible_architecture }}" nomad_node_role: "both" nomad_node_class: "builder" nomad_options: diff --git a/fdio.infra.ansible/roles/consul/defaults/main.yaml b/fdio.infra.ansible/roles/consul/defaults/main.yaml index b875c88c74..1bb72f1a15 100644 --- a/fdio.infra.ansible/roles/consul/defaults/main.yaml +++ b/fdio.infra.ansible/roles/consul/defaults/main.yaml @@ -36,7 +36,6 @@ consul_inst_dir: "/opt" consul_lockfile: "/var/lock/subsys/consul" consul_run_dir: "/var/run/consul" consul_ssl_dir: "/etc/consul.d/ssl" -nomad_config_dir: "/etc/nomad.d" # Conf - Service. consul_node_role: "both" @@ -50,25 +49,6 @@ consul_group_state: "present" consul_user: "consul" consul_user_state: "present" -# Conf - nomad.d/consul.hcl -consul_nomad_integration: true -consul_certificates: - - src: "{{ file_consul_ca_pem }}" - dest: "{{ consul_ca_file }}" - - src: "{{ file_consul_server_0_pem }}" - dest: "{{ consul_cert_file }}" - - src: "{{ file_consul_server_0_key_pem }}" - dest: "{{ consul_key_file }}" - -consul_auto_advertise: true -consul_checks_use_advertise: true -consul_server_service_name: "nomad" -consul_client_service_name: "nomad-client" -consul_server_auto_join: false -consul_client_auto_join: true -consul_ACL_token_set: false -consul_token: "consul_token_default" - # Conf - base.hcl consul_bind_addr: "{{ ansible_default_ipv4.address }}" consul_client_addr: "0.0.0.0" @@ -89,6 +69,13 @@ consul_ui_config: consul_recursors: - 1.1.1.1 - 8.8.8.8 +consul_certificates: + - src: "{{ file_consul_ca_pem }}" + dest: "{{ consul_ca_file }}" + - src: "{{ file_consul_server_0_pem }}" + dest: "{{ consul_cert_file }}" + - src: "{{ file_consul_server_0_key_pem }}" + dest: "{{ consul_key_file }}" # Conf - ports.hcl consul_port_dns: 53 diff --git a/fdio.infra.ansible/roles/consul/tasks/main.yaml b/fdio.infra.ansible/roles/consul/tasks/main.yaml index 834d7f1798..f4e08259da 100644 --- a/fdio.infra.ansible/roles/consul/tasks/main.yaml +++ b/fdio.infra.ansible/roles/consul/tasks/main.yaml @@ -96,18 +96,6 @@ tags: - consul-conf -- name: Conf - Nomad integration Consul Configuration - template: - src: consul.hcl.j2 - dest: "{{ nomad_config_dir }}/consul.hcl" - owner: "nomad" - group: "nomad" - mode: 0644 - when: - - consul_nomad_integration | bool - tags: - - consul-conf - - name: Conf - Base Configuration template: src: base.hcl.j2 diff --git a/fdio.infra.ansible/roles/consul/templates/consul.hcl.j2 b/fdio.infra.ansible/roles/consul/templates/consul.hcl.j2 deleted file mode 100644 index c78e5e1ce5..0000000000 --- a/fdio.infra.ansible/roles/consul/templates/consul.hcl.j2 +++ /dev/null @@ -1,12 +0,0 @@ -consul { - auto_advertise = {{ consul_auto_advertise | bool | lower }} - checks_use_advertise = {{ consul_checks_use_advertise | bool | lower }} - client_auto_join = {{ consul_client_auto_join | bool | lower }} - client_service_name = "{{ consul_client_service_name }}" - server_service_name = "{{ consul_server_service_name }}" - server_auto_join = {{ consul_server_auto_join | bool | lower }} -{% if consul_ACL_token_set == True %} - token = "{{ consul_token }}" -{% endif %} - -} \ No newline at end of file diff --git a/fdio.infra.ansible/roles/nomad/defaults/main.yaml b/fdio.infra.ansible/roles/nomad/defaults/main.yaml index 01f020ad81..2ace6b22d5 100644 --- a/fdio.infra.ansible/roles/nomad/defaults/main.yaml +++ b/fdio.infra.ansible/roles/nomad/defaults/main.yaml @@ -1,7 +1,7 @@ --- # file: roles/nomad/defaults/main.yaml -# Inst - Prerequisites. +# Prerequisites packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}" packages_base: - "curl" @@ -16,7 +16,8 @@ packages_by_arch: x86_64: - [] -# Inst - Nomad Map. +# Package +nomad_version: "{{ lookup('env','NOMAD_VERSION') | default('1.0.4', true) }}" nomad_architecture_map: amd64: "amd64" x86_64: "amd64" @@ -25,63 +26,49 @@ nomad_architecture_map: 32-bit: "386" 64-bit: "amd64" nomad_architecture: "{{ nomad_architecture_map[ansible_architecture] }}" -nomad_version: "1.0.4" -nomad_pkg: "nomad_{{ nomad_version }}_linux_{{ nomad_architecture }}.zip" -nomad_zip_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/{{ nomad_pkg }}" +nomad_pkg: "nomad_{{ nomad_version }}_linux_{{nomad_architecture}}.zip" +nomad_zip_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/nomad_{{ nomad_version }}_linux_{{nomad_architecture}}.zip" +nomad_checksum_file_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/nomad_{{ nomad_version}}_SHA256SUMS" +nomad_podman_enable: false +nomad_podman_version: "{{ lookup('env','NOMAD_PODMAN_VERSION') | default('0.1.0', true) }}" +nomad_podman_pkg: "nomad-driver-podman_{{ nomad_podman_version }}_linux_{{nomad_architecture}}.zip" +nomad_podman_url: "https://releases.hashicorp.com/nomad-driver-podman/{{ nomad_podman_version }}" +nomad_podman_zip_url: "{{ nomad_podman_url }}/{{ nomad_podman_pkg }}" +nomad_podman_checksum_file_url: "{{ nomad_podman_url }}/nomad-driver-podman_{{ nomad_podman_version }}_SHA256SUMS" -# Inst - System paths. +# Paths +nomad_inst_dir: "/opt" nomad_bin_dir: "/usr/local/bin" nomad_config_dir: "/etc/nomad.d" nomad_data_dir: "/var/nomad" -nomad_inst_dir: "/opt" +nomad_plugin_dir: "{{ nomad_data_dir }}/plugins" nomad_lockfile: "/var/lock/subsys/nomad" nomad_run_dir: "/var/run/nomad" nomad_ssl_dir: "/etc/nomad.d/ssl" -# Conf - Service. -nomad_node_role: "both" +# Initialization and startup script templates nomad_restart_handler_state: "restarted" -# Conf - User and group. +# System user and group nomad_group: "nomad" nomad_group_state: "present" nomad_user: "nomad" nomad_user_state: "present" -# Conf - base.hcl -nomad_bind_addr: "0.0.0.0" +# Nomad settings nomad_datacenter: "dc1" -nomad_disable_update_check: true -nomad_enable_debug: false -nomad_log_level: "INFO" -nomad_name: "{{ inventory_hostname }}" nomad_region: "global" +nomad_log_level: "INFO" nomad_syslog_enable: true +nomad_iface: "{{ lookup('env','NOMAD_IFACE') | default(ansible_default_ipv4.interface, true) }}" +nomad_node_name: "{{ inventory_hostname }}" +nomad_node_role: "{{ lookup('env','NOMAD_NODE_ROLE') | default('client', true) }}" +nomad_leave_on_terminate: true +nomad_leave_on_interrupt: false +nomad_disable_update_check: true +nomad_enable_debug: false -# Conf - tls.hcl -nomad_ca_file: "{{ nomad_ssl_dir }}/ca.pem" -nomad_cert_file: "{{ nomad_ssl_dir }}/nomad.pem" -nomad_http: false -nomad_key_file: "{{ nomad_ssl_dir }}/nomad-key.pem" -nomad_rpc: false -nomad_verify_https_client: false -nomad_verify_server_hostname: false - -# Conf - client.hcl -nomad_certificates: - - src: "{{ file_nomad_ca_pem }}" - dest: "{{ nomad_ca_file }}" - - src: "{{ file_nomad_client_pem }}" - dest: "{{ nomad_cert_file }}" - - src: "{{ file_nomad_client_key_pem }}" - dest: "{{ nomad_key_file }}" -nomad_node_class: "" -nomad_no_host_uuid: true -nomad_options: {} -nomad_servers: [] -nomad_volumes: [] - -# Conf - server.hcl +# Server settings nomad_bootstrap_expect: 2 nomad_encrypt: "" nomad_retry_join: true @@ -90,25 +77,127 @@ nomad_retry_join: true nomad_node_gc_threshold: "24h" # Specifies the interval between the job garbage collections. Only jobs who have # been terminal for at least job_gc_threshold will be collected. -nomad_job_gc_interval: "1m" +nomad_job_gc_interval: "10m" # Specifies the minimum time a job must be in the terminal state before it is # eligible for garbage collection. -nomad_job_gc_threshold: "1m" +nomad_job_gc_threshold: "4h" # Specifies the minimum time an evaluation must be in the terminal state before # it is eligible for garbage collection. -nomad_eval_gc_threshold: "1m" +nomad_eval_gc_threshold: "1h" # Specifies the minimum time a deployment must be in the terminal state before # it is eligible for garbage collection. -nomad_deployment_gc_threshold: "1m" - -# Conf - telemetry.hcl -nomad_disable_hostname: false -nomad_collection_interval: 60s -nomad_use_node_name: false -nomad_publish_allocation_metrics: true -nomad_publish_node_metrics: true -nomad_telemetry_provider_parameters: - prometheus_metrics: true - -# Conf - custom.hcl -# empty +nomad_deployment_gc_threshold: "1h" +nomad_encrypt_enable: "{{ lookup('env','NOMAD_ENCRYPT_ENABLE') | default('false', true) }}" +nomad_raft_protocol: 2 + +# Client settings +nomad_certificates: + - src: "{{ file_nomad_ca_pem }}" + dest: "{{ nomad_ca_file }}" + - src: "{{ file_nomad_client_pem }}" + dest: "{{ nomad_cert_file }}" + - src: "{{ file_nomad_client_key_pem }}" + dest: "{{ nomad_key_file }}" +nomad_node_class: "" +nomad_no_host_uuid: true +nomad_max_kill_timeout: "30s" +nomad_gc_interval: "1m" +nomad_gc_disk_usage_threshold: 80 +nomad_gc_inode_usage_threshold: 70 +nomad_gc_parallel_destroys: 2 +nomad_reserved: + cpu: "{{ nomad_reserved_cpu | default('0', true) }}" + memory: "{{ nomad_reserved_memory | default('0', true) }}" + disk: "{{ nomad_reserved_disk | default('0', true) }}" + ports: "{{ nomad_reserved_ports | default('22', true) }}" +nomad_volumes: [] +nomad_options: {} +nomad_meta: {} +nomad_chroot_env: false +nomad_plugins: {} + +# Addresses +nomad_bind_address: "{{ hostvars[inventory_hostname]['ansible_'+ nomad_iface ]['ipv4']['address'] }}" +nomad_advertise_address: "{{ hostvars[inventory_hostname]['ansible_' + nomad_iface]['ipv4']['address'] }}" + +# Ports +nomad_ports: + http: "{{ nomad_ports_http | default('4646', true) }}" + rpc: "{{ nomad_ports_rpc | default('4647', true) }}" + serf: "{{ nomad_ports_serf | default('4648', true) }}" + +# Servers +nomad_group_name: "nomad" +nomad_servers: "\ + {% if nomad_use_consul==false %}\ + {% set _nomad_servers = [] %}\ + {% for host in groups[nomad_group_name] %}\ + {% set _nomad_node_role = hostvars[host]['nomad_node_role'] | default('client', true) %}\ + {% if ( _nomad_node_role == 'server' or _nomad_node_role == 'both') %}\ + {% if _nomad_servers.append(host) %}{% endif %}\ + {% endif %}\ + {% endfor %}\ + {{ _nomad_servers }}\ + {% else %}\ + []\ + {% endif %}" +nomad_gather_server_facts: false + +# Consul +nomad_use_consul: true +nomad_consul_address: "localhost:8500" +nomad_consul_token: "" +nomad_consul_servers_service_name: "nomad" +nomad_consul_clients_service_name: "nomad-client" +nomad_consul_tags: {} + +# ACLs +nomad_acl_enabled: "{{ lookup('env', 'NOMAD_ACL_ENABLED') | default('no', true) }}" +nomad_acl_token_ttl: "30s" +nomad_acl_policy_ttl: "30s" +nomad_acl_replication_token: "" + +# Vault +nomad_vault_enabled: "{{ lookup('env', 'NOMAD_VAULT_ENABLED') | default('no', true) }}" +nomad_vault_address: "{{ vault_address | default('0.0.0.0', true) }}" +nomad_vault_allow_unauthenticated: true +nomad_vault_create_from_role: "" +nomad_vault_task_token_ttl: "" +nomad_vault_ca_file: "" +nomad_vault_ca_path: "" +nomad_vault_cert_file: "" +nomad_vault_key_file: "" +nomad_vault_tls_server_name: "" +nomad_vault_tls_skip_verify: false +nomad_vault_token: "" +nomad_vault_namespace: "" + +# Docker +nomad_docker_enable: "{{ lookup('env','NOMAD_DOCKER_ENABLE') | default('false', true) }}" +nomad_docker_dmsetup: true + +# TLS +nomad_tls_enable: true +nomad_ca_file: "{{ nomad_ssl_dir }}/ca.pem" +nomad_cert_file: "{{ nomad_ssl_dir }}/nomad.pem" +nomad_key_file: "{{ nomad_ssl_dir }}/nomad-key.pem" +nomad_http: false +nomad_rpc: false +nomad_rpc_upgrade_mode: false +nomad_verify_server_hostname: false +nomad_verify_https_client: false + +# Conf - autopilot.hcl +nomad_autopilot_cleanup_dead_servers: true +nomad_autopilot_last_contact_threshold: "200ms" +nomad_autopilot_max_trailing_logs: 250 +nomad_autopilot_server_stabilization_time: "10s" + +# Telemetry +nomad_telemetry: true +nomad_telemetry_disable_hostname: false +nomad_telemetry_collection_interval: 60s +nomad_telemetry_use_node_name: false +nomad_telemetry_publish_allocation_metrics: true +nomad_telemetry_publish_node_metrics: true +nomad_telemetry_prometheus_metrics: true diff --git a/fdio.infra.ansible/roles/nomad/tasks/main.yaml b/fdio.infra.ansible/roles/nomad/tasks/main.yaml index 63025a6ead..997b1e9c91 100644 --- a/fdio.infra.ansible/roles/nomad/tasks/main.yaml +++ b/fdio.infra.ansible/roles/nomad/tasks/main.yaml @@ -150,15 +150,13 @@ tags: - nomad-conf -- name: Conf - Custom Configuration +- name: Conf - Consul Configuration template: - src: custom.json.j2 - dest: "{{ nomad_config_dir }}/custom.json" + src: consul.hcl.j2 + dest: "{{ nomad_config_dir }}/consul.hcl" owner: "{{ nomad_user }}" group: "{{ nomad_group }}" mode: 0644 - when: - - nomad_config_custom is defined tags: - nomad-conf diff --git a/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2 index 7badecf9e0..cd7fb54f9c 100644 --- a/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2 +++ b/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2 @@ -1,11 +1,26 @@ -name = "{{ nomad_name }}" +name = "{{ nomad_node_name }}" region = "{{ nomad_region }}" datacenter = "{{ nomad_datacenter }}" -bind_addr = "{{ nomad_bind_addr }}" -data_dir = "{{ nomad_data_dir }}" - -enable_syslog = {{ nomad_syslog_enable | bool | lower }} enable_debug = {{ nomad_enable_debug | bool | lower }} disable_update_check = {{ nomad_disable_update_check | bool | lower }} + +bind_addr = "{{ nomad_bind_address }}" +advertise { + http = "{{ nomad_advertise_address }}:{{ nomad_ports.http }}" + rpc = "{{ nomad_advertise_address }}:{{ nomad_ports.rpc }}" + serf = "{{ nomad_advertise_address }}:{{ nomad_ports.serf }}" +} +ports { + http = {{ nomad_ports['http'] }} + rpc = {{ nomad_ports['rpc'] }} + serf = {{ nomad_ports['serf'] }} +} + +data_dir = "{{ nomad_data_dir }}" + log_level = "{{ nomad_log_level }}" +enable_syslog = {{ nomad_syslog_enable | bool | lower }} + +leave_on_terminate = {{ nomad_leave_on_terminate | bool | lower }} +leave_on_interrupt = {{ nomad_leave_on_interrupt | bool | lower }} diff --git a/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2 index f245697a22..f82f38a4e4 100644 --- a/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2 +++ b/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2 @@ -1,14 +1,44 @@ client { enabled = {{ nomad_node_client | bool | lower }} - no_host_uuid = {{ nomad_no_host_uuid | bool | lower }} + node_class = "{{ nomad_node_class }}" + no_host_uuid = {{ nomad_no_host_uuid | bool | lower }} + +{% if nomad_use_consul == False %} + {% if nomad_servers -%} + servers = [ {% for ip_port in nomad_servers -%} "{{ ip_port }}" {% if not loop.last %},{% endif %}{%- endfor -%} ] + {% endif -%} +{% endif %} + {% if nomad_network_interface is defined -%} + network_interface = "{{ nomad_network_interface }}" + {% endif -%} + {% if nomad_network_speed is defined -%} + network_speed = "{{ nomad_network_speed }}" + {% endif -%} {% if nomad_cpu_total_compute is defined -%} cpu_total_compute = {{ nomad_cpu_total_compute }} {% endif -%} - {% if nomad_servers -%} - servers = [ {% for ip_port in nomad_servers -%} "{{ ip_port }}" {% if not loop.last %},{% endif %}{%- endfor -%} ] + reserved { + cpu = {{ nomad_reserved['cpu'] }} + memory = {{ nomad_reserved['memory'] }} + disk = {{ nomad_reserved['disk'] }} + } + + {% for nomad_host_volume in nomad_volumes -%} + host_volume "{{ nomad_host_volume.name }}" { + path = "{{ nomad_host_volume.path }}" + read_only = {{ nomad_host_volume.read_only | bool | lower }} + } + {% endfor %} + + {% if nomad_chroot_env != False -%} + chroot_env = { + {% for key, value in nomad_chroot_env.items() %} + "{{ key }}" = "{{ value }}" + {% endfor -%} + } {% endif %} {% if nomad_options -%} @@ -19,13 +49,12 @@ client { } {% endif %} - {% if nomad_volumes -%} - {% for volume in nomad_volumes -%} - host_volume "{{ volume.name }}" { - path = "{{ volume.path }}" - read_only = {{ volume.read_only | bool | lower }} - } + {% if nomad_meta -%} + meta = { + {% for key, value in nomad_meta.items() %} + "{{ key }}" = "{{ value }}" {% endfor -%} + } {% endif %} } diff --git a/fdio.infra.ansible/roles/nomad/templates/consul.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/consul.hcl.j2 new file mode 100644 index 0000000000..6d30676ca0 --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/consul.hcl.j2 @@ -0,0 +1,18 @@ +{% if nomad_use_consul | bool == True %} +consul { + # The address to the Consul agent. + address = "{{ nomad_consul_address }}" + token = "{{ nomad_consul_token }}" + # The service name to register the server and client with Consul. + server_service_name = "{{ nomad_consul_servers_service_name }}" + client_service_name = "{{ nomad_consul_clients_service_name }}" + tags = {{ nomad_consul_tags | to_json }} + + # Enables automatically registering the services. + auto_advertise = true + + # Enabling the server and client to bootstrap using Consul. + server_auto_join = true + client_auto_join = true +} +{% endif %} \ No newline at end of file diff --git a/fdio.infra.ansible/roles/nomad/templates/custom.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/custom.hcl.j2 deleted file mode 100644 index 37ff6f3496..0000000000 --- a/fdio.infra.ansible/roles/nomad/templates/custom.hcl.j2 +++ /dev/null @@ -1,5 +0,0 @@ -{% if nomad_config_custom -%} -{{ nomad_config_custom | to_nice_json }} -{% else %} -{} -{% endif %} diff --git a/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2 index 5ccf45c1ac..663ee3a549 100644 --- a/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2 +++ b/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2 @@ -5,7 +5,29 @@ server { bootstrap_expect = {{ nomad_bootstrap_expect }} {%- endif %} - encrypt = "{{ nomad_encrypt }}" + {% if nomad_authoritative_region is defined %} + authoritative_region = "{{ nomad_authoritative_region }}" + {% endif %} + +{% if nomad_use_consul == False %} + {% if nomad_retry_join | bool -%} + retry_join = [ + {%- set comma = joiner(",") -%} + {% for server in nomad_servers -%} + {{ comma() }}"{{ hostvars[server]['nomad_advertise_address'] | ipwrap }}" + {%- endfor -%} ] + retry_max = {{ nomad_retry_max }} + retry_interval = "{{ nomad_retry_interval }}" + {% else -%} + start_join = [ + {%- set comma = joiner(",") -%} + {% for server in nomad_servers -%} + {{ comma() }}"{{ hostvars[server]['nomad_advertise_address'] | ipwrap }}" + {%- endfor -%} ] + {%- endif %} +{% endif %} + + encrypt = "{{ nomad_encrypt | default('') }}" {% if nomad_node_gc_threshold -%} node_gc_threshold = "{{ nomad_node_gc_threshold }}" @@ -27,10 +49,4 @@ server { deployment_gc_threshold = "{{ nomad_deployment_gc_threshold }}" {%- endif %} - {% if nomad_retry_join | bool -%} - server_join { - retry_join = [ {% for ip_port in nomad_retry_servers -%} "{{ ip_port }}" {% if not loop.last %},{% endif %}{%- endfor -%} ] - } - {%- endif %} - } diff --git a/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2 index 7b62f76976..4ad5330d1b 100644 --- a/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2 +++ b/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2 @@ -1,19 +1,10 @@ +{% if nomad_telemetry | bool == True %} telemetry { - # Telemetry provider parameters - {% for key, value in nomad_telemetry_provider_parameters.items() -%} - {% if value|bool -%} - {{ key }} = {{ value | bool | lower }} - {% elif value|string or value == "" -%} - {{ key }} = "{{ value }}" - {% else %} - {{ key }} = {{ value }} - {% endif -%} - {% endfor -%} - - # Common parameters - disable_hostname = {{ nomad_disable_hostname | bool | lower }} - collection_interval = "{{ nomad_collection_interval }}" - use_node_name = {{ nomad_use_node_name | bool | lower }} - publish_allocation_metrics = {{ nomad_publish_allocation_metrics | bool | lower }} - publish_node_metrics = {{ nomad_publish_node_metrics | bool | lower }} + disable_hostname = "{{ nomad_telemetry_disable_hostname | default(false) | bool | lower }}" + collection_interval = "{{ nomad_telemetry_collection_interval | default("1s") }}" + use_node_name = "{{ nomad_telemetry_use_node_name | default(false) | bool | lower }}" + publish_allocation_metrics = "{{ nomad_telemetry_publish_allocation_metrics | default(false) | bool | lower }}" + publish_node_metrics = "{{ nomad_telemetry_publish_node_metrics | default(false) | bool | lower }}" + prometheus_metrics = "{{ nomad_telemetry_prometheus_metrics | default(false) | bool | lower }}" } +{% endif %} diff --git a/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2 index 46dc1fe6b1..ceccdc8be5 100644 --- a/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2 +++ b/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2 @@ -1,14 +1,12 @@ -{% if ( nomad_ca_file ) and - ( nomad_cert_file ) and - ( nomad_key_file ) -%} +{% if nomad_tls_enable | bool %} tls { http = {{ nomad_http | bool | lower }} rpc = {{ nomad_rpc | bool | lower }} - verify_server_hostname = {{ nomad_verify_server_hostname | bool | lower }} - verify_https_client = {{ nomad_verify_https_client | bool | lower }} ca_file = "{{ nomad_ca_file }}" cert_file = "{{ nomad_cert_file }}" key_file = "{{ nomad_key_file }}" + rpc_upgrade_mode = {{ nomad_rpc_upgrade_mode | bool | lower }} + verify_server_hostname = {{ nomad_verify_server_hostname | bool | lower }} + verify_https_client = {{ nomad_verify_https_client | bool | lower }} } {% endif %} diff --git a/fdio.infra.terraform/1n_nmd/nginx/conf/nomad/nginx.hcl b/fdio.infra.terraform/1n_nmd/nginx/conf/nomad/nginx.hcl index 3bbbe5309f..281960dad5 100644 --- a/fdio.infra.terraform/1n_nmd/nginx/conf/nomad/nginx.hcl +++ b/fdio.infra.terraform/1n_nmd/nginx/conf/nomad/nginx.hcl @@ -250,7 +250,7 @@ job "${job_name}" { } } EOH - destination = "custom/logs.conf" + destination = "custom/server_logs.conf" } # The service stanza instructs Nomad to register a service with Consul.