feat(terraform): Update ETL

Signed-off-by: Peter Mikus <peter.mikus@protonmail.ch>
Change-Id: I840cbc5a95e9af41af1811fc28db71e6b0d19d1e

diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.21.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.21.yaml
index 581dc3c..a866316 100644 (file)
--- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.21.yaml
+++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.21.yaml
@@ -39,6 +39,7 @@ nomad_options:
   docker.volumes.enabled: true
   driver.whitelist: "docker,raw_exec,exec"
   fingerprint.network.disallow_link_local: true
+  artifact.disable_filesystem_isolation: true
 nomad_service_mgr: "systemd"
 nomad_consul_use_ssl: false
 nomad_use_tls: false

diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml
index 5de1de4..196f8b6 100644 (file)
--- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml
+++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml
@@ -39,6 +39,7 @@ nomad_options:
   docker.volumes.enabled: true
   driver.whitelist: "docker,raw_exec,exec"
   fingerprint.network.disallow_link_local: true
+  artifact.disable_filesystem_isolation: true
 nomad_service_mgr: "systemd"
 nomad_consul_use_ssl: false
 nomad_use_tls: false

diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.23.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.23.yaml
index dd55ffa..70b75f7 100644 (file)
--- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.23.yaml
+++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.23.yaml
@@ -39,6 +39,7 @@ nomad_options:
   docker.volumes.enabled: true
   driver.whitelist: "docker,raw_exec,exec"
   fingerprint.network.disallow_link_local: true
+  artifact.disable_filesystem_isolation: true
 nomad_service_mgr: "systemd"
 nomad_consul_use_ssl: false
 nomad_use_tls: false

diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml
index 579f6d6..6bb2c1f 100644 (file)
--- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml
+++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml
@@ -39,6 +39,7 @@ nomad_options:
   docker.volumes.enabled: true
   driver.whitelist: "docker,raw_exec,exec"
   fingerprint.network.disallow_link_local: true
+  artifact.disable_filesystem_isolation: true
 nomad_service_mgr: "systemd"
 nomad_consul_use_ssl: false
 nomad_use_tls: false

diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml
index 528383b..c016c56 100644 (file)
--- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml
+++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml
@@ -39,6 +39,7 @@ nomad_options:
   docker.volumes.enabled: true
   driver.whitelist: "docker,raw_exec,exec"
   fingerprint.network.disallow_link_local: true
+  artifact.disable_filesystem_isolation: true
 nomad_service_mgr: "systemd"
 nomad_consul_use_ssl: false
 nomad_use_tls: false

diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml
index 018174c..99333dd 100644 (file)
--- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml
+++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml
@@ -39,6 +39,7 @@ nomad_options:
   docker.volumes.enabled: true
   driver.whitelist: "docker,raw_exec,exec"
   fingerprint.network.disallow_link_local: true
+  artifact.disable_filesystem_isolation: true
 nomad_service_mgr: "systemd"
 nomad_consul_use_ssl: false
 nomad_use_tls: false

diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.27.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.27.yaml
index a73d79f..4383dd2 100644 (file)
--- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.27.yaml
+++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.27.yaml
@@ -38,6 +38,7 @@ nomad_options:
   docker.volumes.enabled: true
   driver.whitelist: "docker,raw_exec,exec"
   fingerprint.network.disallow_link_local: true
+  artifact.disable_filesystem_isolation: true
 nomad_service_mgr: "systemd"
 nomad_consul_use_ssl: false
 nomad_use_tls: false

diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml
index 5880d07..329b0dc 100644 (file)
--- a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml
+++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml
@@ -38,6 +38,7 @@ nomad_options:
   docker.volumes.enabled: true
   driver.whitelist: "docker,raw_exec,exec"
   fingerprint.network.disallow_link_local: true
+  artifact.disable_filesystem_isolation: true
 nomad_service_mgr: "systemd"
 nomad_consul_use_ssl: false
 nomad_use_tls: false

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl
index 86ca584..6634018 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/stats.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/stats_sra.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -33,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "10g",
           "stats.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl
index 24aa409..b8f7cc2 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_hoststack.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_hoststack.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -33,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "30g",
           "trending_hoststack.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
@@ -50,4 +54,4 @@ job "${job_name}" {
       }
     }
   }
-}
\ No newline at end of file
+}

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl
index 47d6149..d3b301d 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_mrr.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_mrr.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -33,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "30g",
           "trending_mrr.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
@@ -50,4 +54,4 @@ job "${job_name}" {
       }
     }
   }
-}
\ No newline at end of file
+}

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl
index 8cd40f5..53a0aa2 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_ndrpdr.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_ndrpdr.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "trending_ndrpdr.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
@@ -52,4 +56,4 @@ job "${job_name}" {
       }
     }
   }
-}
\ No newline at end of file
+}

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl
index 6d77a89..b18ce52 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_soak.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_soak.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "trending_soak.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf
index 28234bf..026ab16 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf
@@ -1,23 +1,33 @@
-data "vault_generic_secret" "fdio_logs" {
-  path = "kv/secret/data/etl/fdio_logs"
+data "vault_kv_secret_v2" "fdio_logs" {
+  mount = "kv"
+  name  = "etl/fdio_logs"
 }
 
-data "vault_generic_secret" "fdio_docs" {
-  path = "kv/secret/data/etl/fdio_docs"
+data "vault_kv_secret_v2" "fdio_docs" {
+  mount = "kv"
+  name  = "etl/fdio_docs"
 }
 
+#data "vault_kv_secret_v2" "fdio_logs" {
+#  path = "kv/data/etl/fdio_logs"
+#}
+#
+#data "vault_kv_secret_v2" "fdio_docs" {
+#  path = "kv/data/etl/fdio_docs"
+#}
+
 module "etl-stats" {
   providers = {
     nomad = nomad.yul1
   }
   source = "../"
 
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
+  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data.access_key
+  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data.secret_key
+  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data.region
+  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data.access_key
+  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key
+  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data.region
   cron                      = "0 30 0 * * * *"
   datacenters               = ["yul1"]
   job_name                  = "etl-stats"
@@ -29,12 +39,12 @@ module "etl-trending-hoststack" {
   }
   source = "../"
 
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
+  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data.access_key
+  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data.secret_key
+  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data.region
+  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data.access_key
+  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key
+  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data.region
   cron                      = "0 30 0 * * * *"
   datacenters               = ["yul1"]
   job_name                  = "etl-trending-hoststack"
@@ -46,12 +56,12 @@ module "etl-trending-mrr" {
   }
   source = "../"
 
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
+  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data.access_key
+  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data.secret_key
+  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data.region
+  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data.access_key
+  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key
+  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data.region
   cron                      = "0 30 0 * * * *"
   datacenters               = ["yul1"]
   job_name                  = "etl-trending-mrr"
@@ -64,12 +74,12 @@ module "etl-trending-ndrpdr" {
   }
   source = "../"
 
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
+  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data.access_key
+  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data.secret_key
+  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data.region
+  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data.access_key
+  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key
+  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data.region
   cron                      = "0 30 0 * * * *"
   datacenters               = ["yul1"]
   job_name                  = "etl-trending-ndrpdr"
@@ -82,202 +92,201 @@ module "etl-trending-soak" {
   }
   source = "../"
 
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
+  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data.access_key
+  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data.secret_key
+  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data.region
+  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data.access_key
+  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key
+  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data.region
   cron                      = "0 30 0 * * * *"
   datacenters               = ["yul1"]
   job_name                  = "etl-trending-soak"
   memory                    = 60000
 }
 
-module "etl-iterative-hoststack-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-hoststack-rls2406"
-}
-
-module "etl-iterative-mrr-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-mrr-rls2406"
-}
-
-module "etl-iterative-ndrpdr-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-ndrpdr-rls2406"
-}
-
-module "etl-iterative-reconf-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-reconf-rls2406"
-}
-
-module "etl-iterative-soak-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-soak-rls2406"
-}
-
-module "etl-coverage-device-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-device-rls2406"
-}
-
-module "etl-coverage-hoststack-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-hoststack-rls2406"
-}
-
-module "etl-coverage-mrr-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-mrr-rls2406"
-}
-
-module "etl-coverage-ndrpdr-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-ndrpdr-rls2406"
-}
-
-module "etl-coverage-reconf-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-reconf-rls2406"
-}
-
-module "etl-coverage-soak-rls2406" {
-  providers = {
-    nomad = nomad.yul1
-  }
-  source = "../"
-
-  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
-  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
-  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
-  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
-  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
-  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
-  cron                      = "0 30 0 * * * *"
-  datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-soak-rls2406"
-}
-
+#module "etl-iterative-hoststack-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-iterative-hoststack-rls2406"
+#}
+#
+#module "etl-iterative-mrr-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-iterative-mrr-rls2406"
+#}
+#
+#module "etl-iterative-ndrpdr-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-iterative-ndrpdr-rls2406"
+#}
+#
+#module "etl-iterative-reconf-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-iterative-reconf-rls2406"
+#}
+#
+#module "etl-iterative-soak-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-iterative-soak-rls2406"
+#}
+#
+#module "etl-coverage-device-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-coverage-device-rls2406"
+#}
+#
+#module "etl-coverage-hoststack-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-coverage-hoststack-rls2406"
+#}
+#
+#module "etl-coverage-mrr-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-coverage-mrr-rls2406"
+#}
+#
+#module "etl-coverage-ndrpdr-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-coverage-ndrpdr-rls2406"
+#}
+#
+#module "etl-coverage-reconf-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-coverage-reconf-rls2406"
+#}
+#
+#module "etl-coverage-soak-rls2406" {
+#  providers = {
+#    nomad = nomad.yul1
+#  }
+#  source = "../"
+#
+#  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data["access_key"]
+#  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data["secret_key"]
+#  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data["region"]
+#  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data["access_key"]
+#  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"]
+#  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data["region"]
+#  cron                      = "0 30 0 * * * *"
+#  datacenters               = ["yul1"]
+#  job_name                  = "etl-coverage-soak-rls2406"
+#}

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf
index db24bdf..60298d4 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf
@@ -19,13 +19,13 @@ variable "nomad_provider_ca_file" {
 variable "nomad_provider_cert_file" {
   description = "A local file path to a PEM-encoded certificate."
   type        = string
-  default     = "/etc/nomad.d/ssl/nomad-cli.pem"
+  default     = "/etc/nomad.d/ssl/nomad.pem"
 }
 
 variable "nomad_provider_key_file" {
   description = "A local file path to a PEM-encoded private key."
   type        = string
-  default     = "/etc/nomad.d/ssl/nomad-cli-key.pem"
+  default     = "/etc/nomad.d/ssl/nomad-key.pem"
 }
 
 variable "vault_provider_address" {

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf
index 0c05e76..ffe25bb 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf
@@ -7,10 +7,10 @@ terraform {
   required_providers {
     nomad = {
       source  = "hashicorp/nomad"
-      version = ">= 1.4.20"
+      version = ">= 2.3.0"
     }
     vault = {
-      version = ">= 3.12.0"
+      version = ">= 4.3.0"
     }
   }
   required_version = ">= 1.5.4"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
index f6d318e..86d1b45 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
@@ -106,9 +106,9 @@ variable "vault_secret" {
   })
   description = "Set of properties to be able to fetch secret from vault."
   default = {
-    use_vault_provider        = false
+    use_vault_provider        = true
     vault_kv_policy_name      = "kv"
-    vault_kv_path             = "secret/data/etl"
+    vault_kv_path             = "data/etl"
     vault_kv_field_access_key = "access_key"
     vault_kv_field_secret_key = "secret_key"
   }