fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf

   1 data "vault_generic_secret" "fdio_logs" {
   2   path = "kv/secret/data/etl/fdio_logs"
   3 }
   4
   5 data "vault_generic_secret" "fdio_docs" {
   6   path = "kv/secret/data/etl/fdio_docs"
   7 }
   8
   9 module "etl-stats" {
  10   providers = {
  11     nomad = nomad.yul1
  12   }
  13   source = "../"
  14
  15   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  16   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  17   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  18   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  19   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  20   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  21   cron                      = "0 30 0 * * * *"
  22   datacenters               = ["yul1"]
  23   job_name                  = "etl-stats"
  24 }
  25
  26 module "etl-trending-hoststack" {
  27   providers = {
  28     nomad = nomad.yul1
  29   }
  30   source = "../"
  31
  32   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  33   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  34   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  35   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  36   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  37   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  38   cron                      = "0 30 0 * * * *"
  39   datacenters               = ["yul1"]
  40   job_name                  = "etl-trending-hoststack"
  41 }
  42
  43 module "etl-trending-mrr" {
  44   providers = {
  45     nomad = nomad.yul1
  46   }
  47   source = "../"
  48
  49   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  50   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  51   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  52   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  53   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  54   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  55   cron                      = "0 30 0 * * * *"
  56   datacenters               = ["yul1"]
  57   job_name                  = "etl-trending-mrr"
  58   memory                    = 60000
  59 }
  60
  61 module "etl-trending-ndrpdr" {
  62   providers = {
  63     nomad = nomad.yul1
  64   }
  65   source = "../"
  66
  67   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  68   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  69   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  70   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  71   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  72   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  73   cron                      = "0 30 0 * * * *"
  74   datacenters               = ["yul1"]
  75   job_name                  = "etl-trending-ndrpdr"
  76   memory                    = 60000
  77 }
  78
  79 module "etl-iterative-hoststack-rls2402" {
  80   providers = {
  81     nomad = nomad.yul1
  82   }
  83   source = "../"
  84
  85   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  86   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  87   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  88   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  89   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  90   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  91   cron                      = "0 30 0 * * * *"
  92   datacenters               = ["yul1"]
  93   job_name                  = "etl-iterative-hoststack-rls2402"
  94 }
  95
  96 module "etl-iterative-mrr-rls2402" {
  97   providers = {
  98     nomad = nomad.yul1
  99   }
 100   source = "../"
 101
 102   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 103   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 104   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 105   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 106   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 107   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 108   cron                      = "0 30 0 * * * *"
 109   datacenters               = ["yul1"]
 110   job_name                  = "etl-iterative-mrr-rls2402"
 111 }
 112
 113 module "etl-iterative-ndrpdr-rls2402" {
 114   providers = {
 115     nomad = nomad.yul1
 116   }
 117   source = "../"
 118
 119   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 120   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 121   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 122   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 123   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 124   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 125   cron                      = "0 30 0 * * * *"
 126   datacenters               = ["yul1"]
 127   job_name                  = "etl-iterative-ndrpdr-rls2402"
 128 }
 129
 130 module "etl-iterative-reconf-rls2402" {
 131   providers = {
 132     nomad = nomad.yul1
 133   }
 134   source = "../"
 135
 136   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 137   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 138   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 139   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 140   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 141   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 142   cron                      = "0 30 0 * * * *"
 143   datacenters               = ["yul1"]
 144   job_name                  = "etl-iterative-reconf-rls2402"
 145 }
 146
 147 module "etl-iterative-soak-rls2402" {
 148   providers = {
 149     nomad = nomad.yul1
 150   }
 151   source = "../"
 152
 153   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 154   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 155   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 156   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 157   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 158   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 159   cron                      = "0 30 0 * * * *"
 160   datacenters               = ["yul1"]
 161   job_name                  = "etl-iterative-soak-rls2402"
 162 }
 163
 164 module "etl-coverage-device-rls2402" {
 165   providers = {
 166     nomad = nomad.yul1
 167   }
 168   source = "../"
 169
 170   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 171   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 172   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 173   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 174   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 175   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 176   cron                      = "0 30 0 * * * *"
 177   datacenters               = ["yul1"]
 178   job_name                  = "etl-coverage-device-rls2402"
 179 }
 180
 181 module "etl-coverage-hoststack-rls2402" {
 182   providers = {
 183     nomad = nomad.yul1
 184   }
 185   source = "../"
 186
 187   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 188   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 189   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 190   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 191   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 192   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 193   cron                      = "0 30 0 * * * *"
 194   datacenters               = ["yul1"]
 195   job_name                  = "etl-coverage-hoststack-rls2402"
 196 }
 197
 198 module "etl-coverage-mrr-rls2402" {
 199   providers = {
 200     nomad = nomad.yul1
 201   }
 202   source = "../"
 203
 204   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 205   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 206   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 207   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 208   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 209   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 210   cron                      = "0 30 0 * * * *"
 211   datacenters               = ["yul1"]
 212   job_name                  = "etl-coverage-mrr-rls2402"
 213 }
 214
 215 module "etl-coverage-ndrpdr-rls2402" {
 216   providers = {
 217     nomad = nomad.yul1
 218   }
 219   source = "../"
 220
 221   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 222   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 223   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 224   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 225   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 226   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 227   cron                      = "0 30 0 * * * *"
 228   datacenters               = ["yul1"]
 229   job_name                  = "etl-coverage-ndrpdr-rls2402"
 230 }
 231
 232 module "etl-coverage-reconf-rls2402" {
 233   providers = {
 234     nomad = nomad.yul1
 235   }
 236   source = "../"
 237
 238   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 239   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 240   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 241   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 242   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 243   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 244   cron                      = "0 30 0 * * * *"
 245   datacenters               = ["yul1"]
 246   job_name                  = "etl-coverage-reconf-rls2402"
 247 }
 248
 249 module "etl-coverage-soak-rls2402" {
 250   providers = {
 251     nomad = nomad.yul1
 252   }
 253   source = "../"
 254
 255   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 256   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 257   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 258   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 259   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 260   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 261   cron                      = "0 30 0 * * * *"
 262   datacenters               = ["yul1"]
 263   job_name                  = "etl-coverage-soak-rls2402"
 264 }