fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf

   1 data "vault_generic_secret" "fdio_logs" {
   2   path = "kv/secret/data/etl/fdio_logs"
   3 }
   4
   5 data "vault_generic_secret" "fdio_docs" {
   6   path = "kv/secret/data/etl/fdio_docs"
   7 }
   8
   9 module "etl-stats" {
  10   providers = {
  11     nomad = nomad.yul1
  12   }
  13   source = "../"
  14
  15   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  16   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  17   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  18   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  19   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  20   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  21   cron                      = "0 30 0 * * * *"
  22   datacenters               = ["yul1"]
  23   job_name                  = "etl-stats"
  24 }
  25
  26 module "etl-trending-hoststack" {
  27   providers = {
  28     nomad = nomad.yul1
  29   }
  30   source = "../"
  31
  32   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  33   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  34   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  35   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  36   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  37   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  38   cron                      = "0 30 0 * * * *"
  39   datacenters               = ["yul1"]
  40   job_name                  = "etl-trending-hoststack"
  41 }
  42
  43 module "etl-trending-mrr" {
  44   providers = {
  45     nomad = nomad.yul1
  46   }
  47   source = "../"
  48
  49   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  50   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  51   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  52   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  53   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  54   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  55   cron                      = "0 30 0 * * * *"
  56   datacenters               = ["yul1"]
  57   job_name                  = "etl-trending-mrr"
  58   memory                    = 60000
  59 }
  60
  61 module "etl-trending-ndrpdr" {
  62   providers = {
  63     nomad = nomad.yul1
  64   }
  65   source = "../"
  66
  67   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  68   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  69   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  70   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  71   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  72   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  73   cron                      = "0 30 0 * * * *"
  74   datacenters               = ["yul1"]
  75   job_name                  = "etl-trending-ndrpdr"
  76   memory                    = 60000
  77 }
  78
  79 module "etl-trending-soak" {
  80   providers = {
  81     nomad = nomad.yul1
  82   }
  83   source = "../"
  84
  85   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  86   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  87   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  88   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  89   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  90   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  91   cron                      = "0 30 0 * * * *"
  92   datacenters               = ["yul1"]
  93   job_name                  = "etl-trending-soak"
  94   memory                    = 60000
  95 }
  96
  97 module "etl-iterative-hoststack-rls2406" {
  98   providers = {
  99     nomad = nomad.yul1
 100   }
 101   source = "../"
 102
 103   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 104   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 105   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 106   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 107   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 108   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 109   cron                      = "0 30 0 * * * *"
 110   datacenters               = ["yul1"]
 111   job_name                  = "etl-iterative-hoststack-rls2406"
 112 }
 113
 114 module "etl-iterative-mrr-rls2406" {
 115   providers = {
 116     nomad = nomad.yul1
 117   }
 118   source = "../"
 119
 120   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 121   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 122   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 123   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 124   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 125   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 126   cron                      = "0 30 0 * * * *"
 127   datacenters               = ["yul1"]
 128   job_name                  = "etl-iterative-mrr-rls2406"
 129 }
 130
 131 module "etl-iterative-ndrpdr-rls2406" {
 132   providers = {
 133     nomad = nomad.yul1
 134   }
 135   source = "../"
 136
 137   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 138   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 139   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 140   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 141   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 142   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 143   cron                      = "0 30 0 * * * *"
 144   datacenters               = ["yul1"]
 145   job_name                  = "etl-iterative-ndrpdr-rls2406"
 146 }
 147
 148 module "etl-iterative-reconf-rls2406" {
 149   providers = {
 150     nomad = nomad.yul1
 151   }
 152   source = "../"
 153
 154   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 155   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 156   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 157   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 158   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 159   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 160   cron                      = "0 30 0 * * * *"
 161   datacenters               = ["yul1"]
 162   job_name                  = "etl-iterative-reconf-rls2406"
 163 }
 164
 165 module "etl-iterative-soak-rls2406" {
 166   providers = {
 167     nomad = nomad.yul1
 168   }
 169   source = "../"
 170
 171   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 172   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 173   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 174   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 175   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 176   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 177   cron                      = "0 30 0 * * * *"
 178   datacenters               = ["yul1"]
 179   job_name                  = "etl-iterative-soak-rls2406"
 180 }
 181
 182 module "etl-coverage-device-rls2406" {
 183   providers = {
 184     nomad = nomad.yul1
 185   }
 186   source = "../"
 187
 188   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 189   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 190   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 191   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 192   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 193   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 194   cron                      = "0 30 0 * * * *"
 195   datacenters               = ["yul1"]
 196   job_name                  = "etl-coverage-device-rls2406"
 197 }
 198
 199 module "etl-coverage-hoststack-rls2406" {
 200   providers = {
 201     nomad = nomad.yul1
 202   }
 203   source = "../"
 204
 205   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 206   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 207   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 208   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 209   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 210   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 211   cron                      = "0 30 0 * * * *"
 212   datacenters               = ["yul1"]
 213   job_name                  = "etl-coverage-hoststack-rls2406"
 214 }
 215
 216 module "etl-coverage-mrr-rls2406" {
 217   providers = {
 218     nomad = nomad.yul1
 219   }
 220   source = "../"
 221
 222   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 223   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 224   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 225   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 226   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 227   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 228   cron                      = "0 30 0 * * * *"
 229   datacenters               = ["yul1"]
 230   job_name                  = "etl-coverage-mrr-rls2406"
 231 }
 232
 233 module "etl-coverage-ndrpdr-rls2406" {
 234   providers = {
 235     nomad = nomad.yul1
 236   }
 237   source = "../"
 238
 239   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 240   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 241   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 242   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 243   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 244   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 245   cron                      = "0 30 0 * * * *"
 246   datacenters               = ["yul1"]
 247   job_name                  = "etl-coverage-ndrpdr-rls2406"
 248 }
 249
 250 module "etl-coverage-reconf-rls2406" {
 251   providers = {
 252     nomad = nomad.yul1
 253   }
 254   source = "../"
 255
 256   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 257   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 258   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 259   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 260   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 261   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 262   cron                      = "0 30 0 * * * *"
 263   datacenters               = ["yul1"]
 264   job_name                  = "etl-coverage-reconf-rls2406"
 265 }
 266
 267 module "etl-coverage-soak-rls2406" {
 268   providers = {
 269     nomad = nomad.yul1
 270   }
 271   source = "../"
 272
 273   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 274   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 275   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 276   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 277   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 278   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 279   cron                      = "0 30 0 * * * *"
 280   datacenters               = ["yul1"]
 281   job_name                  = "etl-coverage-soak-rls2406"
 282 }
 283