fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf

   1 data "vault_generic_secret" "fdio_logs" {
   2   path = "kv/secret/data/etl/fdio_logs"
   3 }
   4
   5 data "vault_generic_secret" "fdio_docs" {
   6   path = "kv/secret/data/etl/fdio_docs"
   7 }
   8
   9 module "etl-stats" {
  10   providers = {
  11     nomad = nomad.yul1
  12   }
  13   source = "../"
  14
  15   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  16   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  17   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  18   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  19   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  20   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  21   cron                      = "@daily"
  22   datacenters               = ["yul1"]
  23   job_name                  = "etl-stats"
  24 }
  25
  26 module "etl-trending-hoststack" {
  27   providers = {
  28     nomad = nomad.yul1
  29   }
  30   source = "../"
  31
  32   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  33   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  34   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  35   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  36   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  37   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  38   cron                      = "@daily"
  39   datacenters               = ["yul1"]
  40   job_name                  = "etl-trending-hoststack"
  41 }
  42
  43 module "etl-trending-mrr" {
  44   providers = {
  45     nomad = nomad.yul1
  46   }
  47   source = "../"
  48
  49   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  50   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  51   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  52   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  53   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  54   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  55   cron                      = "@daily"
  56   datacenters               = ["yul1"]
  57   job_name                  = "etl-trending-mrr"
  58 }
  59
  60 module "etl-trending-ndrpdr" {
  61   providers = {
  62     nomad = nomad.yul1
  63   }
  64   source = "../"
  65
  66   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  67   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  68   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  69   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  70   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  71   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  72   cron                      = "@daily"
  73   datacenters               = ["yul1"]
  74   job_name                  = "etl-trending-ndrpdr"
  75 }
  76
  77 module "etl-iterative-hoststack-rls2306" {
  78   providers = {
  79     nomad = nomad.yul1
  80   }
  81   source = "../"
  82
  83   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  84   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  85   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  86   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  87   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  88   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  89   cron                      = "@daily"
  90   datacenters               = ["yul1"]
  91   job_name                  = "etl-iterative-hoststack-rls2306"
  92 }
  93
  94 module "etl-iterative-mrr-rls2306" {
  95   providers = {
  96     nomad = nomad.yul1
  97   }
  98   source = "../"
  99
 100   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 101   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 102   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 103   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 104   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 105   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 106   cron                      = "@daily"
 107   datacenters               = ["yul1"]
 108   job_name                  = "etl-iterative-mrr-rls2306"
 109 }
 110
 111 module "etl-iterative-ndrpdr-rls2306" {
 112   providers = {
 113     nomad = nomad.yul1
 114   }
 115   source = "../"
 116
 117   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 118   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 119   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 120   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 121   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 122   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 123   cron                      = "@daily"
 124   datacenters               = ["yul1"]
 125   job_name                  = "etl-iterative-ndrpdr-rls2306"
 126 }
 127
 128 module "etl-iterative-reconf-rls2306" {
 129   providers = {
 130     nomad = nomad.yul1
 131   }
 132   source = "../"
 133
 134   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 135   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 136   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 137   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 138   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 139   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 140   cron                      = "@daily"
 141   datacenters               = ["yul1"]
 142   job_name                  = "etl-iterative-reconf-rls2306"
 143 }
 144
 145 module "etl-iterative-soak-rls2306" {
 146   providers = {
 147     nomad = nomad.yul1
 148   }
 149   source = "../"
 150
 151   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 152   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 153   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 154   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 155   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 156   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 157   cron                      = "@daily"
 158   datacenters               = ["yul1"]
 159   job_name                  = "etl-iterative-soak-rls2306"
 160 }
 161
 162 module "etl-coverage-device-rls2306" {
 163   providers = {
 164     nomad = nomad.yul1
 165   }
 166   source = "../"
 167
 168   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 169   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 170   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 171   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 172   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 173   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 174   cron                      = "@daily"
 175   datacenters               = ["yul1"]
 176   job_name                  = "etl-coverage-device-rls2306"
 177 }
 178
 179 module "etl-coverage-hoststack-rls2306" {
 180   providers = {
 181     nomad = nomad.yul1
 182   }
 183   source = "../"
 184
 185   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 186   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 187   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 188   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 189   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 190   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 191   cron                      = "@daily"
 192   datacenters               = ["yul1"]
 193   job_name                  = "etl-coverage-hoststack-rls2306"
 194 }
 195
 196 module "etl-coverage-mrr-rls2306" {
 197   providers = {
 198     nomad = nomad.yul1
 199   }
 200   source = "../"
 201
 202   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 203   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 204   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 205   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 206   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 207   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 208   cron                      = "@daily"
 209   datacenters               = ["yul1"]
 210   job_name                  = "etl-coverage-mrr-rls2306"
 211 }
 212
 213 module "etl-coverage-ndrpdr-rls2306" {
 214   providers = {
 215     nomad = nomad.yul1
 216   }
 217   source = "../"
 218
 219   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 220   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 221   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 222   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 223   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 224   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 225   cron                      = "@daily"
 226   datacenters               = ["yul1"]
 227   job_name                  = "etl-coverage-ndrpdr-rls2306"
 228 }
 229
 230 module "etl-coverage-reconf-rls2306" {
 231   providers = {
 232     nomad = nomad.yul1
 233   }
 234   source = "../"
 235
 236   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 237   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 238   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 239   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 240   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 241   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 242   cron                      = "@daily"
 243   datacenters               = ["yul1"]
 244   job_name                  = "etl-coverage-reconf-rls2306"
 245 }
 246
 247 module "etl-coverage-soak-rls2306" {
 248   providers = {
 249     nomad = nomad.yul1
 250   }
 251   source = "../"
 252
 253   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 254   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 255   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 256   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 257   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 258   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 259   cron                      = "@daily"
 260   datacenters               = ["yul1"]
 261   job_name                  = "etl-coverage-soak-rls2306"
 262 }