fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf

   1 data "vault_generic_secret" "fdio_logs" {
   2   path = "kv/secret/data/etl/fdio_logs"
   3 }
   4
   5 data "vault_generic_secret" "fdio_docs" {
   6   path = "kv/secret/data/etl/fdio_docs"
   7 }
   8
   9 module "etl-stats" {
  10   providers = {
  11     nomad = nomad.yul1
  12   }
  13   source = "../"
  14
  15   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  16   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  17   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  18   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  19   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  20   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  21   cron                      = "0 30 0 * * * *"
  22   datacenters               = ["yul1"]
  23   job_name                  = "etl-stats"
  24 }
  25
  26 module "etl-trending-hoststack" {
  27   providers = {
  28     nomad = nomad.yul1
  29   }
  30   source = "../"
  31
  32   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  33   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  34   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  35   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  36   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  37   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  38   cron                      = "0 30 0 * * * *"
  39   datacenters               = ["yul1"]
  40   job_name                  = "etl-trending-hoststack"
  41 }
  42
  43 module "etl-trending-mrr" {
  44   providers = {
  45     nomad = nomad.yul1
  46   }
  47   source = "../"
  48
  49   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  50   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  51   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  52   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  53   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  54   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  55   cron                      = "0 30 0 * * * *"
  56   datacenters               = ["yul1"]
  57   job_name                  = "etl-trending-mrr"
  58   memory                    = 60000
  59 }
  60
  61 module "etl-trending-ndrpdr" {
  62   providers = {
  63     nomad = nomad.yul1
  64   }
  65   source = "../"
  66
  67   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  68   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  69   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  70   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  71   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  72   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  73   cron                      = "0 30 0 * * * *"
  74   datacenters               = ["yul1"]
  75   job_name                  = "etl-trending-ndrpdr"
  76   memory                    = 60000
  77 }
  78
  79 module "etl-trending-soak" {
  80   providers = {
  81     nomad = nomad.yul1
  82   }
  83   source = "../"
  84
  85   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  86   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  87   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  88   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  89   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  90   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  91   cron                      = "0 30 0 * * * *"
  92   datacenters               = ["yul1"]
  93   job_name                  = "etl-trending-soak"
  94   memory                    = 60000
  95 }
  96
  97 #module "etl-iterative-hoststack-rls2402" {
  98 #  providers = {
  99 #    nomad = nomad.yul1
 100 #  }
 101 #  source = "../"
 102 #
 103 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 104 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 105 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 106 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 107 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 108 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 109 #  cron                      = "0 30 0 * * * *"
 110 #  datacenters               = ["yul1"]
 111 #  job_name                  = "etl-iterative-hoststack-rls2402"
 112 #}
 113 #
 114 #module "etl-iterative-mrr-rls2402" {
 115 #  providers = {
 116 #    nomad = nomad.yul1
 117 #  }
 118 #  source = "../"
 119 #
 120 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 121 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 122 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 123 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 124 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 125 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 126 #  cron                      = "0 30 0 * * * *"
 127 #  datacenters               = ["yul1"]
 128 #  job_name                  = "etl-iterative-mrr-rls2402"
 129 #}
 130 #
 131 #module "etl-iterative-ndrpdr-rls2402" {
 132 #  providers = {
 133 #    nomad = nomad.yul1
 134 #  }
 135 #  source = "../"
 136 #
 137 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 138 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 139 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 140 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 141 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 142 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 143 #  cron                      = "0 30 0 * * * *"
 144 #  datacenters               = ["yul1"]
 145 #  job_name                  = "etl-iterative-ndrpdr-rls2402"
 146 #}
 147 #
 148 #module "etl-iterative-reconf-rls2402" {
 149 #  providers = {
 150 #    nomad = nomad.yul1
 151 #  }
 152 #  source = "../"
 153 #
 154 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 155 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 156 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 157 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 158 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 159 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 160 #  cron                      = "0 30 0 * * * *"
 161 #  datacenters               = ["yul1"]
 162 #  job_name                  = "etl-iterative-reconf-rls2402"
 163 #}
 164 #
 165 #module "etl-iterative-soak-rls2402" {
 166 #  providers = {
 167 #    nomad = nomad.yul1
 168 #  }
 169 #  source = "../"
 170 #
 171 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 172 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 173 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 174 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 175 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 176 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 177 #  cron                      = "0 30 0 * * * *"
 178 #  datacenters               = ["yul1"]
 179 #  job_name                  = "etl-iterative-soak-rls2402"
 180 #}
 181 #
 182 #module "etl-coverage-device-rls2402" {
 183 #  providers = {
 184 #    nomad = nomad.yul1
 185 #  }
 186 #  source = "../"
 187 #
 188 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 189 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 190 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 191 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 192 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 193 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 194 #  cron                      = "0 30 0 * * * *"
 195 #  datacenters               = ["yul1"]
 196 #  job_name                  = "etl-coverage-device-rls2402"
 197 #}
 198 #
 199 #module "etl-coverage-hoststack-rls2402" {
 200 #  providers = {
 201 #    nomad = nomad.yul1
 202 #  }
 203 #  source = "../"
 204 #
 205 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 206 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 207 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 208 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 209 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 210 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 211 #  cron                      = "0 30 0 * * * *"
 212 #  datacenters               = ["yul1"]
 213 #  job_name                  = "etl-coverage-hoststack-rls2402"
 214 #}
 215 #
 216 #module "etl-coverage-mrr-rls2402" {
 217 #  providers = {
 218 #    nomad = nomad.yul1
 219 #  }
 220 #  source = "../"
 221 #
 222 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 223 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 224 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 225 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 226 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 227 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 228 #  cron                      = "0 30 0 * * * *"
 229 #  datacenters               = ["yul1"]
 230 #  job_name                  = "etl-coverage-mrr-rls2402"
 231 #}
 232 #
 233 #module "etl-coverage-ndrpdr-rls2402" {
 234 #  providers = {
 235 #    nomad = nomad.yul1
 236 #  }
 237 #  source = "../"
 238 #
 239 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 240 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 241 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 242 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 243 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 244 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 245 #  cron                      = "0 30 0 * * * *"
 246 #  datacenters               = ["yul1"]
 247 #  job_name                  = "etl-coverage-ndrpdr-rls2402"
 248 #}
 249 #
 250 #module "etl-coverage-reconf-rls2402" {
 251 #  providers = {
 252 #    nomad = nomad.yul1
 253 #  }
 254 #  source = "../"
 255 #
 256 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 257 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 258 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 259 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 260 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 261 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 262 #  cron                      = "0 30 0 * * * *"
 263 #  datacenters               = ["yul1"]
 264 #  job_name                  = "etl-coverage-reconf-rls2402"
 265 #}
 266 #
 267 #module "etl-coverage-soak-rls2402" {
 268 #  providers = {
 269 #    nomad = nomad.yul1
 270 #  }
 271 #  source = "../"
 272 #
 273 #  aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 274 #  aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 275 #  aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 276 #  out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 277 #  out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 278 #  out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 279 #  cron                      = "0 30 0 * * * *"
 280 #  datacenters               = ["yul1"]
 281 #  job_name                  = "etl-coverage-soak-rls2402"
 282 #}
 283 #