fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf

   1 data "vault_generic_secret" "fdio_logs" {
   2   path = "kv/secret/data/etl/fdio_logs"
   3 }
   4
   5 data "vault_generic_secret" "fdio_docs" {
   6   path = "kv/secret/data/etl/fdio_docs"
   7 }
   8
   9 module "etl-stats" {
  10   providers = {
  11     nomad = nomad.yul1
  12   }
  13   source = "../"
  14
  15   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  16   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  17   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  18   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  19   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  20   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  21   cron                      = "@daily"
  22   datacenters               = ["yul1"]
  23   job_name                  = "etl-stats"
  24 }
  25
  26 module "etl-trending-hoststack" {
  27   providers = {
  28     nomad = nomad.yul1
  29   }
  30   source = "../"
  31
  32   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  33   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  34   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  35   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  36   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  37   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  38   cron                      = "@daily"
  39   datacenters               = ["yul1"]
  40   job_name                  = "etl-trending-hoststack"
  41 }
  42
  43 module "etl-trending-mrr" {
  44   providers = {
  45     nomad = nomad.yul1
  46   }
  47   source = "../"
  48
  49   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  50   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  51   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  52   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  53   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  54   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  55   cron                      = "@daily"
  56   datacenters               = ["yul1"]
  57   job_name                  = "etl-trending-mrr"
  58 }
  59
  60 module "etl-trending-ndrpdr" {
  61   providers = {
  62     nomad = nomad.yul1
  63   }
  64   source = "../"
  65
  66   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  67   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  68   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  69   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  70   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  71   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  72   cron                      = "@daily"
  73   datacenters               = ["yul1"]
  74   job_name                  = "etl-trending-ndrpdr"
  75 }
  76
  77 module "etl-iterative-rls2302" {
  78   providers = {
  79     nomad = nomad.yul1
  80   }
  81   source = "../"
  82
  83   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
  84   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
  85   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
  86   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
  87   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
  88   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
  89   cron                      = "@daily"
  90   datacenters               = ["yul1"]
  91   job_name                  = "etl-iterative-rls2302"
  92 }
  93
  94 module "etl-coverage-rls2302" {
  95   providers = {
  96     nomad = nomad.yul1
  97   }
  98   source = "../"
  99
 100   aws_access_key_id         = data.vault_generic_secret.fdio_logs.data["access_key"]
 101   aws_secret_access_key     = data.vault_generic_secret.fdio_logs.data["secret_key"]
 102   aws_default_region        = data.vault_generic_secret.fdio_logs.data["region"]
 103   out_aws_access_key_id     = data.vault_generic_secret.fdio_docs.data["access_key"]
 104   out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"]
 105   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
 106   cron                      = "@daily"
 107   datacenters               = ["yul1"]
 108   job_name                  = "etl-coverage-rls2302"
 109 }