fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl

   1 job "${job_name}" {
   2   datacenters = "${datacenters}"
   3   type        = "${type}"
   4   periodic {
   5     cron             = "${cron}"
   6     prohibit_overlap = "${prohibit_overlap}"
   7     time_zone        = "${time_zone}"
   8   }
   9   group "${job_name}" {
  10     restart {
  11       mode = "fail"
  12     }
  13     constraint {
  14       attribute       = "$${attr.cpu.arch}"
  15       operator        = "!="
  16       value           = "arm64"
  17     }
  18     constraint {
  19       attribute      = "$${node.class}"
  20       value          = "builder"
  21     }
  22     task "${job_name}" {
  23       artifact {
  24         source      = "git::https://github.com/FDio/csit"
  25         destination = "local/csit"
  26       }
  27       driver = "docker"
  28       config {
  29         image   = "${image}"
  30         command = "gluesparksubmit"
  31         args = [
  32           "--driver-memory", "30g",
  33           "--executor-memory", "30g",
  34           "trending_hoststack.py"
  35         ]
  36         work_dir = "/local/csit/csit.infra.etl"
  37       }
  38       env {
  39         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  40         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  41         AWS_DEFAULT_REGION        = "${aws_default_region}"
  42         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  43         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  44         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  45         ${ envs }
  46       }
  47       resources {
  48         cpu    = ${cpu}
  49         memory = ${memory}
  50       }
  51     }
  52   }
  53 }