fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-rls2302.hcl.tftpl

   1 job "${job_name}" {
   2   datacenters = "${datacenters}"
   3   type        = "${type}"
   4   periodic {
   5     cron             = "${cron}"
   6     prohibit_overlap = "${prohibit_overlap}"
   7     time_zone        = "${time_zone}"
   8   }
   9   group "${job_name}" {
  10     restart {
  11       mode = "fail"
  12     }
  13     constraint {
  14       attribute = "$${attr.cpu.arch}"
  15       operator  = "!="
  16       value     = "arm64"
  17     }
  18     constraint {
  19       attribute = "$${node.class}"
  20       value     = "builder"
  21     }
  22     task "${job_name}-hoststack" {
  23       artifact {
  24         source      = "git::https://github.com/FDio/csit"
  25         destination = "local/csit"
  26       }
  27       driver = "docker"
  28       config {
  29         image   = "${image}"
  30         command = "gluesparksubmit"
  31         args = [
  32           "--driver-memory", "20g",
  33           "--executor-memory", "20g",
  34           "--executor-cores", "2",
  35           "--master", "local[2]",
  36           "iterative_hoststack_rls2302.py"
  37         ]
  38         work_dir = "/local/csit/csit.infra.etl"
  39       }
  40       env {
  41         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  42         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  43         AWS_DEFAULT_REGION        = "${aws_default_region}"
  44         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  45         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  46         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  47         ${ envs }
  48       }
  49       resources {
  50         cpu    = ${cpu}
  51         memory = ${memory}
  52       }
  53     }
  54     task "${job_name}-mrr" {
  55       artifact {
  56         source      = "git::https://github.com/FDio/csit"
  57         destination = "local/csit"
  58       }
  59       driver = "docker"
  60       config {
  61         image   = "${image}"
  62         command = "gluesparksubmit"
  63         args = [
  64           "--driver-memory", "20g",
  65           "--executor-memory", "20g",
  66           "--executor-cores", "2",
  67           "--master", "local[2]",
  68           "iterative_mrr_rls2302.py"
  69         ]
  70         work_dir = "/local/csit/csit.infra.etl"
  71       }
  72       env {
  73         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  74         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  75         AWS_DEFAULT_REGION        = "${aws_default_region}"
  76         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  77         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  78         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  79         ${ envs }
  80       }
  81       resources {
  82         cpu    = ${cpu}
  83         memory = ${memory}
  84       }
  85     }
  86     task "${job_name}-ndrpdr" {
  87       artifact {
  88         source      = "git::https://github.com/FDio/csit"
  89         destination = "local/csit"
  90       }
  91       driver = "docker"
  92       config {
  93         image   = "${image}"
  94         command = "gluesparksubmit"
  95         args = [
  96           "--driver-memory", "20g",
  97           "--executor-memory", "20g",
  98           "--executor-cores", "2",
  99           "--master", "local[2]",
 100           "iterative_ndrpdr_rls2302.py"
 101         ]
 102         work_dir = "/local/csit/csit.infra.etl"
 103       }
 104       env {
 105         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 106         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 107         AWS_DEFAULT_REGION        = "${aws_default_region}"
 108         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 109         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 110         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 111         ${ envs }
 112       }
 113       resources {
 114         cpu    = ${cpu}
 115         memory = ${memory}
 116       }
 117     }
 118     task "${job_name}-reconf" {
 119       artifact {
 120         source      = "git::https://github.com/FDio/csit"
 121         destination = "local/csit"
 122       }
 123       driver = "docker"
 124       config {
 125         image   = "${image}"
 126         command = "gluesparksubmit"
 127         args = [
 128           "--driver-memory", "20g",
 129           "--executor-memory", "20g",
 130           "--executor-cores", "2",
 131           "--master", "local[2]",
 132           "iterative_reconf_rls2302.py"
 133         ]
 134         work_dir = "/local/csit/csit.infra.etl"
 135       }
 136       env {
 137         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 138         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 139         AWS_DEFAULT_REGION        = "${aws_default_region}"
 140         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 141         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 142         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 143         ${ envs }
 144       }
 145       resources {
 146         cpu    = ${cpu}
 147         memory = ${memory}
 148       }
 149     }
 150     task "${job_name}-soak" {
 151       artifact {
 152         source      = "git::https://github.com/FDio/csit"
 153         destination = "local/csit"
 154       }
 155       driver = "docker"
 156       config {
 157         image   = "${image}"
 158         command = "gluesparksubmit"
 159         args = [
 160           "--driver-memory", "20g",
 161           "--executor-memory", "20g",
 162           "--executor-cores", "2",
 163           "--master", "local[2]",
 164           "iterative_soak_rls2302.py"
 165         ]
 166         work_dir = "/local/csit/csit.infra.etl"
 167       }
 168       env {
 169         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 170         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 171         AWS_DEFAULT_REGION        = "${aws_default_region}"
 172         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 173         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 174         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 175         ${ envs }
 176       }
 177       resources {
 178         cpu    = ${cpu}
 179         memory = ${memory}
 180       }
 181     }
 182   }
 183 }