fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl

   1 job "${job_name}" {
   2   datacenters = "${datacenters}"
   3   type        = "${type}"
   4   periodic {
   5     cron             = "${cron}"
   6     prohibit_overlap = "${prohibit_overlap}"
   7     time_zone        = "${time_zone}"
   8   }
   9   group "${job_name}-master" {
  10     restart {
  11       mode = "fail"
  12     }
  13     constraint {
  14       attribute       = "$${attr.cpu.arch}"
  15       operator        = "!="
  16       value           = "arm64"
  17     }
  18     constraint {
  19       attribute      = "$${node.class}"
  20       value          = "builder"
  21     }
  22     task "${job_name}-trending-mrr" {
  23       artifact {
  24         source      = "git::https://github.com/FDio/csit"
  25         destination = "local/csit"
  26       }
  27       driver = "docker"
  28       config {
  29         image   = "${image}"
  30         command = "gluesparksubmit"
  31         args = [
  32           "--driver-memory", "30g",
  33           "--executor-memory", "30g",
  34           "trending_mrr.py"
  35         ]
  36         work_dir = "/local/csit/csit.infra.etl"
  37       }
  38       env {
  39         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  40         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  41         AWS_DEFAULT_REGION        = "${aws_default_region}"
  42         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  43         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  44         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  45         ${ envs }
  46       }
  47       resources {
  48         cpu    = ${cpu}
  49         memory = ${memory}
  50       }
  51     }
  52     task "${job_name}-trending-ndrpdr" {
  53       artifact {
  54         source      = "git::https://github.com/FDio/csit"
  55         destination = "local/csit"
  56       }
  57       driver = "docker"
  58       config {
  59         image   = "${image}"
  60         command = "gluesparksubmit"
  61         args = [
  62           "--driver-memory", "30g",
  63           "--executor-memory", "30g",
  64           "trending_ndrpdr.py"
  65         ]
  66         work_dir = "/local/csit/csit.infra.etl"
  67       }
  68       env {
  69         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  70         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  71         AWS_DEFAULT_REGION        = "${aws_default_region}"
  72         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  73         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  74         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  75         ${ envs }
  76       }
  77       resources {
  78         cpu    = ${cpu}
  79         memory = ${memory}
  80       }
  81     }
  82     task "${job_name}-trending-soak" {
  83       artifact {
  84         source      = "git::https://github.com/FDio/csit"
  85         destination = "local/csit"
  86       }
  87       driver = "docker"
  88       config {
  89         image   = "${image}"
  90         command = "gluesparksubmit"
  91         args = [
  92           "--driver-memory", "30g",
  93           "--executor-memory", "30g",
  94           "trending_soak.py"
  95         ]
  96         work_dir = "/local/csit/csit.infra.etl"
  97       }
  98       env {
  99         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 100         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 101         AWS_DEFAULT_REGION        = "${aws_default_region}"
 102         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 103         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 104         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 105         ${ envs }
 106       }
 107       resources {
 108         cpu    = ${cpu}
 109         memory = ${memory}
 110       }
 111     }
 112     task "${job_name}-stats" {
 113       artifact {
 114         source      = "git::https://github.com/FDio/csit"
 115         destination = "local/csit"
 116       }
 117       driver = "docker"
 118       config {
 119         image   = "${image}"
 120         command = "gluesparksubmit"
 121         args = [
 122           "--driver-memory", "10g",
 123           "--executor-memory", "10g",
 124           "stats.py"
 125         ]
 126         work_dir = "/local/csit/csit.infra.etl"
 127       }
 128       env {
 129         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 130         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 131         AWS_DEFAULT_REGION        = "${aws_default_region}"
 132         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 133         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 134         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 135         ${ envs }
 136       }
 137       resources {
 138         cpu    = ${cpu}
 139         memory = ${memory}
 140       }
 141     }
 142   }
 143   group "${job_name}-rls2210" {
 144     restart {
 145       mode = "fail"
 146     }
 147     constraint {
 148       attribute = "$${attr.cpu.arch}"
 149       operator  = "!="
 150       value     = "arm64"
 151     }
 152     constraint {
 153       attribute = "$${node.class}"
 154       value     = "builder"
 155     }
 156     task "${job_name}-coverage-device" {
 157       artifact {
 158         source      = "git::https://github.com/FDio/csit"
 159         destination = "local/csit"
 160       }
 161       driver = "docker"
 162       config {
 163         image   = "${image}"
 164         command = "gluesparksubmit"
 165         args = [
 166           "--driver-memory", "20g",
 167           "--executor-memory", "20g",
 168           "--executor-cores", "2",
 169           "--master", "local[2]",
 170           "coverage_device_rls2210.py"
 171         ]
 172         work_dir = "/local/csit/csit.infra.etl"
 173       }
 174       env {
 175         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 176         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 177         AWS_DEFAULT_REGION        = "${aws_default_region}"
 178         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 179         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 180         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 181         ${ envs }
 182       }
 183       resources {
 184         cpu    = ${cpu}
 185         memory = ${memory}
 186       }
 187     }
 188     task "${job_name}-coverage-mrr" {
 189       artifact {
 190         source      = "git::https://github.com/FDio/csit"
 191         destination = "local/csit"
 192       }
 193       driver = "docker"
 194       config {
 195         image   = "${image}"
 196         command = "gluesparksubmit"
 197         args = [
 198           "--driver-memory", "20g",
 199           "--executor-memory", "20g",
 200           "--executor-cores", "2",
 201           "--master", "local[2]",
 202           "coverage_mrr_rls2210.py"
 203         ]
 204         work_dir = "/local/csit/csit.infra.etl"
 205       }
 206       env {
 207         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 208         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 209         AWS_DEFAULT_REGION        = "${aws_default_region}"
 210         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 211         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 212         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 213         ${ envs }
 214       }
 215       resources {
 216         cpu    = ${cpu}
 217         memory = ${memory}
 218       }
 219     }
 220     task "${job_name}-coverage-ndrpdr" {
 221       artifact {
 222         source      = "git::https://github.com/FDio/csit"
 223         destination = "local/csit"
 224       }
 225       driver = "docker"
 226       config {
 227         image   = "${image}"
 228         command = "gluesparksubmit"
 229         args = [
 230           "--driver-memory", "20g",
 231           "--executor-memory", "20g",
 232           "--executor-cores", "2",
 233           "--master", "local[2]",
 234           "coverage_ndrpdr_rls2210.py"
 235         ]
 236         work_dir = "/local/csit/csit.infra.etl"
 237       }
 238       env {
 239         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 240         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 241         AWS_DEFAULT_REGION        = "${aws_default_region}"
 242         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 243         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 244         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 245         ${ envs }
 246       }
 247       resources {
 248         cpu    = ${cpu}
 249         memory = ${memory}
 250       }
 251     }
 252     task "${job_name}-coverage-soak" {
 253       artifact {
 254         source      = "git::https://github.com/FDio/csit"
 255         destination = "local/csit"
 256       }
 257       driver = "docker"
 258       config {
 259         image   = "${image}"
 260         command = "gluesparksubmit"
 261         args = [
 262           "--driver-memory", "20g",
 263           "--executor-memory", "20g",
 264           "--executor-cores", "2",
 265           "--master", "local[2]",
 266           "coverage_soak_rls2210.py"
 267         ]
 268         work_dir = "/local/csit/csit.infra.etl"
 269       }
 270       env {
 271         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 272         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 273         AWS_DEFAULT_REGION        = "${aws_default_region}"
 274         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 275         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 276         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 277         ${ envs }
 278       }
 279       resources {
 280         cpu    = ${cpu}
 281         memory = ${memory}
 282       }
 283     }
 284     task "${job_name}-iterative-mrr" {
 285       artifact {
 286         source      = "git::https://github.com/FDio/csit"
 287         destination = "local/csit"
 288       }
 289       driver = "docker"
 290       config {
 291         image   = "${image}"
 292         command = "gluesparksubmit"
 293         args = [
 294           "--driver-memory", "20g",
 295           "--executor-memory", "20g",
 296           "--executor-cores", "2",
 297           "--master", "local[2]",
 298           "iterative_mrr_rls2210.py"
 299         ]
 300         work_dir = "/local/csit/csit.infra.etl"
 301       }
 302       env {
 303         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 304         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 305         AWS_DEFAULT_REGION        = "${aws_default_region}"
 306         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 307         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 308         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 309         ${ envs }
 310       }
 311       resources {
 312         cpu    = ${cpu}
 313         memory = ${memory}
 314       }
 315     }
 316     task "${job_name}-iterative-ndrpdr" {
 317       artifact {
 318         source      = "git::https://github.com/FDio/csit"
 319         destination = "local/csit"
 320       }
 321       driver = "docker"
 322       config {
 323         image   = "${image}"
 324         command = "gluesparksubmit"
 325         args = [
 326           "--driver-memory", "20g",
 327           "--executor-memory", "20g",
 328           "--executor-cores", "2",
 329           "--master", "local[2]",
 330           "iterative_ndrpdr_rls2210.py"
 331         ]
 332         work_dir = "/local/csit/csit.infra.etl"
 333       }
 334       env {
 335         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 336         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 337         AWS_DEFAULT_REGION        = "${aws_default_region}"
 338         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 339         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 340         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 341         ${ envs }
 342       }
 343       resources {
 344         cpu    = ${cpu}
 345         memory = ${memory}
 346       }
 347     }
 348     task "${job_name}-iterative-soak" {
 349       artifact {
 350         source      = "git::https://github.com/FDio/csit"
 351         destination = "local/csit"
 352       }
 353       driver = "docker"
 354       config {
 355         image   = "${image}"
 356         command = "gluesparksubmit"
 357         args = [
 358           "--driver-memory", "20g",
 359           "--executor-memory", "20g",
 360           "--executor-cores", "2",
 361           "--master", "local[2]",
 362           "iterative_soak_rls2210.py"
 363         ]
 364         work_dir = "/local/csit/csit.infra.etl"
 365       }
 366       env {
 367         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 368         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 369         AWS_DEFAULT_REGION        = "${aws_default_region}"
 370         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 371         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 372         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 373         ${ envs }
 374       }
 375       resources {
 376         cpu    = ${cpu}
 377         memory = ${memory}
 378       }
 379     }
 380   }
 381 }