fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl

   1 job "${job_name}" {
   2   datacenters = "${datacenters}"
   3   type        = "${type}"
   4   periodic {
   5     cron             = "${cron}"
   6     prohibit_overlap = "${prohibit_overlap}"
   7     time_zone        = "${time_zone}"
   8   }
   9   group "${job_name}-master" {
  10     restart {
  11       mode = "fail"
  12     }
  13     constraint {
  14       attribute       = "$${attr.cpu.arch}"
  15       operator        = "!="
  16       value           = "arm64"
  17     }
  18     constraint {
  19       attribute      = "$${node.class}"
  20       value          = "builder"
  21     }
  22     task "${job_name}-trending-mrr" {
  23       artifact {
  24         source      = "git::https://github.com/FDio/csit"
  25         destination = "local/csit"
  26       }
  27       driver = "docker"
  28       config {
  29         image   = "${image}"
  30         command = "gluesparksubmit"
  31         args = [
  32           "--driver-memory", "30g",
  33           "--executor-memory", "30g",
  34           "trending_mrr.py"
  35         ]
  36         work_dir = "/local/csit/csit.infra.etl"
  37       }
  38       env {
  39         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  40         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  41         AWS_DEFAULT_REGION        = "${aws_default_region}"
  42         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  43         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  44         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  45         ${ envs }
  46       }
  47       resources {
  48         cpu    = ${cpu}
  49         memory = ${memory}
  50       }
  51     }
  52     task "${job_name}-trending-ndrpdr" {
  53       artifact {
  54         source      = "git::https://github.com/FDio/csit"
  55         destination = "local/csit"
  56       }
  57       driver = "docker"
  58       config {
  59         image   = "${image}"
  60         command = "gluesparksubmit"
  61         args = [
  62           "--driver-memory", "30g",
  63           "--executor-memory", "30g",
  64           "trending_ndrpdr.py"
  65         ]
  66         work_dir = "/local/csit/csit.infra.etl"
  67       }
  68       env {
  69         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  70         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  71         AWS_DEFAULT_REGION        = "${aws_default_region}"
  72         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  73         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  74         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  75         ${ envs }
  76       }
  77       resources {
  78         cpu    = ${cpu}
  79         memory = ${memory}
  80       }
  81     }
  82     task "${job_name}-trending-soak" {
  83       artifact {
  84         source      = "git::https://github.com/FDio/csit"
  85         destination = "local/csit"
  86       }
  87       driver = "docker"
  88       config {
  89         image   = "${image}"
  90         command = "gluesparksubmit"
  91         args = [
  92           "--driver-memory", "30g",
  93           "--executor-memory", "30g",
  94           "trending_soak.py"
  95         ]
  96         work_dir = "/local/csit/csit.infra.etl"
  97       }
  98       env {
  99         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 100         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 101         AWS_DEFAULT_REGION        = "${aws_default_region}"
 102         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 103         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 104         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 105         ${ envs }
 106       }
 107       resources {
 108         cpu    = ${cpu}
 109         memory = ${memory}
 110       }
 111     }
 112     task "${job_name}-trending-reconf" {
 113       artifact {
 114         source      = "git::https://github.com/FDio/csit"
 115         destination = "local/csit"
 116       }
 117       driver = "docker"
 118       config {
 119         image   = "${image}"
 120         command = "gluesparksubmit"
 121         args = [
 122           "--driver-memory", "30g",
 123           "--executor-memory", "30g",
 124           "trending_reconf.py"
 125         ]
 126         work_dir = "/local/csit/csit.infra.etl"
 127       }
 128       env {
 129         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 130         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 131         AWS_DEFAULT_REGION        = "${aws_default_region}"
 132         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 133         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 134         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 135         ${ envs }
 136       }
 137       resources {
 138         cpu    = ${cpu}
 139         memory = ${memory}
 140       }
 141     }
 142     task "${job_name}-trending-hoststack" {
 143       artifact {
 144         source      = "git::https://github.com/FDio/csit"
 145         destination = "local/csit"
 146       }
 147       driver = "docker"
 148       config {
 149         image   = "${image}"
 150         command = "gluesparksubmit"
 151         args = [
 152           "--driver-memory", "30g",
 153           "--executor-memory", "30g",
 154           "trending_hoststack.py"
 155         ]
 156         work_dir = "/local/csit/csit.infra.etl"
 157       }
 158       env {
 159         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 160         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 161         AWS_DEFAULT_REGION        = "${aws_default_region}"
 162         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 163         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 164         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 165         ${ envs }
 166       }
 167       resources {
 168         cpu    = ${cpu}
 169         memory = ${memory}
 170       }
 171     }
 172     task "${job_name}-stats" {
 173       artifact {
 174         source      = "git::https://github.com/FDio/csit"
 175         destination = "local/csit"
 176       }
 177       driver = "docker"
 178       config {
 179         image   = "${image}"
 180         command = "gluesparksubmit"
 181         args = [
 182           "--driver-memory", "10g",
 183           "--executor-memory", "10g",
 184           "stats.py"
 185         ]
 186         work_dir = "/local/csit/csit.infra.etl"
 187       }
 188       env {
 189         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 190         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 191         AWS_DEFAULT_REGION        = "${aws_default_region}"
 192         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 193         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 194         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 195         ${ envs }
 196       }
 197       resources {
 198         cpu    = ${cpu}
 199         memory = ${memory}
 200       }
 201     }
 202   }
 203   group "${job_name}-rls2302" {
 204     restart {
 205       mode = "fail"
 206     }
 207     constraint {
 208       attribute = "$${attr.cpu.arch}"
 209       operator  = "!="
 210       value     = "arm64"
 211     }
 212     constraint {
 213       attribute = "$${node.class}"
 214       value     = "builder"
 215     }
 216     task "${job_name}-coverage-device" {
 217       artifact {
 218         source      = "git::https://github.com/FDio/csit"
 219         destination = "local/csit"
 220       }
 221       driver = "docker"
 222       config {
 223         image   = "${image}"
 224         command = "gluesparksubmit"
 225         args = [
 226           "--driver-memory", "20g",
 227           "--executor-memory", "20g",
 228           "--executor-cores", "2",
 229           "--master", "local[2]",
 230           "coverage_device_rls2302.py"
 231         ]
 232         work_dir = "/local/csit/csit.infra.etl"
 233       }
 234       env {
 235         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 236         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 237         AWS_DEFAULT_REGION        = "${aws_default_region}"
 238         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 239         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 240         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 241         ${ envs }
 242       }
 243       resources {
 244         cpu    = ${cpu}
 245         memory = ${memory}
 246       }
 247     }
 248     task "${job_name}-coverage-hoststack" {
 249       artifact {
 250         source      = "git::https://github.com/FDio/csit"
 251         destination = "local/csit"
 252       }
 253       driver = "docker"
 254       config {
 255         image   = "${image}"
 256         command = "gluesparksubmit"
 257         args = [
 258           "--driver-memory", "20g",
 259           "--executor-memory", "20g",
 260           "--executor-cores", "2",
 261           "--master", "local[2]",
 262           "coverage_hoststack_rls2302.py"
 263         ]
 264         work_dir = "/local/csit/csit.infra.etl"
 265       }
 266       env {
 267         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 268         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 269         AWS_DEFAULT_REGION        = "${aws_default_region}"
 270         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 271         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 272         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 273         ${ envs }
 274       }
 275       resources {
 276         cpu    = ${cpu}
 277         memory = ${memory}
 278       }
 279     }
 280     task "${job_name}-coverage-mrr" {
 281       artifact {
 282         source      = "git::https://github.com/FDio/csit"
 283         destination = "local/csit"
 284       }
 285       driver = "docker"
 286       config {
 287         image   = "${image}"
 288         command = "gluesparksubmit"
 289         args = [
 290           "--driver-memory", "20g",
 291           "--executor-memory", "20g",
 292           "--executor-cores", "2",
 293           "--master", "local[2]",
 294           "coverage_mrr_rls2302.py"
 295         ]
 296         work_dir = "/local/csit/csit.infra.etl"
 297       }
 298       env {
 299         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 300         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 301         AWS_DEFAULT_REGION        = "${aws_default_region}"
 302         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 303         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 304         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 305         ${ envs }
 306       }
 307       resources {
 308         cpu    = ${cpu}
 309         memory = ${memory}
 310       }
 311     }
 312     task "${job_name}-coverage-ndrpdr" {
 313       artifact {
 314         source      = "git::https://github.com/FDio/csit"
 315         destination = "local/csit"
 316       }
 317       driver = "docker"
 318       config {
 319         image   = "${image}"
 320         command = "gluesparksubmit"
 321         args = [
 322           "--driver-memory", "20g",
 323           "--executor-memory", "20g",
 324           "--executor-cores", "2",
 325           "--master", "local[2]",
 326           "coverage_ndrpdr_rls2302.py"
 327         ]
 328         work_dir = "/local/csit/csit.infra.etl"
 329       }
 330       env {
 331         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 332         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 333         AWS_DEFAULT_REGION        = "${aws_default_region}"
 334         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 335         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 336         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 337         ${ envs }
 338       }
 339       resources {
 340         cpu    = ${cpu}
 341         memory = ${memory}
 342       }
 343     }
 344     task "${job_name}-coverage-reconf" {
 345       artifact {
 346         source      = "git::https://github.com/FDio/csit"
 347         destination = "local/csit"
 348       }
 349       driver = "docker"
 350       config {
 351         image   = "${image}"
 352         command = "gluesparksubmit"
 353         args = [
 354           "--driver-memory", "20g",
 355           "--executor-memory", "20g",
 356           "--executor-cores", "2",
 357           "--master", "local[2]",
 358           "coverage_reconf_rls2302.py"
 359         ]
 360         work_dir = "/local/csit/csit.infra.etl"
 361       }
 362       env {
 363         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 364         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 365         AWS_DEFAULT_REGION        = "${aws_default_region}"
 366         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 367         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 368         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 369         ${ envs }
 370       }
 371       resources {
 372         cpu    = ${cpu}
 373         memory = ${memory}
 374       }
 375     }
 376     task "${job_name}-coverage-soak" {
 377       artifact {
 378         source      = "git::https://github.com/FDio/csit"
 379         destination = "local/csit"
 380       }
 381       driver = "docker"
 382       config {
 383         image   = "${image}"
 384         command = "gluesparksubmit"
 385         args = [
 386           "--driver-memory", "20g",
 387           "--executor-memory", "20g",
 388           "--executor-cores", "2",
 389           "--master", "local[2]",
 390           "coverage_soak_rls2302.py"
 391         ]
 392         work_dir = "/local/csit/csit.infra.etl"
 393       }
 394       env {
 395         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 396         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 397         AWS_DEFAULT_REGION        = "${aws_default_region}"
 398         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 399         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 400         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 401         ${ envs }
 402       }
 403       resources {
 404         cpu    = ${cpu}
 405         memory = ${memory}
 406       }
 407     }
 408     task "${job_name}-iterative-hoststack" {
 409       artifact {
 410         source      = "git::https://github.com/FDio/csit"
 411         destination = "local/csit"
 412       }
 413       driver = "docker"
 414       config {
 415         image   = "${image}"
 416         command = "gluesparksubmit"
 417         args = [
 418           "--driver-memory", "20g",
 419           "--executor-memory", "20g",
 420           "--executor-cores", "2",
 421           "--master", "local[2]",
 422           "iterative_hoststack_rls2302.py"
 423         ]
 424         work_dir = "/local/csit/csit.infra.etl"
 425       }
 426       env {
 427         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 428         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 429         AWS_DEFAULT_REGION        = "${aws_default_region}"
 430         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 431         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 432         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 433         ${ envs }
 434       }
 435       resources {
 436         cpu    = ${cpu}
 437         memory = ${memory}
 438       }
 439     }
 440     task "${job_name}-iterative-mrr" {
 441       artifact {
 442         source      = "git::https://github.com/FDio/csit"
 443         destination = "local/csit"
 444       }
 445       driver = "docker"
 446       config {
 447         image   = "${image}"
 448         command = "gluesparksubmit"
 449         args = [
 450           "--driver-memory", "20g",
 451           "--executor-memory", "20g",
 452           "--executor-cores", "2",
 453           "--master", "local[2]",
 454           "iterative_mrr_rls2302.py"
 455         ]
 456         work_dir = "/local/csit/csit.infra.etl"
 457       }
 458       env {
 459         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 460         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 461         AWS_DEFAULT_REGION        = "${aws_default_region}"
 462         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 463         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 464         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 465         ${ envs }
 466       }
 467       resources {
 468         cpu    = ${cpu}
 469         memory = ${memory}
 470       }
 471     }
 472     task "${job_name}-iterative-ndrpdr" {
 473       artifact {
 474         source      = "git::https://github.com/FDio/csit"
 475         destination = "local/csit"
 476       }
 477       driver = "docker"
 478       config {
 479         image   = "${image}"
 480         command = "gluesparksubmit"
 481         args = [
 482           "--driver-memory", "20g",
 483           "--executor-memory", "20g",
 484           "--executor-cores", "2",
 485           "--master", "local[2]",
 486           "iterative_ndrpdr_rls2302.py"
 487         ]
 488         work_dir = "/local/csit/csit.infra.etl"
 489       }
 490       env {
 491         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 492         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 493         AWS_DEFAULT_REGION        = "${aws_default_region}"
 494         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 495         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 496         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 497         ${ envs }
 498       }
 499       resources {
 500         cpu    = ${cpu}
 501         memory = ${memory}
 502       }
 503     }
 504     task "${job_name}-iterative-reconf" {
 505       artifact {
 506         source      = "git::https://github.com/FDio/csit"
 507         destination = "local/csit"
 508       }
 509       driver = "docker"
 510       config {
 511         image   = "${image}"
 512         command = "gluesparksubmit"
 513         args = [
 514           "--driver-memory", "20g",
 515           "--executor-memory", "20g",
 516           "--executor-cores", "2",
 517           "--master", "local[2]",
 518           "iterative_reconf_rls2302.py"
 519         ]
 520         work_dir = "/local/csit/csit.infra.etl"
 521       }
 522       env {
 523         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 524         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 525         AWS_DEFAULT_REGION        = "${aws_default_region}"
 526         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 527         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 528         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 529         ${ envs }
 530       }
 531       resources {
 532         cpu    = ${cpu}
 533         memory = ${memory}
 534       }
 535     }
 536     task "${job_name}-iterative-soak" {
 537       artifact {
 538         source      = "git::https://github.com/FDio/csit"
 539         destination = "local/csit"
 540       }
 541       driver = "docker"
 542       config {
 543         image   = "${image}"
 544         command = "gluesparksubmit"
 545         args = [
 546           "--driver-memory", "20g",
 547           "--executor-memory", "20g",
 548           "--executor-cores", "2",
 549           "--master", "local[2]",
 550           "iterative_soak_rls2302.py"
 551         ]
 552         work_dir = "/local/csit/csit.infra.etl"
 553       }
 554       env {
 555         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 556         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 557         AWS_DEFAULT_REGION        = "${aws_default_region}"
 558         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 559         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 560         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 561         ${ envs }
 562       }
 563       resources {
 564         cpu    = ${cpu}
 565         memory = ${memory}
 566       }
 567     }
 568   }
 569 }