fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl

   1 job "${job_name}" {
   2   datacenters = "${datacenters}"
   3   type        = "${type}"
   4   periodic {
   5     cron             = "${cron}"
   6     prohibit_overlap = "${prohibit_overlap}"
   7     time_zone        = "${time_zone}"
   8   }
   9   group "${job_name}-trending" {
  10     restart {
  11       mode = "fail"
  12     }
  13     constraint {
  14       attribute       = "$${attr.cpu.arch}"
  15       operator        = "!="
  16       value           = "arm64"
  17     }
  18     constraint {
  19       attribute      = "$${node.class}"
  20       value          = "builder"
  21     }
  22     task "${job_name}-trending-mrr" {
  23       artifact {
  24         source      = "git::https://github.com/FDio/csit"
  25         destination = "local/csit"
  26       }
  27       driver = "docker"
  28       config {
  29         image   = "${image}"
  30         command = "gluesparksubmit"
  31         args = [
  32           "--driver-memory", "30g",
  33           "--executor-memory", "30g",
  34           "trending_mrr.py"
  35         ]
  36         work_dir = "/local/csit/csit.infra.etl"
  37       }
  38       env {
  39         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  40         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  41         AWS_DEFAULT_REGION        = "${aws_default_region}"
  42         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  43         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  44         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  45         ${ envs }
  46       }
  47       resources {
  48         cpu    = ${cpu}
  49         memory = ${memory}
  50       }
  51     }
  52     task "${job_name}-trending-ndrpdr" {
  53       artifact {
  54         source      = "git::https://github.com/FDio/csit"
  55         destination = "local/csit"
  56       }
  57       driver = "docker"
  58       config {
  59         image   = "${image}"
  60         command = "gluesparksubmit"
  61         args = [
  62           "--driver-memory", "30g",
  63           "--executor-memory", "30g",
  64           "trending_ndrpdr.py"
  65         ]
  66         work_dir = "/local/csit/csit.infra.etl"
  67       }
  68       env {
  69         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  70         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  71         AWS_DEFAULT_REGION        = "${aws_default_region}"
  72         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  73         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  74         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  75         ${ envs }
  76       }
  77       resources {
  78         cpu    = ${cpu}
  79         memory = ${memory}
  80       }
  81     }
  82     task "${job_name}-trending-soak" {
  83       artifact {
  84         source      = "git::https://github.com/FDio/csit"
  85         destination = "local/csit"
  86       }
  87       driver = "docker"
  88       config {
  89         image   = "${image}"
  90         command = "gluesparksubmit"
  91         args = [
  92           "--driver-memory", "30g",
  93           "--executor-memory", "30g",
  94           "trending_soak.py"
  95         ]
  96         work_dir = "/local/csit/csit.infra.etl"
  97       }
  98       env {
  99         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 100         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 101         AWS_DEFAULT_REGION        = "${aws_default_region}"
 102         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 103         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 104         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 105         ${ envs }
 106       }
 107       resources {
 108         cpu    = ${cpu}
 109         memory = ${memory}
 110       }
 111     }
 112     task "${job_name}-trending-reconf" {
 113       artifact {
 114         source      = "git::https://github.com/FDio/csit"
 115         destination = "local/csit"
 116       }
 117       driver = "docker"
 118       config {
 119         image   = "${image}"
 120         command = "gluesparksubmit"
 121         args = [
 122           "--driver-memory", "30g",
 123           "--executor-memory", "30g",
 124           "trending_reconf.py"
 125         ]
 126         work_dir = "/local/csit/csit.infra.etl"
 127       }
 128       env {
 129         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 130         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 131         AWS_DEFAULT_REGION        = "${aws_default_region}"
 132         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 133         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 134         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 135         ${ envs }
 136       }
 137       resources {
 138         cpu    = ${cpu}
 139         memory = ${memory}
 140       }
 141     }
 142     task "${job_name}-trending-hoststack" {
 143       artifact {
 144         source      = "git::https://github.com/FDio/csit"
 145         destination = "local/csit"
 146       }
 147       driver = "docker"
 148       config {
 149         image   = "${image}"
 150         command = "gluesparksubmit"
 151         args = [
 152           "--driver-memory", "30g",
 153           "--executor-memory", "30g",
 154           "trending_hoststack.py"
 155         ]
 156         work_dir = "/local/csit/csit.infra.etl"
 157       }
 158       env {
 159         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 160         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 161         AWS_DEFAULT_REGION        = "${aws_default_region}"
 162         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 163         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 164         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 165         ${ envs }
 166       }
 167       resources {
 168         cpu    = ${cpu}
 169         memory = ${memory}
 170       }
 171     }
 172   }
 173   group "${job_name}-stats" {
 174     restart {
 175       mode = "fail"
 176     }
 177     constraint {
 178       attribute       = "$${attr.cpu.arch}"
 179       operator        = "!="
 180       value           = "arm64"
 181     }
 182     constraint {
 183       attribute      = "$${node.class}"
 184       value          = "builder"
 185     }
 186     task "${job_name}-stats" {
 187       artifact {
 188         source      = "git::https://github.com/FDio/csit"
 189         destination = "local/csit"
 190       }
 191       driver = "docker"
 192       config {
 193         image   = "${image}"
 194         command = "gluesparksubmit"
 195         args = [
 196           "--driver-memory", "10g",
 197           "--executor-memory", "10g",
 198           "stats.py"
 199         ]
 200         work_dir = "/local/csit/csit.infra.etl"
 201       }
 202       env {
 203         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 204         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 205         AWS_DEFAULT_REGION        = "${aws_default_region}"
 206         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 207         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 208         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 209         ${ envs }
 210       }
 211       resources {
 212         cpu    = ${cpu}
 213         memory = ${memory}
 214       }
 215     }
 216   }
 217   group "${job_name}-coverage-rls2302" {
 218     restart {
 219       mode = "fail"
 220     }
 221     constraint {
 222       attribute = "$${attr.cpu.arch}"
 223       operator  = "!="
 224       value     = "arm64"
 225     }
 226     constraint {
 227       attribute = "$${node.class}"
 228       value     = "builder"
 229     }
 230     task "${job_name}-coverage-device" {
 231       artifact {
 232         source      = "git::https://github.com/FDio/csit"
 233         destination = "local/csit"
 234       }
 235       driver = "docker"
 236       config {
 237         image   = "${image}"
 238         command = "gluesparksubmit"
 239         args = [
 240           "--driver-memory", "20g",
 241           "--executor-memory", "20g",
 242           "--executor-cores", "2",
 243           "--master", "local[2]",
 244           "coverage_device_rls2302.py"
 245         ]
 246         work_dir = "/local/csit/csit.infra.etl"
 247       }
 248       env {
 249         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 250         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 251         AWS_DEFAULT_REGION        = "${aws_default_region}"
 252         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 253         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 254         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 255         ${ envs }
 256       }
 257       resources {
 258         cpu    = ${cpu}
 259         memory = ${memory}
 260       }
 261     }
 262     task "${job_name}-coverage-hoststack" {
 263       artifact {
 264         source      = "git::https://github.com/FDio/csit"
 265         destination = "local/csit"
 266       }
 267       driver = "docker"
 268       config {
 269         image   = "${image}"
 270         command = "gluesparksubmit"
 271         args = [
 272           "--driver-memory", "20g",
 273           "--executor-memory", "20g",
 274           "--executor-cores", "2",
 275           "--master", "local[2]",
 276           "coverage_hoststack_rls2302.py"
 277         ]
 278         work_dir = "/local/csit/csit.infra.etl"
 279       }
 280       env {
 281         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 282         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 283         AWS_DEFAULT_REGION        = "${aws_default_region}"
 284         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 285         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 286         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 287         ${ envs }
 288       }
 289       resources {
 290         cpu    = ${cpu}
 291         memory = ${memory}
 292       }
 293     }
 294     task "${job_name}-coverage-mrr" {
 295       artifact {
 296         source      = "git::https://github.com/FDio/csit"
 297         destination = "local/csit"
 298       }
 299       driver = "docker"
 300       config {
 301         image   = "${image}"
 302         command = "gluesparksubmit"
 303         args = [
 304           "--driver-memory", "20g",
 305           "--executor-memory", "20g",
 306           "--executor-cores", "2",
 307           "--master", "local[2]",
 308           "coverage_mrr_rls2302.py"
 309         ]
 310         work_dir = "/local/csit/csit.infra.etl"
 311       }
 312       env {
 313         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 314         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 315         AWS_DEFAULT_REGION        = "${aws_default_region}"
 316         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 317         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 318         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 319         ${ envs }
 320       }
 321       resources {
 322         cpu    = ${cpu}
 323         memory = ${memory}
 324       }
 325     }
 326     task "${job_name}-coverage-ndrpdr" {
 327       artifact {
 328         source      = "git::https://github.com/FDio/csit"
 329         destination = "local/csit"
 330       }
 331       driver = "docker"
 332       config {
 333         image   = "${image}"
 334         command = "gluesparksubmit"
 335         args = [
 336           "--driver-memory", "20g",
 337           "--executor-memory", "20g",
 338           "--executor-cores", "2",
 339           "--master", "local[2]",
 340           "coverage_ndrpdr_rls2302.py"
 341         ]
 342         work_dir = "/local/csit/csit.infra.etl"
 343       }
 344       env {
 345         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 346         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 347         AWS_DEFAULT_REGION        = "${aws_default_region}"
 348         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 349         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 350         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 351         ${ envs }
 352       }
 353       resources {
 354         cpu    = ${cpu}
 355         memory = ${memory}
 356       }
 357     }
 358     task "${job_name}-coverage-reconf" {
 359       artifact {
 360         source      = "git::https://github.com/FDio/csit"
 361         destination = "local/csit"
 362       }
 363       driver = "docker"
 364       config {
 365         image   = "${image}"
 366         command = "gluesparksubmit"
 367         args = [
 368           "--driver-memory", "20g",
 369           "--executor-memory", "20g",
 370           "--executor-cores", "2",
 371           "--master", "local[2]",
 372           "coverage_reconf_rls2302.py"
 373         ]
 374         work_dir = "/local/csit/csit.infra.etl"
 375       }
 376       env {
 377         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 378         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 379         AWS_DEFAULT_REGION        = "${aws_default_region}"
 380         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 381         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 382         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 383         ${ envs }
 384       }
 385       resources {
 386         cpu    = ${cpu}
 387         memory = ${memory}
 388       }
 389     }
 390     task "${job_name}-coverage-soak" {
 391       artifact {
 392         source      = "git::https://github.com/FDio/csit"
 393         destination = "local/csit"
 394       }
 395       driver = "docker"
 396       config {
 397         image   = "${image}"
 398         command = "gluesparksubmit"
 399         args = [
 400           "--driver-memory", "20g",
 401           "--executor-memory", "20g",
 402           "--executor-cores", "2",
 403           "--master", "local[2]",
 404           "coverage_soak_rls2302.py"
 405         ]
 406         work_dir = "/local/csit/csit.infra.etl"
 407       }
 408       env {
 409         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 410         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 411         AWS_DEFAULT_REGION        = "${aws_default_region}"
 412         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 413         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 414         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 415         ${ envs }
 416       }
 417       resources {
 418         cpu    = ${cpu}
 419         memory = ${memory}
 420       }
 421     }
 422   }
 423   group "${job_name}-iterative-rls2302" {
 424     restart {
 425       mode = "fail"
 426     }
 427     constraint {
 428       attribute = "$${attr.cpu.arch}"
 429       operator  = "!="
 430       value     = "arm64"
 431     }
 432     constraint {
 433       attribute = "$${node.class}"
 434       value     = "builder"
 435     }
 436     task "${job_name}-iterative-hoststack" {
 437       artifact {
 438         source      = "git::https://github.com/FDio/csit"
 439         destination = "local/csit"
 440       }
 441       driver = "docker"
 442       config {
 443         image   = "${image}"
 444         command = "gluesparksubmit"
 445         args = [
 446           "--driver-memory", "20g",
 447           "--executor-memory", "20g",
 448           "--executor-cores", "2",
 449           "--master", "local[2]",
 450           "iterative_hoststack_rls2302.py"
 451         ]
 452         work_dir = "/local/csit/csit.infra.etl"
 453       }
 454       env {
 455         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 456         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 457         AWS_DEFAULT_REGION        = "${aws_default_region}"
 458         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 459         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 460         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 461         ${ envs }
 462       }
 463       resources {
 464         cpu    = ${cpu}
 465         memory = ${memory}
 466       }
 467     }
 468     task "${job_name}-iterative-mrr" {
 469       artifact {
 470         source      = "git::https://github.com/FDio/csit"
 471         destination = "local/csit"
 472       }
 473       driver = "docker"
 474       config {
 475         image   = "${image}"
 476         command = "gluesparksubmit"
 477         args = [
 478           "--driver-memory", "20g",
 479           "--executor-memory", "20g",
 480           "--executor-cores", "2",
 481           "--master", "local[2]",
 482           "iterative_mrr_rls2302.py"
 483         ]
 484         work_dir = "/local/csit/csit.infra.etl"
 485       }
 486       env {
 487         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 488         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 489         AWS_DEFAULT_REGION        = "${aws_default_region}"
 490         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 491         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 492         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 493         ${ envs }
 494       }
 495       resources {
 496         cpu    = ${cpu}
 497         memory = ${memory}
 498       }
 499     }
 500     task "${job_name}-iterative-ndrpdr" {
 501       artifact {
 502         source      = "git::https://github.com/FDio/csit"
 503         destination = "local/csit"
 504       }
 505       driver = "docker"
 506       config {
 507         image   = "${image}"
 508         command = "gluesparksubmit"
 509         args = [
 510           "--driver-memory", "20g",
 511           "--executor-memory", "20g",
 512           "--executor-cores", "2",
 513           "--master", "local[2]",
 514           "iterative_ndrpdr_rls2302.py"
 515         ]
 516         work_dir = "/local/csit/csit.infra.etl"
 517       }
 518       env {
 519         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 520         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 521         AWS_DEFAULT_REGION        = "${aws_default_region}"
 522         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 523         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 524         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 525         ${ envs }
 526       }
 527       resources {
 528         cpu    = ${cpu}
 529         memory = ${memory}
 530       }
 531     }
 532     task "${job_name}-iterative-reconf" {
 533       artifact {
 534         source      = "git::https://github.com/FDio/csit"
 535         destination = "local/csit"
 536       }
 537       driver = "docker"
 538       config {
 539         image   = "${image}"
 540         command = "gluesparksubmit"
 541         args = [
 542           "--driver-memory", "20g",
 543           "--executor-memory", "20g",
 544           "--executor-cores", "2",
 545           "--master", "local[2]",
 546           "iterative_reconf_rls2302.py"
 547         ]
 548         work_dir = "/local/csit/csit.infra.etl"
 549       }
 550       env {
 551         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 552         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 553         AWS_DEFAULT_REGION        = "${aws_default_region}"
 554         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 555         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 556         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 557         ${ envs }
 558       }
 559       resources {
 560         cpu    = ${cpu}
 561         memory = ${memory}
 562       }
 563     }
 564     task "${job_name}-iterative-soak" {
 565       artifact {
 566         source      = "git::https://github.com/FDio/csit"
 567         destination = "local/csit"
 568       }
 569       driver = "docker"
 570       config {
 571         image   = "${image}"
 572         command = "gluesparksubmit"
 573         args = [
 574           "--driver-memory", "20g",
 575           "--executor-memory", "20g",
 576           "--executor-cores", "2",
 577           "--master", "local[2]",
 578           "iterative_soak_rls2302.py"
 579         ]
 580         work_dir = "/local/csit/csit.infra.etl"
 581       }
 582       env {
 583         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 584         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 585         AWS_DEFAULT_REGION        = "${aws_default_region}"
 586         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 587         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 588         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 589         ${ envs }
 590       }
 591       resources {
 592         cpu    = ${cpu}
 593         memory = ${memory}
 594       }
 595     }
 596   }
 597 }