fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl

   1 job "${job_name}" {
   2   datacenters = "${datacenters}"
   3   type        = "${type}"
   4   periodic {
   5     cron             = "${cron}"
   6     prohibit_overlap = "${prohibit_overlap}"
   7     time_zone        = "${time_zone}"
   8   }
   9   group "${job_name}-master" {
  10     restart {
  11       mode = "fail"
  12     }
  13     constraint {
  14       attribute       = "$${attr.cpu.arch}"
  15       operator        = "!="
  16       value           = "arm64"
  17     }
  18     constraint {
  19       attribute      = "$${node.class}"
  20       value          = "builder"
  21     }
  22     task "${job_name}-trending-mrr" {
  23       artifact {
  24         source      = "git::https://github.com/FDio/csit"
  25         destination = "local/csit"
  26       }
  27       driver = "docker"
  28       config {
  29         image   = "${image}"
  30         command = "gluesparksubmit"
  31         args = [
  32           "--driver-memory", "30g",
  33           "--executor-memory", "30g",
  34           "trending_mrr.py"
  35         ]
  36         work_dir = "/local/csit/csit.infra.etl"
  37       }
  38       env {
  39         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  40         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  41         AWS_DEFAULT_REGION        = "${aws_default_region}"
  42         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  43         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  44         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  45         ${ envs }
  46       }
  47       resources {
  48         cpu    = ${cpu}
  49         memory = ${memory}
  50       }
  51     }
  52     task "${job_name}-trending-ndrpdr" {
  53       artifact {
  54         source      = "git::https://github.com/FDio/csit"
  55         destination = "local/csit"
  56       }
  57       driver = "docker"
  58       config {
  59         image   = "${image}"
  60         command = "gluesparksubmit"
  61         args = [
  62           "--driver-memory", "30g",
  63           "--executor-memory", "30g",
  64           "trending_ndrpdr.py"
  65         ]
  66         work_dir = "/local/csit/csit.infra.etl"
  67       }
  68       env {
  69         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  70         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  71         AWS_DEFAULT_REGION        = "${aws_default_region}"
  72         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  73         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  74         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  75         ${ envs }
  76       }
  77       resources {
  78         cpu    = ${cpu}
  79         memory = ${memory}
  80       }
  81     }
  82     task "${job_name}-trending-soak" {
  83       artifact {
  84         source      = "git::https://github.com/FDio/csit"
  85         destination = "local/csit"
  86       }
  87       driver = "docker"
  88       config {
  89         image   = "${image}"
  90         command = "gluesparksubmit"
  91         args = [
  92           "--driver-memory", "30g",
  93           "--executor-memory", "30g",
  94           "trending_soak.py"
  95         ]
  96         work_dir = "/local/csit/csit.infra.etl"
  97       }
  98       env {
  99         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 100         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 101         AWS_DEFAULT_REGION        = "${aws_default_region}"
 102         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 103         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 104         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 105         ${ envs }
 106       }
 107       resources {
 108         cpu    = ${cpu}
 109         memory = ${memory}
 110       }
 111     }
 112     task "${job_name}-trending-reconf" {
 113       artifact {
 114         source      = "git::https://github.com/FDio/csit"
 115         destination = "local/csit"
 116       }
 117       driver = "docker"
 118       config {
 119         image   = "${image}"
 120         command = "gluesparksubmit"
 121         args = [
 122           "--driver-memory", "30g",
 123           "--executor-memory", "30g",
 124           "trending_reconf.py"
 125         ]
 126         work_dir = "/local/csit/csit.infra.etl"
 127       }
 128       env {
 129         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 130         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 131         AWS_DEFAULT_REGION        = "${aws_default_region}"
 132         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 133         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 134         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 135         ${ envs }
 136       }
 137       resources {
 138         cpu    = ${cpu}
 139         memory = ${memory}
 140       }
 141     }
 142     task "${job_name}-stats" {
 143       artifact {
 144         source      = "git::https://github.com/FDio/csit"
 145         destination = "local/csit"
 146       }
 147       driver = "docker"
 148       config {
 149         image   = "${image}"
 150         command = "gluesparksubmit"
 151         args = [
 152           "--driver-memory", "10g",
 153           "--executor-memory", "10g",
 154           "stats.py"
 155         ]
 156         work_dir = "/local/csit/csit.infra.etl"
 157       }
 158       env {
 159         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 160         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 161         AWS_DEFAULT_REGION        = "${aws_default_region}"
 162         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 163         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 164         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 165         ${ envs }
 166       }
 167       resources {
 168         cpu    = ${cpu}
 169         memory = ${memory}
 170       }
 171     }
 172   }
 173 #  group "${job_name}-rls2210" {
 174 #    restart {
 175 #      mode = "fail"
 176 #    }
 177 #    constraint {
 178 #      attribute = "$${attr.cpu.arch}"
 179 #      operator  = "!="
 180 #      value     = "arm64"
 181 #    }
 182 #    constraint {
 183 #      attribute = "$${node.class}"
 184 #      value     = "builder"
 185 #    }
 186 #    task "${job_name}-coverage-device" {
 187 #      artifact {
 188 #        source      = "git::https://github.com/FDio/csit"
 189 #        destination = "local/csit"
 190 #      }
 191 #      driver = "docker"
 192 #      config {
 193 #        image   = "${image}"
 194 #        command = "gluesparksubmit"
 195 #        args = [
 196 #          "--driver-memory", "20g",
 197 #          "--executor-memory", "20g",
 198 #          "--executor-cores", "2",
 199 #          "--master", "local[2]",
 200 #          "coverage_device_rls2210.py"
 201 #        ]
 202 #        work_dir = "/local/csit/csit.infra.etl"
 203 #      }
 204 #      env {
 205 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 206 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 207 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 208 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 209 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 210 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 211 #        ${ envs }
 212 #      }
 213 #      resources {
 214 #        cpu    = ${cpu}
 215 #        memory = ${memory}
 216 #      }
 217 #    }
 218 #    task "${job_name}-coverage-mrr" {
 219 #      artifact {
 220 #        source      = "git::https://github.com/FDio/csit"
 221 #        destination = "local/csit"
 222 #      }
 223 #      driver = "docker"
 224 #      config {
 225 #        image   = "${image}"
 226 #        command = "gluesparksubmit"
 227 #        args = [
 228 #          "--driver-memory", "20g",
 229 #          "--executor-memory", "20g",
 230 #          "--executor-cores", "2",
 231 #          "--master", "local[2]",
 232 #          "coverage_mrr_rls2210.py"
 233 #        ]
 234 #        work_dir = "/local/csit/csit.infra.etl"
 235 #      }
 236 #      env {
 237 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 238 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 239 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 240 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 241 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 242 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 243 #        ${ envs }
 244 #      }
 245 #      resources {
 246 #        cpu    = ${cpu}
 247 #        memory = ${memory}
 248 #      }
 249 #    }
 250 #    task "${job_name}-coverage-ndrpdr" {
 251 #      artifact {
 252 #        source      = "git::https://github.com/FDio/csit"
 253 #        destination = "local/csit"
 254 #      }
 255 #      driver = "docker"
 256 #      config {
 257 #        image   = "${image}"
 258 #        command = "gluesparksubmit"
 259 #        args = [
 260 #          "--driver-memory", "20g",
 261 #          "--executor-memory", "20g",
 262 #          "--executor-cores", "2",
 263 #          "--master", "local[2]",
 264 #          "coverage_ndrpdr_rls2210.py"
 265 #        ]
 266 #        work_dir = "/local/csit/csit.infra.etl"
 267 #      }
 268 #      env {
 269 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 270 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 271 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 272 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 273 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 274 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 275 #        ${ envs }
 276 #      }
 277 #      resources {
 278 #        cpu    = ${cpu}
 279 #        memory = ${memory}
 280 #      }
 281 #    }
 282 #    task "${job_name}-coverage-soak" {
 283 #      artifact {
 284 #        source      = "git::https://github.com/FDio/csit"
 285 #        destination = "local/csit"
 286 #      }
 287 #      driver = "docker"
 288 #      config {
 289 #        image   = "${image}"
 290 #        command = "gluesparksubmit"
 291 #        args = [
 292 #          "--driver-memory", "20g",
 293 #          "--executor-memory", "20g",
 294 #          "--executor-cores", "2",
 295 #          "--master", "local[2]",
 296 #          "coverage_soak_rls2210.py"
 297 #        ]
 298 #        work_dir = "/local/csit/csit.infra.etl"
 299 #      }
 300 #      env {
 301 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 302 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 303 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 304 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 305 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 306 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 307 #        ${ envs }
 308 #      }
 309 #      resources {
 310 #        cpu    = ${cpu}
 311 #        memory = ${memory}
 312 #      }
 313 #    }
 314 #    task "${job_name}-iterative-mrr" {
 315 #      artifact {
 316 #        source      = "git::https://github.com/FDio/csit"
 317 #        destination = "local/csit"
 318 #      }
 319 #      driver = "docker"
 320 #      config {
 321 #        image   = "${image}"
 322 #        command = "gluesparksubmit"
 323 #        args = [
 324 #          "--driver-memory", "20g",
 325 #          "--executor-memory", "20g",
 326 #          "--executor-cores", "2",
 327 #          "--master", "local[2]",
 328 #          "iterative_mrr_rls2210.py"
 329 #        ]
 330 #        work_dir = "/local/csit/csit.infra.etl"
 331 #      }
 332 #      env {
 333 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 334 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 335 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 336 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 337 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 338 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 339 #        ${ envs }
 340 #      }
 341 #      resources {
 342 #        cpu    = ${cpu}
 343 #        memory = ${memory}
 344 #      }
 345 #    }
 346 #    task "${job_name}-iterative-ndrpdr" {
 347 #      artifact {
 348 #        source      = "git::https://github.com/FDio/csit"
 349 #        destination = "local/csit"
 350 #      }
 351 #      driver = "docker"
 352 #      config {
 353 #        image   = "${image}"
 354 #        command = "gluesparksubmit"
 355 #        args = [
 356 #          "--driver-memory", "20g",
 357 #          "--executor-memory", "20g",
 358 #          "--executor-cores", "2",
 359 #          "--master", "local[2]",
 360 #          "iterative_ndrpdr_rls2210.py"
 361 #        ]
 362 #        work_dir = "/local/csit/csit.infra.etl"
 363 #      }
 364 #      env {
 365 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 366 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 367 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 368 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 369 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 370 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 371 #        ${ envs }
 372 #      }
 373 #      resources {
 374 #        cpu    = ${cpu}
 375 #        memory = ${memory}
 376 #      }
 377 #    }
 378 #    task "${job_name}-iterative-soak" {
 379 #      artifact {
 380 #        source      = "git::https://github.com/FDio/csit"
 381 #        destination = "local/csit"
 382 #      }
 383 #      driver = "docker"
 384 #      config {
 385 #        image   = "${image}"
 386 #        command = "gluesparksubmit"
 387 #        args = [
 388 #          "--driver-memory", "20g",
 389 #          "--executor-memory", "20g",
 390 #          "--executor-cores", "2",
 391 #          "--master", "local[2]",
 392 #          "iterative_soak_rls2210.py"
 393 #        ]
 394 #        work_dir = "/local/csit/csit.infra.etl"
 395 #      }
 396 #      env {
 397 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 398 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 399 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 400 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 401 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 402 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 403 #        ${ envs }
 404 #      }
 405 #      resources {
 406 #        cpu    = ${cpu}
 407 #        memory = ${memory}
 408 #      }
 409 #    }
 410 #  }
 411 }