fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl

   1 job "${job_name}" {
   2   datacenters = "${datacenters}"
   3   type        = "${type}"
   4   periodic {
   5     cron             = "${cron}"
   6     prohibit_overlap = "${prohibit_overlap}"
   7     time_zone        = "${time_zone}"
   8   }
   9   group "${job_name}-master" {
  10     restart {
  11       mode = "fail"
  12     }
  13     constraint {
  14       attribute       = "$${attr.cpu.arch}"
  15       operator        = "!="
  16       value           = "arm64"
  17     }
  18     constraint {
  19       attribute      = "$${node.class}"
  20       value          = "builder"
  21     }
  22     task "${job_name}-trending-mrr" {
  23       artifact {
  24         source      = "git::https://github.com/FDio/csit"
  25         destination = "local/csit"
  26       }
  27       driver = "docker"
  28       config {
  29         image   = "${image}"
  30         command = "gluesparksubmit"
  31         args = [
  32           "--driver-memory", "30g",
  33           "--executor-memory", "30g",
  34           "trending_mrr.py"
  35         ]
  36         work_dir = "/local/csit/csit.infra.etl"
  37       }
  38       env {
  39         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  40         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  41         AWS_DEFAULT_REGION        = "${aws_default_region}"
  42         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  43         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  44         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  45         ${ envs }
  46       }
  47       resources {
  48         cpu    = ${cpu}
  49         memory = ${memory}
  50       }
  51     }
  52     task "${job_name}-trending-ndrpdr" {
  53       artifact {
  54         source      = "git::https://github.com/FDio/csit"
  55         destination = "local/csit"
  56       }
  57       driver = "docker"
  58       config {
  59         image   = "${image}"
  60         command = "gluesparksubmit"
  61         args = [
  62           "--driver-memory", "30g",
  63           "--executor-memory", "30g",
  64           "trending_ndrpdr.py"
  65         ]
  66         work_dir = "/local/csit/csit.infra.etl"
  67       }
  68       env {
  69         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
  70         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
  71         AWS_DEFAULT_REGION        = "${aws_default_region}"
  72         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
  73         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
  74         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
  75         ${ envs }
  76       }
  77       resources {
  78         cpu    = ${cpu}
  79         memory = ${memory}
  80       }
  81     }
  82     task "${job_name}-trending-soak" {
  83       artifact {
  84         source      = "git::https://github.com/FDio/csit"
  85         destination = "local/csit"
  86       }
  87       driver = "docker"
  88       config {
  89         image   = "${image}"
  90         command = "gluesparksubmit"
  91         args = [
  92           "--driver-memory", "30g",
  93           "--executor-memory", "30g",
  94           "trending_soak.py"
  95         ]
  96         work_dir = "/local/csit/csit.infra.etl"
  97       }
  98       env {
  99         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 100         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 101         AWS_DEFAULT_REGION        = "${aws_default_region}"
 102         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 103         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 104         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 105         ${ envs }
 106       }
 107       resources {
 108         cpu    = ${cpu}
 109         memory = ${memory}
 110       }
 111     }
 112     task "${job_name}-stats" {
 113       artifact {
 114         source      = "git::https://github.com/FDio/csit"
 115         destination = "local/csit"
 116       }
 117       driver = "docker"
 118       config {
 119         image   = "${image}"
 120         command = "gluesparksubmit"
 121         args = [
 122           "--driver-memory", "10g",
 123           "--executor-memory", "10g",
 124           "stats.py"
 125         ]
 126         work_dir = "/local/csit/csit.infra.etl"
 127       }
 128       env {
 129         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 130         AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 131         AWS_DEFAULT_REGION        = "${aws_default_region}"
 132         OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 133         OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 134         OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 135         ${ envs }
 136       }
 137       resources {
 138         cpu    = ${cpu}
 139         memory = ${memory}
 140       }
 141     }
 142   }
 143 #  group "${job_name}-rls2210" {
 144 #    restart {
 145 #      mode = "fail"
 146 #    }
 147 #    constraint {
 148 #      attribute = "$${attr.cpu.arch}"
 149 #      operator  = "!="
 150 #      value     = "arm64"
 151 #    }
 152 #    constraint {
 153 #      attribute = "$${node.class}"
 154 #      value     = "builder"
 155 #    }
 156 #    task "${job_name}-coverage-device" {
 157 #      artifact {
 158 #        source      = "git::https://github.com/FDio/csit"
 159 #        destination = "local/csit"
 160 #      }
 161 #      driver = "docker"
 162 #      config {
 163 #        image   = "${image}"
 164 #        command = "gluesparksubmit"
 165 #        args = [
 166 #          "--driver-memory", "20g",
 167 #          "--executor-memory", "20g",
 168 #          "--executor-cores", "2",
 169 #          "--master", "local[2]",
 170 #          "coverage_device_rls2210.py"
 171 #        ]
 172 #        work_dir = "/local/csit/csit.infra.etl"
 173 #      }
 174 #      env {
 175 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 176 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 177 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 178 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 179 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 180 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 181 #        ${ envs }
 182 #      }
 183 #      resources {
 184 #        cpu    = ${cpu}
 185 #        memory = ${memory}
 186 #      }
 187 #    }
 188 #    task "${job_name}-coverage-mrr" {
 189 #      artifact {
 190 #        source      = "git::https://github.com/FDio/csit"
 191 #        destination = "local/csit"
 192 #      }
 193 #      driver = "docker"
 194 #      config {
 195 #        image   = "${image}"
 196 #        command = "gluesparksubmit"
 197 #        args = [
 198 #          "--driver-memory", "20g",
 199 #          "--executor-memory", "20g",
 200 #          "--executor-cores", "2",
 201 #          "--master", "local[2]",
 202 #          "coverage_mrr_rls2210.py"
 203 #        ]
 204 #        work_dir = "/local/csit/csit.infra.etl"
 205 #      }
 206 #      env {
 207 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 208 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 209 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 210 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 211 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 212 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 213 #        ${ envs }
 214 #      }
 215 #      resources {
 216 #        cpu    = ${cpu}
 217 #        memory = ${memory}
 218 #      }
 219 #    }
 220 #    task "${job_name}-coverage-ndrpdr" {
 221 #      artifact {
 222 #        source      = "git::https://github.com/FDio/csit"
 223 #        destination = "local/csit"
 224 #      }
 225 #      driver = "docker"
 226 #      config {
 227 #        image   = "${image}"
 228 #        command = "gluesparksubmit"
 229 #        args = [
 230 #          "--driver-memory", "20g",
 231 #          "--executor-memory", "20g",
 232 #          "--executor-cores", "2",
 233 #          "--master", "local[2]",
 234 #          "coverage_ndrpdr_rls2210.py"
 235 #        ]
 236 #        work_dir = "/local/csit/csit.infra.etl"
 237 #      }
 238 #      env {
 239 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 240 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 241 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 242 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 243 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 244 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 245 #        ${ envs }
 246 #      }
 247 #      resources {
 248 #        cpu    = ${cpu}
 249 #        memory = ${memory}
 250 #      }
 251 #    }
 252 #    task "${job_name}-coverage-soak" {
 253 #      artifact {
 254 #        source      = "git::https://github.com/FDio/csit"
 255 #        destination = "local/csit"
 256 #      }
 257 #      driver = "docker"
 258 #      config {
 259 #        image   = "${image}"
 260 #        command = "gluesparksubmit"
 261 #        args = [
 262 #          "--driver-memory", "20g",
 263 #          "--executor-memory", "20g",
 264 #          "--executor-cores", "2",
 265 #          "--master", "local[2]",
 266 #          "coverage_soak_rls2210.py"
 267 #        ]
 268 #        work_dir = "/local/csit/csit.infra.etl"
 269 #      }
 270 #      env {
 271 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 272 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 273 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 274 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 275 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 276 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 277 #        ${ envs }
 278 #      }
 279 #      resources {
 280 #        cpu    = ${cpu}
 281 #        memory = ${memory}
 282 #      }
 283 #    }
 284 #    task "${job_name}-iterative-mrr" {
 285 #      artifact {
 286 #        source      = "git::https://github.com/FDio/csit"
 287 #        destination = "local/csit"
 288 #      }
 289 #      driver = "docker"
 290 #      config {
 291 #        image   = "${image}"
 292 #        command = "gluesparksubmit"
 293 #        args = [
 294 #          "--driver-memory", "20g",
 295 #          "--executor-memory", "20g",
 296 #          "--executor-cores", "2",
 297 #          "--master", "local[2]",
 298 #          "iterative_mrr_rls2210.py"
 299 #        ]
 300 #        work_dir = "/local/csit/csit.infra.etl"
 301 #      }
 302 #      env {
 303 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 304 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 305 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 306 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 307 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 308 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 309 #        ${ envs }
 310 #      }
 311 #      resources {
 312 #        cpu    = ${cpu}
 313 #        memory = ${memory}
 314 #      }
 315 #    }
 316 #    task "${job_name}-iterative-ndrpdr" {
 317 #      artifact {
 318 #        source      = "git::https://github.com/FDio/csit"
 319 #        destination = "local/csit"
 320 #      }
 321 #      driver = "docker"
 322 #      config {
 323 #        image   = "${image}"
 324 #        command = "gluesparksubmit"
 325 #        args = [
 326 #          "--driver-memory", "20g",
 327 #          "--executor-memory", "20g",
 328 #          "--executor-cores", "2",
 329 #          "--master", "local[2]",
 330 #          "iterative_ndrpdr_rls2210.py"
 331 #        ]
 332 #        work_dir = "/local/csit/csit.infra.etl"
 333 #      }
 334 #      env {
 335 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 336 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 337 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 338 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 339 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 340 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 341 #        ${ envs }
 342 #      }
 343 #      resources {
 344 #        cpu    = ${cpu}
 345 #        memory = ${memory}
 346 #      }
 347 #    }
 348 #    task "${job_name}-iterative-soak" {
 349 #      artifact {
 350 #        source      = "git::https://github.com/FDio/csit"
 351 #        destination = "local/csit"
 352 #      }
 353 #      driver = "docker"
 354 #      config {
 355 #        image   = "${image}"
 356 #        command = "gluesparksubmit"
 357 #        args = [
 358 #          "--driver-memory", "20g",
 359 #          "--executor-memory", "20g",
 360 #          "--executor-cores", "2",
 361 #          "--master", "local[2]",
 362 #          "iterative_soak_rls2210.py"
 363 #        ]
 364 #        work_dir = "/local/csit/csit.infra.etl"
 365 #      }
 366 #      env {
 367 #        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
 368 #        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
 369 #        AWS_DEFAULT_REGION        = "${aws_default_region}"
 370 #        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
 371 #        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
 372 #        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
 373 #        ${ envs }
 374 #      }
 375 #      resources {
 376 #        cpu    = ${cpu}
 377 #        memory = ${memory}
 378 #      }
 379 #    }
 380 #  }
 381 }