fix(etl): Jobs

Signed-off-by: Peter Mikus <[email protected]>
Change-Id: If530cb1fe1c415f6988508281f3eea22e1d5fa69

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl
index cba9954..96fef71 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "coverage_device_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl
index 34518d3..974e915 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "coverage_hoststack_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl
index e5defb6..c2c2827 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "coverage_mrr_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl
index 49569f2..ee4771b 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "coverage_ndrpdr_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl
index ad973cc..d31ebd8 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "coverage_reconf_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl
index f142632..9d71dd8 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "coverage_soak_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl
index 4d7217b..79397ed 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "iterative_hoststack_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl
index ed0d1c0..c1c298d 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "iterative_mrr_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl
index 5df38a2..42202f3 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "iterative_ndrpdr_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl
index 4337ef1..4a7773c 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "iterative_reconf_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl
index fb8db8c..3759f4b 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak_rls2410.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "iterative_soak_rls2410.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"