feat(etl): Add telemetry, remove release

Signed-off-by: pmikus <peter.mikus@protonmail.ch>
Change-Id: I26471a041c83ffc5d221b2b6d69b4443a40f3de8

diff --git a/csit.infra.etl/trending_mrr.json b/csit.infra.etl/trending_mrr.json
index 4e222d3..d3d7746 100644 (file)
--- a/csit.infra.etl/trending_mrr.json
+++ b/csit.infra.etl/trending_mrr.json
@@ -128,6 +128,16 @@
             "nullable": true,
             "type": "boolean"
         },
+        {
+            "metadata": {},
+            "name": "telemetry",
+            "nullable": true,
+            "type": {
+                "containsNull": true,
+                "elementType": "string",
+                "type": "array"
+            }
+        },
         {
             "metadata": {},
             "name": "test_id",

diff --git a/csit.infra.etl/trending_ndrpdr.json b/csit.infra.etl/trending_ndrpdr.json
index fd833aa..22cd505 100644 (file)
--- a/csit.infra.etl/trending_ndrpdr.json
+++ b/csit.infra.etl/trending_ndrpdr.json
@@ -656,6 +656,16 @@
             "nullable": true,
             "type": "boolean"
         },
+        {
+            "metadata": {},
+            "name": "telemetry",
+            "nullable": true,
+            "type": {
+                "containsNull": true,
+                "elementType": "string",
+                "type": "array"
+            }
+        },
         {
             "metadata": {},
             "name": "test_id",

diff --git a/csit.infra.etl/trending_soak.json b/csit.infra.etl/trending_soak.json
index 819d314..1aba814 100644 (file)
--- a/csit.infra.etl/trending_soak.json
+++ b/csit.infra.etl/trending_soak.json
@@ -198,6 +198,16 @@
             "nullable": true,
             "type": "boolean"
         },
+        {
+            "metadata": {},
+            "name": "telemetry",
+            "nullable": true,
+            "type": {
+                "containsNull": true,
+                "elementType": "string",
+                "type": "array"
+            }
+        },
         {
             "metadata": {},
             "name": "test_id",

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl
index 0abb0e5..1fb2f22 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl
@@ -140,242 +140,242 @@ job "${job_name}" {
       }
     }
   }
-  group "${job_name}-rls2210" {
-    restart {
-      mode = "fail"
-    }
-    constraint {
-      attribute = "$${attr.cpu.arch}"
-      operator  = "!="
-      value     = "arm64"
-    }
-    constraint {
-      attribute = "$${node.class}"
-      value     = "builder"
-    }
-    task "${job_name}-coverage-device" {
-      artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
-      }
-      driver = "docker"
-      config {
-        image   = "${image}"
-        command = "gluesparksubmit"
-        args = [
-          "--driver-memory", "20g",
-          "--executor-memory", "20g",
-          "--executor-cores", "2",
-          "--master", "local[2]",
-          "coverage_device_rls2210.py"
-        ]
-        work_dir = "/local/csit/csit.infra.etl"
-      }
-      env {
-        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
-        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
-        AWS_DEFAULT_REGION        = "${aws_default_region}"
-        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
-        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
-        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
-        ${ envs }
-      }
-      resources {
-        cpu    = ${cpu}
-        memory = ${memory}
-      }
-    }
-    task "${job_name}-coverage-mrr" {
-      artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
-      }
-      driver = "docker"
-      config {
-        image   = "${image}"
-        command = "gluesparksubmit"
-        args = [
-          "--driver-memory", "20g",
-          "--executor-memory", "20g",
-          "--executor-cores", "2",
-          "--master", "local[2]",
-          "coverage_mrr_rls2210.py"
-        ]
-        work_dir = "/local/csit/csit.infra.etl"
-      }
-      env {
-        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
-        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
-        AWS_DEFAULT_REGION        = "${aws_default_region}"
-        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
-        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
-        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
-        ${ envs }
-      }
-      resources {
-        cpu    = ${cpu}
-        memory = ${memory}
-      }
-    }
-    task "${job_name}-coverage-ndrpdr" {
-      artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
-      }
-      driver = "docker"
-      config {
-        image   = "${image}"
-        command = "gluesparksubmit"
-        args = [
-          "--driver-memory", "20g",
-          "--executor-memory", "20g",
-          "--executor-cores", "2",
-          "--master", "local[2]",
-          "coverage_ndrpdr_rls2210.py"
-        ]
-        work_dir = "/local/csit/csit.infra.etl"
-      }
-      env {
-        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
-        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
-        AWS_DEFAULT_REGION        = "${aws_default_region}"
-        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
-        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
-        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
-        ${ envs }
-      }
-      resources {
-        cpu    = ${cpu}
-        memory = ${memory}
-      }
-    }
-    task "${job_name}-coverage-soak" {
-      artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
-      }
-      driver = "docker"
-      config {
-        image   = "${image}"
-        command = "gluesparksubmit"
-        args = [
-          "--driver-memory", "20g",
-          "--executor-memory", "20g",
-          "--executor-cores", "2",
-          "--master", "local[2]",
-          "coverage_soak_rls2210.py"
-        ]
-        work_dir = "/local/csit/csit.infra.etl"
-      }
-      env {
-        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
-        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
-        AWS_DEFAULT_REGION        = "${aws_default_region}"
-        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
-        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
-        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
-        ${ envs }
-      }
-      resources {
-        cpu    = ${cpu}
-        memory = ${memory}
-      }
-    }
-    task "${job_name}-iterative-mrr" {
-      artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
-      }
-      driver = "docker"
-      config {
-        image   = "${image}"
-        command = "gluesparksubmit"
-        args = [
-          "--driver-memory", "20g",
-          "--executor-memory", "20g",
-          "--executor-cores", "2",
-          "--master", "local[2]",
-          "iterative_mrr_rls2210.py"
-        ]
-        work_dir = "/local/csit/csit.infra.etl"
-      }
-      env {
-        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
-        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
-        AWS_DEFAULT_REGION        = "${aws_default_region}"
-        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
-        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
-        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
-        ${ envs }
-      }
-      resources {
-        cpu    = ${cpu}
-        memory = ${memory}
-      }
-    }
-    task "${job_name}-iterative-ndrpdr" {
-      artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
-      }
-      driver = "docker"
-      config {
-        image   = "${image}"
-        command = "gluesparksubmit"
-        args = [
-          "--driver-memory", "20g",
-          "--executor-memory", "20g",
-          "--executor-cores", "2",
-          "--master", "local[2]",
-          "iterative_ndrpdr_rls2210.py"
-        ]
-        work_dir = "/local/csit/csit.infra.etl"
-      }
-      env {
-        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
-        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
-        AWS_DEFAULT_REGION        = "${aws_default_region}"
-        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
-        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
-        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
-        ${ envs }
-      }
-      resources {
-        cpu    = ${cpu}
-        memory = ${memory}
-      }
-    }
-    task "${job_name}-iterative-soak" {
-      artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
-      }
-      driver = "docker"
-      config {
-        image   = "${image}"
-        command = "gluesparksubmit"
-        args = [
-          "--driver-memory", "20g",
-          "--executor-memory", "20g",
-          "--executor-cores", "2",
-          "--master", "local[2]",
-          "iterative_soak_rls2210.py"
-        ]
-        work_dir = "/local/csit/csit.infra.etl"
-      }
-      env {
-        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
-        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
-        AWS_DEFAULT_REGION        = "${aws_default_region}"
-        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
-        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
-        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
-        ${ envs }
-      }
-      resources {
-        cpu    = ${cpu}
-        memory = ${memory}
-      }
-    }
-  }
+#  group "${job_name}-rls2210" {
+#    restart {
+#      mode = "fail"
+#    }
+#    constraint {
+#      attribute = "$${attr.cpu.arch}"
+#      operator  = "!="
+#      value     = "arm64"
+#    }
+#    constraint {
+#      attribute = "$${node.class}"
+#      value     = "builder"
+#    }
+#    task "${job_name}-coverage-device" {
+#      artifact {
+#        source      = "git::https://github.com/FDio/csit"
+#        destination = "local/csit"
+#      }
+#      driver = "docker"
+#      config {
+#        image   = "${image}"
+#        command = "gluesparksubmit"
+#        args = [
+#          "--driver-memory", "20g",
+#          "--executor-memory", "20g",
+#          "--executor-cores", "2",
+#          "--master", "local[2]",
+#          "coverage_device_rls2210.py"
+#        ]
+#        work_dir = "/local/csit/csit.infra.etl"
+#      }
+#      env {
+#        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
+#        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
+#        AWS_DEFAULT_REGION        = "${aws_default_region}"
+#        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
+#        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+#        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
+#        ${ envs }
+#      }
+#      resources {
+#        cpu    = ${cpu}
+#        memory = ${memory}
+#      }
+#    }
+#    task "${job_name}-coverage-mrr" {
+#      artifact {
+#        source      = "git::https://github.com/FDio/csit"
+#        destination = "local/csit"
+#      }
+#      driver = "docker"
+#      config {
+#        image   = "${image}"
+#        command = "gluesparksubmit"
+#        args = [
+#          "--driver-memory", "20g",
+#          "--executor-memory", "20g",
+#          "--executor-cores", "2",
+#          "--master", "local[2]",
+#          "coverage_mrr_rls2210.py"
+#        ]
+#        work_dir = "/local/csit/csit.infra.etl"
+#      }
+#      env {
+#        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
+#        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
+#        AWS_DEFAULT_REGION        = "${aws_default_region}"
+#        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
+#        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+#        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
+#        ${ envs }
+#      }
+#      resources {
+#        cpu    = ${cpu}
+#        memory = ${memory}
+#      }
+#    }
+#    task "${job_name}-coverage-ndrpdr" {
+#      artifact {
+#        source      = "git::https://github.com/FDio/csit"
+#        destination = "local/csit"
+#      }
+#      driver = "docker"
+#      config {
+#        image   = "${image}"
+#        command = "gluesparksubmit"
+#        args = [
+#          "--driver-memory", "20g",
+#          "--executor-memory", "20g",
+#          "--executor-cores", "2",
+#          "--master", "local[2]",
+#          "coverage_ndrpdr_rls2210.py"
+#        ]
+#        work_dir = "/local/csit/csit.infra.etl"
+#      }
+#      env {
+#        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
+#        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
+#        AWS_DEFAULT_REGION        = "${aws_default_region}"
+#        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
+#        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+#        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
+#        ${ envs }
+#      }
+#      resources {
+#        cpu    = ${cpu}
+#        memory = ${memory}
+#      }
+#    }
+#    task "${job_name}-coverage-soak" {
+#      artifact {
+#        source      = "git::https://github.com/FDio/csit"
+#        destination = "local/csit"
+#      }
+#      driver = "docker"
+#      config {
+#        image   = "${image}"
+#        command = "gluesparksubmit"
+#        args = [
+#          "--driver-memory", "20g",
+#          "--executor-memory", "20g",
+#          "--executor-cores", "2",
+#          "--master", "local[2]",
+#          "coverage_soak_rls2210.py"
+#        ]
+#        work_dir = "/local/csit/csit.infra.etl"
+#      }
+#      env {
+#        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
+#        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
+#        AWS_DEFAULT_REGION        = "${aws_default_region}"
+#        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
+#        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+#        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
+#        ${ envs }
+#      }
+#      resources {
+#        cpu    = ${cpu}
+#        memory = ${memory}
+#      }
+#    }
+#    task "${job_name}-iterative-mrr" {
+#      artifact {
+#        source      = "git::https://github.com/FDio/csit"
+#        destination = "local/csit"
+#      }
+#      driver = "docker"
+#      config {
+#        image   = "${image}"
+#        command = "gluesparksubmit"
+#        args = [
+#          "--driver-memory", "20g",
+#          "--executor-memory", "20g",
+#          "--executor-cores", "2",
+#          "--master", "local[2]",
+#          "iterative_mrr_rls2210.py"
+#        ]
+#        work_dir = "/local/csit/csit.infra.etl"
+#      }
+#      env {
+#        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
+#        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
+#        AWS_DEFAULT_REGION        = "${aws_default_region}"
+#        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
+#        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+#        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
+#        ${ envs }
+#      }
+#      resources {
+#        cpu    = ${cpu}
+#        memory = ${memory}
+#      }
+#    }
+#    task "${job_name}-iterative-ndrpdr" {
+#      artifact {
+#        source      = "git::https://github.com/FDio/csit"
+#        destination = "local/csit"
+#      }
+#      driver = "docker"
+#      config {
+#        image   = "${image}"
+#        command = "gluesparksubmit"
+#        args = [
+#          "--driver-memory", "20g",
+#          "--executor-memory", "20g",
+#          "--executor-cores", "2",
+#          "--master", "local[2]",
+#          "iterative_ndrpdr_rls2210.py"
+#        ]
+#        work_dir = "/local/csit/csit.infra.etl"
+#      }
+#      env {
+#        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
+#        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
+#        AWS_DEFAULT_REGION        = "${aws_default_region}"
+#        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
+#        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+#        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
+#        ${ envs }
+#      }
+#      resources {
+#        cpu    = ${cpu}
+#        memory = ${memory}
+#      }
+#    }
+#    task "${job_name}-iterative-soak" {
+#      artifact {
+#        source      = "git::https://github.com/FDio/csit"
+#        destination = "local/csit"
+#      }
+#      driver = "docker"
+#      config {
+#        image   = "${image}"
+#        command = "gluesparksubmit"
+#        args = [
+#          "--driver-memory", "20g",
+#          "--executor-memory", "20g",
+#          "--executor-cores", "2",
+#          "--master", "local[2]",
+#          "iterative_soak_rls2210.py"
+#        ]
+#        work_dir = "/local/csit/csit.infra.etl"
+#      }
+#      env {
+#        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
+#        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
+#        AWS_DEFAULT_REGION        = "${aws_default_region}"
+#        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
+#        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+#        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
+#        ${ envs }
+#      }
+#      resources {
+#        cpu    = ${cpu}
+#        memory = ${memory}
+#      }
+#    }
+#  }
 }