feat(etl): Add release 2506

Signed-off-by: Peter Mikus <peter.mikus@icloud.com>
Change-Id: Iaecddb1156ef899526b7228f9448983d1892a9c5

diff --git a/csit.infra.etl/coverage_hoststack.py b/csit.infra.etl/coverage_hoststack.py
index 559dd71..1b4c87e 100644 (file)
--- a/csit.infra.etl/coverage_hoststack.py
+++ b/csit.infra.etl/coverage_hoststack.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2502" in path]
+filtered_paths = [path for path in paths if "report-coverage-2506" in path]
 
 out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_mrr.py b/csit.infra.etl/coverage_mrr.py
index 5cbb982..c371229 100644 (file)
--- a/csit.infra.etl/coverage_mrr.py
+++ b/csit.infra.etl/coverage_mrr.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2502" in path]
+filtered_paths = [path for path in paths if "report-coverage-2506" in path]
 
 out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_ndrpdr.py b/csit.infra.etl/coverage_ndrpdr.py
index f3c9b63..8a4b0a9 100644 (file)
--- a/csit.infra.etl/coverage_ndrpdr.py
+++ b/csit.infra.etl/coverage_ndrpdr.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2502" in path]
+filtered_paths = [path for path in paths if "report-coverage-2506" in path]
 
 out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_reconf.py b/csit.infra.etl/coverage_reconf.py
index 307c50f..f24ffa9 100644 (file)
--- a/csit.infra.etl/coverage_reconf.py
+++ b/csit.infra.etl/coverage_reconf.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2502" in path]
+filtered_paths = [path for path in paths if "report-coverage-2506" in path]
 
 out_sdf = process_json_to_dataframe("reconf", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_soak.py b/csit.infra.etl/coverage_soak.py
index bb6e9fb..82b6f4f 100644 (file)
--- a/csit.infra.etl/coverage_soak.py
+++ b/csit.infra.etl/coverage_soak.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2502" in path]
+filtered_paths = [path for path in paths if "report-coverage-2506" in path]
 
 out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_hoststack.py b/csit.infra.etl/iterative_hoststack.py
index e0d9284..1a97962 100644 (file)
--- a/csit.infra.etl/iterative_hoststack.py
+++ b/csit.infra.etl/iterative_hoststack.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2502" in path]
+filtered_paths = [path for path in paths if "report-iterative-2506" in path]
 
 out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_mrr.py b/csit.infra.etl/iterative_mrr.py
index b1a1e00..c3c8ee3 100644 (file)
--- a/csit.infra.etl/iterative_mrr.py
+++ b/csit.infra.etl/iterative_mrr.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2502" in path]
+filtered_paths = [path for path in paths if "report-iterative-2506" in path]
 
 out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_ndrpdr.py b/csit.infra.etl/iterative_ndrpdr.py
index f86a32c..40d783d 100644 (file)
--- a/csit.infra.etl/iterative_ndrpdr.py
+++ b/csit.infra.etl/iterative_ndrpdr.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2502" in path]
+filtered_paths = [path for path in paths if "report-iterative-2506" in path]
 
 out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_reconf.py b/csit.infra.etl/iterative_reconf.py
index 68ee8f0..e5eb2ef 100644 (file)
--- a/csit.infra.etl/iterative_reconf.py
+++ b/csit.infra.etl/iterative_reconf.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2502" in path]
+filtered_paths = [path for path in paths if "report-iterative-2506" in path]
 
 out_sdf = process_json_to_dataframe("reconf", filtered_paths)
 out_sdf.show(truncate=False)
@@ -164,7 +164,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_soak.py b/csit.infra.etl/iterative_soak.py
index e213c31..5b77157 100644 (file)
--- a/csit.infra.etl/iterative_soak.py
+++ b/csit.infra.etl/iterative_soak.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2502" in path]
+filtered_paths = [path for path in paths if "report-iterative-2506" in path]
 
 out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
index dddb405..d0d9fa1 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
@@ -54,46 +54,46 @@ variable "nomad_jobs" {
       job_name = "etl-stats"
       memory = 50000
     },
-    #{
-    #  job_name = "etl-iterative-hoststack"
-    #  memory = 50000
-    #},
-    #{
-    #  job_name = "etl-iterative-mrr"
-    #  memory = 50000
-    #},
-    #{
-    #  job_name = "etl-iterative-ndrpdr"
-    #  memory = 50000
-    #},
-    #{
-    #  job_name = "etl-iterative-reconf"
-    #  memory = 50000
-    #},
-    #{
-    #  job_name = "etl-iterative-soak"
-    #  memory = 50000
-    #},
-    #{
-    #  job_name = "etl-coverage-hoststack"
-    #  memory = 50000
-    #},
-    #{
-    #  job_name = "etl-coverage-mrr"
-    #  memory = 50000
-    #},
-    #{
-    #  job_name = "etl-coverage-ndrpdr"
-    #  memory = 50000
-    #},
-    #{
-    #  job_name = "etl-coverage-reconf"
-    #  memory = 50000
-    #},
-    #{
-    #  job_name = "etl-coverage-soak"
-    #  memory = 50000
-    #},
+    {
+      job_name = "etl-iterative-hoststack"
+      memory = 50000
+    },
+    {
+      job_name = "etl-iterative-mrr"
+      memory = 50000
+    },
+    {
+      job_name = "etl-iterative-ndrpdr"
+      memory = 50000
+    },
+    {
+      job_name = "etl-iterative-reconf"
+      memory = 50000
+    },
+    {
+      job_name = "etl-iterative-soak"
+      memory = 50000
+    },
+    {
+      job_name = "etl-coverage-hoststack"
+      memory = 50000
+    },
+    {
+      job_name = "etl-coverage-mrr"
+      memory = 50000
+    },
+    {
+      job_name = "etl-coverage-ndrpdr"
+      memory = 50000
+    },
+    {
+      job_name = "etl-coverage-reconf"
+      memory = 50000
+    },
+    {
+      job_name = "etl-coverage-soak"
+      memory = 50000
+    },
     {
       job_name = "etl-trending-hoststack"
       memory = 50000