feat(etl): Pipeline enable

Signed-off-by: Peter Mikus <[email protected]>
Change-Id: I1bc7f71a95514d742c58484947c11d219a713be0

diff --git a/csit.infra.etl/coverage_hoststack.py b/csit.infra.etl/coverage_hoststack.py
index 1b4c87e..1c31027 100644 (file)
--- a/csit.infra.etl/coverage_hoststack.py
+++ b/csit.infra.etl/coverage_hoststack.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2506" in path]
+filtered_paths = [path for path in paths if "report-coverage-2510" in path]
 
 out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_mrr.py b/csit.infra.etl/coverage_mrr.py
index c371229..7db04d7 100644 (file)
--- a/csit.infra.etl/coverage_mrr.py
+++ b/csit.infra.etl/coverage_mrr.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2506" in path]
+filtered_paths = [path for path in paths if "report-coverage-2510" in path]
 
 out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_ndrpdr.py b/csit.infra.etl/coverage_ndrpdr.py
index 8a4b0a9..d0cda88 100644 (file)
--- a/csit.infra.etl/coverage_ndrpdr.py
+++ b/csit.infra.etl/coverage_ndrpdr.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2506" in path]
+filtered_paths = [path for path in paths if "report-coverage-2510" in path]
 
 out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_reconf.py b/csit.infra.etl/coverage_reconf.py
index f24ffa9..1452fce 100644 (file)
--- a/csit.infra.etl/coverage_reconf.py
+++ b/csit.infra.etl/coverage_reconf.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2506" in path]
+filtered_paths = [path for path in paths if "report-coverage-2510" in path]
 
 out_sdf = process_json_to_dataframe("reconf", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_soak.py b/csit.infra.etl/coverage_soak.py
index 82b6f4f..24c942b 100644 (file)
--- a/csit.infra.etl/coverage_soak.py
+++ b/csit.infra.etl/coverage_soak.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2506" in path]
+filtered_paths = [path for path in paths if "report-coverage-2510" in path]
 
 out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_hoststack.py b/csit.infra.etl/iterative_hoststack.py
index 1a97962..779122a 100644 (file)
--- a/csit.infra.etl/iterative_hoststack.py
+++ b/csit.infra.etl/iterative_hoststack.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2506" in path]
+filtered_paths = [path for path in paths if "report-iterative-2510" in path]
 
 out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_mrr.py b/csit.infra.etl/iterative_mrr.py
index c3c8ee3..cd47b27 100644 (file)
--- a/csit.infra.etl/iterative_mrr.py
+++ b/csit.infra.etl/iterative_mrr.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2506" in path]
+filtered_paths = [path for path in paths if "report-iterative-2510" in path]
 
 out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_ndrpdr.py b/csit.infra.etl/iterative_ndrpdr.py
index 40d783d..6e27358 100644 (file)
--- a/csit.infra.etl/iterative_ndrpdr.py
+++ b/csit.infra.etl/iterative_ndrpdr.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2506" in path]
+filtered_paths = [path for path in paths if "report-iterative-2510" in path]
 
 out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_reconf.py b/csit.infra.etl/iterative_reconf.py
index e5eb2ef..2e70983 100644 (file)
--- a/csit.infra.etl/iterative_reconf.py
+++ b/csit.infra.etl/iterative_reconf.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2506" in path]
+filtered_paths = [path for path in paths if "report-iterative-2510" in path]
 
 out_sdf = process_json_to_dataframe("reconf", filtered_paths)
 out_sdf.show(truncate=False)
@@ -164,7 +164,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_soak.py b/csit.infra.etl/iterative_soak.py
index 5b77157..de4c81f 100644 (file)
--- a/csit.infra.etl/iterative_soak.py
+++ b/csit.infra.etl/iterative_soak.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2506" in path]
+filtered_paths = [path for path in paths if "report-iterative-2510" in path]
 
 out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
@@ -163,7 +163,7 @@ except KeyError:
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2510",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf
index cf67cc2..44ed2a2 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf
@@ -7,11 +7,11 @@ terraform {
   required_providers {
     nomad = {
       source  = "hashicorp/nomad"
-      version = ">= 2.3.0"
+      version = ">= 2.5.0"
     }
     vault = {
       version = ">= 4.6.0"
     }
   }
-  required_version = ">= 1.10.4"
+  required_version = ">= 1.12.1"
 }