From f6591ba310c498ae6b19731dcce25293c6ca67db Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Tue, 20 May 2025 12:15:36 +0200 Subject: [PATCH] feat(etl): Add release 2506 Signed-off-by: Peter Mikus Change-Id: Iaecddb1156ef899526b7228f9448983d1892a9c5 --- csit.infra.etl/coverage_hoststack.py | 4 +- csit.infra.etl/coverage_mrr.py | 4 +- csit.infra.etl/coverage_ndrpdr.py | 4 +- csit.infra.etl/coverage_reconf.py | 4 +- csit.infra.etl/coverage_soak.py | 4 +- csit.infra.etl/iterative_hoststack.py | 4 +- csit.infra.etl/iterative_mrr.py | 4 +- csit.infra.etl/iterative_ndrpdr.py | 4 +- csit.infra.etl/iterative_reconf.py | 4 +- csit.infra.etl/iterative_soak.py | 4 +- .../terraform-nomad-pyspark-etl/variables.tf | 80 +++++++++++----------- 11 files changed, 60 insertions(+), 60 deletions(-) diff --git a/csit.infra.etl/coverage_hoststack.py b/csit.infra.etl/coverage_hoststack.py index 559dd716e9..1b4c87ef11 100644 --- a/csit.infra.etl/coverage_hoststack.py +++ b/csit.infra.etl/coverage_hoststack.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2502" in path] +filtered_paths = [path for path in paths if "report-coverage-2506" in path] out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.printSchema() @@ -163,7 +163,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_mrr.py b/csit.infra.etl/coverage_mrr.py index 5cbb982f87..c371229f4d 100644 --- a/csit.infra.etl/coverage_mrr.py +++ b/csit.infra.etl/coverage_mrr.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2502" in path] +filtered_paths = [path for path in paths if "report-coverage-2506" in path] out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() @@ -163,7 +163,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_ndrpdr.py b/csit.infra.etl/coverage_ndrpdr.py index f3c9b63f57..8a4b0a96b1 100644 --- a/csit.infra.etl/coverage_ndrpdr.py +++ b/csit.infra.etl/coverage_ndrpdr.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2502" in path] +filtered_paths = [path for path in paths if "report-coverage-2506" in path] out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() @@ -163,7 +163,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_reconf.py b/csit.infra.etl/coverage_reconf.py index 307c50f0a6..f24ffa998f 100644 --- a/csit.infra.etl/coverage_reconf.py +++ b/csit.infra.etl/coverage_reconf.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2502" in path] +filtered_paths = [path for path in paths if "report-coverage-2506" in path] out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.printSchema() @@ -163,7 +163,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_soak.py b/csit.infra.etl/coverage_soak.py index bb6e9fba6e..82b6f4fb60 100644 --- a/csit.infra.etl/coverage_soak.py +++ b/csit.infra.etl/coverage_soak.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2502" in path] +filtered_paths = [path for path in paths if "report-coverage-2506" in path] out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() @@ -163,7 +163,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_hoststack.py b/csit.infra.etl/iterative_hoststack.py index e0d9284d63..1a97962f58 100644 --- a/csit.infra.etl/iterative_hoststack.py +++ b/csit.infra.etl/iterative_hoststack.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2502" in path] +filtered_paths = [path for path in paths if "report-iterative-2506" in path] out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.printSchema() @@ -163,7 +163,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_mrr.py b/csit.infra.etl/iterative_mrr.py index b1a1e00e7a..c3c8ee3586 100644 --- a/csit.infra.etl/iterative_mrr.py +++ b/csit.infra.etl/iterative_mrr.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2502" in path] +filtered_paths = [path for path in paths if "report-iterative-2506" in path] out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() @@ -163,7 +163,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_ndrpdr.py b/csit.infra.etl/iterative_ndrpdr.py index f86a32c8d4..40d783d5db 100644 --- a/csit.infra.etl/iterative_ndrpdr.py +++ b/csit.infra.etl/iterative_ndrpdr.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2502" in path] +filtered_paths = [path for path in paths if "report-iterative-2506" in path] out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() @@ -163,7 +163,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_reconf.py b/csit.infra.etl/iterative_reconf.py index 68ee8f01ec..e5eb2efa71 100644 --- a/csit.infra.etl/iterative_reconf.py +++ b/csit.infra.etl/iterative_reconf.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2502" in path] +filtered_paths = [path for path in paths if "report-iterative-2506" in path] out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.show(truncate=False) @@ -164,7 +164,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_soak.py b/csit.infra.etl/iterative_soak.py index e213c31823..5b77157321 100644 --- a/csit.infra.etl/iterative_soak.py +++ b/csit.infra.etl/iterative_soak.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2502" in path] +filtered_paths = [path for path in paths if "report-iterative-2506" in path] out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() @@ -163,7 +163,7 @@ except KeyError: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2506", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf index dddb40584e..d0d9fa171f 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf @@ -54,46 +54,46 @@ variable "nomad_jobs" { job_name = "etl-stats" memory = 50000 }, - #{ - # job_name = "etl-iterative-hoststack" - # memory = 50000 - #}, - #{ - # job_name = "etl-iterative-mrr" - # memory = 50000 - #}, - #{ - # job_name = "etl-iterative-ndrpdr" - # memory = 50000 - #}, - #{ - # job_name = "etl-iterative-reconf" - # memory = 50000 - #}, - #{ - # job_name = "etl-iterative-soak" - # memory = 50000 - #}, - #{ - # job_name = "etl-coverage-hoststack" - # memory = 50000 - #}, - #{ - # job_name = "etl-coverage-mrr" - # memory = 50000 - #}, - #{ - # job_name = "etl-coverage-ndrpdr" - # memory = 50000 - #}, - #{ - # job_name = "etl-coverage-reconf" - # memory = 50000 - #}, - #{ - # job_name = "etl-coverage-soak" - # memory = 50000 - #}, + { + job_name = "etl-iterative-hoststack" + memory = 50000 + }, + { + job_name = "etl-iterative-mrr" + memory = 50000 + }, + { + job_name = "etl-iterative-ndrpdr" + memory = 50000 + }, + { + job_name = "etl-iterative-reconf" + memory = 50000 + }, + { + job_name = "etl-iterative-soak" + memory = 50000 + }, + { + job_name = "etl-coverage-hoststack" + memory = 50000 + }, + { + job_name = "etl-coverage-mrr" + memory = 50000 + }, + { + job_name = "etl-coverage-ndrpdr" + memory = 50000 + }, + { + job_name = "etl-coverage-reconf" + memory = 50000 + }, + { + job_name = "etl-coverage-soak" + memory = 50000 + }, { job_name = "etl-trending-hoststack" memory = 50000 -- 2.16.6