From 2a242e5b87bcb3117f262864e7547e63147250c5 Mon Sep 17 00:00:00 2001 From: pmikus Date: Fri, 12 Jan 2024 07:25:36 +0000 Subject: [PATCH] feat(etl): Add rls2402 Signed-off-by: Peter Mikus Change-Id: Ie1b124c81f7b81dc431af2d97386dbb4499ea13b --- ...evice_rls2310.py => coverage_device_rls2402.py} | 4 +- ...ck_rls2310.py => coverage_hoststack_rls2402.py} | 4 +- ...rage_mrr_rls2310.py => coverage_mrr_rls2402.py} | 4 +- ...drpdr_rls2310.py => coverage_ndrpdr_rls2402.py} | 4 +- ...econf_rls2310.py => coverage_reconf_rls2402.py} | 4 +- ...ge_soak_rls2310.py => coverage_soak_rls2402.py} | 4 +- ...k_rls2310.py => iterative_hoststack_rls2402.py} | 4 +- ...ive_mrr_rls2310.py => iterative_mrr_rls2402.py} | 4 +- ...rpdr_rls2310.py => iterative_ndrpdr_rls2402.py} | 4 +- ...conf_rls2310.py => iterative_reconf_rls2402.py} | 4 +- ...e_soak_rls2310.py => iterative_soak_rls2402.py} | 4 +- ...tftpl => etl-coverage-device-rls2402.hcl.tftpl} | 2 +- ...pl => etl-coverage-hoststack-rls2402.hcl.tftpl} | 2 +- ...cl.tftpl => etl-coverage-mrr-rls2402.hcl.tftpl} | 2 +- ...tftpl => etl-coverage-ndrpdr-rls2402.hcl.tftpl} | 2 +- ...tftpl => etl-coverage-reconf-rls2402.hcl.tftpl} | 2 +- ...l.tftpl => etl-coverage-soak-rls2402.hcl.tftpl} | 2 +- ...l => etl-iterative-hoststack-rls2402.hcl.tftpl} | 2 +- ...l.tftpl => etl-iterative-mrr-rls2402.hcl.tftpl} | 2 +- ...ftpl => etl-iterative-ndrpdr-rls2402.hcl.tftpl} | 2 +- ...ftpl => etl-iterative-reconf-rls2402.hcl.tftpl} | 2 +- ....tftpl => etl-iterative-soak-rls2402.hcl.tftpl} | 2 +- .../terraform-nomad-pyspark-etl/fdio/main.tf | 44 +++++++++++----------- 23 files changed, 55 insertions(+), 55 deletions(-) rename csit.infra.etl/{coverage_device_rls2310.py => coverage_device_rls2402.py} (97%) rename csit.infra.etl/{coverage_hoststack_rls2310.py => coverage_hoststack_rls2402.py} (97%) rename csit.infra.etl/{coverage_mrr_rls2310.py => coverage_mrr_rls2402.py} (97%) rename csit.infra.etl/{coverage_ndrpdr_rls2310.py => coverage_ndrpdr_rls2402.py} (97%) rename csit.infra.etl/{coverage_reconf_rls2310.py => coverage_reconf_rls2402.py} (97%) rename csit.infra.etl/{coverage_soak_rls2310.py => coverage_soak_rls2402.py} (97%) rename csit.infra.etl/{iterative_hoststack_rls2310.py => iterative_hoststack_rls2402.py} (98%) rename csit.infra.etl/{iterative_mrr_rls2310.py => iterative_mrr_rls2402.py} (98%) rename csit.infra.etl/{iterative_ndrpdr_rls2310.py => iterative_ndrpdr_rls2402.py} (98%) rename csit.infra.etl/{iterative_reconf_rls2310.py => iterative_reconf_rls2402.py} (98%) rename csit.infra.etl/{iterative_soak_rls2310.py => iterative_soak_rls2402.py} (98%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-device-rls2310.hcl.tftpl => etl-coverage-device-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-hoststack-rls2310.hcl.tftpl => etl-coverage-hoststack-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-mrr-rls2310.hcl.tftpl => etl-coverage-mrr-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-reconf-rls2310.hcl.tftpl => etl-coverage-ndrpdr-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-ndrpdr-rls2310.hcl.tftpl => etl-coverage-reconf-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-soak-rls2310.hcl.tftpl => etl-coverage-soak-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-hoststack-rls2310.hcl.tftpl => etl-iterative-hoststack-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-mrr-rls2310.hcl.tftpl => etl-iterative-mrr-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-ndrpdr-rls2310.hcl.tftpl => etl-iterative-ndrpdr-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-reconf-rls2310.hcl.tftpl => etl-iterative-reconf-rls2402.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-soak-rls2310.hcl.tftpl => etl-iterative-soak-rls2402.hcl.tftpl} (97%) diff --git a/csit.infra.etl/coverage_device_rls2310.py b/csit.infra.etl/coverage_device_rls2402.py similarity index 97% rename from csit.infra.etl/coverage_device_rls2310.py rename to csit.infra.etl/coverage_device_rls2402.py index 509a73f0de..2db808164f 100644 --- a/csit.infra.etl/coverage_device_rls2310.py +++ b/csit.infra.etl/coverage_device_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2310" in path] +filtered_paths = [path for path in paths if "report-coverage-2402" in path] out_sdf = process_json_to_dataframe("device", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_hoststack_rls2310.py b/csit.infra.etl/coverage_hoststack_rls2402.py similarity index 97% rename from csit.infra.etl/coverage_hoststack_rls2310.py rename to csit.infra.etl/coverage_hoststack_rls2402.py index 7c9e950bf7..27eb9e8cc6 100644 --- a/csit.infra.etl/coverage_hoststack_rls2310.py +++ b/csit.infra.etl/coverage_hoststack_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2310" in path] +filtered_paths = [path for path in paths if "report-coverage-2402" in path] out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_mrr_rls2310.py b/csit.infra.etl/coverage_mrr_rls2402.py similarity index 97% rename from csit.infra.etl/coverage_mrr_rls2310.py rename to csit.infra.etl/coverage_mrr_rls2402.py index 4f8b5c8799..e68e4f0366 100644 --- a/csit.infra.etl/coverage_mrr_rls2310.py +++ b/csit.infra.etl/coverage_mrr_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2310" in path] +filtered_paths = [path for path in paths if "report-coverage-2402" in path] out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_ndrpdr_rls2310.py b/csit.infra.etl/coverage_ndrpdr_rls2402.py similarity index 97% rename from csit.infra.etl/coverage_ndrpdr_rls2310.py rename to csit.infra.etl/coverage_ndrpdr_rls2402.py index f2b30ed4fa..730e3ea748 100644 --- a/csit.infra.etl/coverage_ndrpdr_rls2310.py +++ b/csit.infra.etl/coverage_ndrpdr_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2310" in path] +filtered_paths = [path for path in paths if "report-coverage-2402" in path] out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_reconf_rls2310.py b/csit.infra.etl/coverage_reconf_rls2402.py similarity index 97% rename from csit.infra.etl/coverage_reconf_rls2310.py rename to csit.infra.etl/coverage_reconf_rls2402.py index aff458c30a..dc1f647ff1 100644 --- a/csit.infra.etl/coverage_reconf_rls2310.py +++ b/csit.infra.etl/coverage_reconf_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2310" in path] +filtered_paths = [path for path in paths if "report-coverage-2402" in path] out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_soak_rls2310.py b/csit.infra.etl/coverage_soak_rls2402.py similarity index 97% rename from csit.infra.etl/coverage_soak_rls2310.py rename to csit.infra.etl/coverage_soak_rls2402.py index 2e1a739d74..7d87afd952 100644 --- a/csit.infra.etl/coverage_soak_rls2310.py +++ b/csit.infra.etl/coverage_soak_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2310" in path] +filtered_paths = [path for path in paths if "report-coverage-2402" in path] out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_hoststack_rls2310.py b/csit.infra.etl/iterative_hoststack_rls2402.py similarity index 98% rename from csit.infra.etl/iterative_hoststack_rls2310.py rename to csit.infra.etl/iterative_hoststack_rls2402.py index 7b2984694a..1c74126c47 100644 --- a/csit.infra.etl/iterative_hoststack_rls2310.py +++ b/csit.infra.etl/iterative_hoststack_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2310" in path] +filtered_paths = [path for path in paths if "report-iterative-2402" in path] out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_mrr_rls2310.py b/csit.infra.etl/iterative_mrr_rls2402.py similarity index 98% rename from csit.infra.etl/iterative_mrr_rls2310.py rename to csit.infra.etl/iterative_mrr_rls2402.py index 12a4c26fb6..e779dbdc36 100644 --- a/csit.infra.etl/iterative_mrr_rls2310.py +++ b/csit.infra.etl/iterative_mrr_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2310" in path] +filtered_paths = [path for path in paths if "report-iterative-2402" in path] out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_ndrpdr_rls2310.py b/csit.infra.etl/iterative_ndrpdr_rls2402.py similarity index 98% rename from csit.infra.etl/iterative_ndrpdr_rls2310.py rename to csit.infra.etl/iterative_ndrpdr_rls2402.py index d368c4a8fc..9231176e10 100644 --- a/csit.infra.etl/iterative_ndrpdr_rls2310.py +++ b/csit.infra.etl/iterative_ndrpdr_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2310" in path] +filtered_paths = [path for path in paths if "report-iterative-2402" in path] out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_reconf_rls2310.py b/csit.infra.etl/iterative_reconf_rls2402.py similarity index 98% rename from csit.infra.etl/iterative_reconf_rls2310.py rename to csit.infra.etl/iterative_reconf_rls2402.py index ef5e604af3..1beeb16d2c 100644 --- a/csit.infra.etl/iterative_reconf_rls2310.py +++ b/csit.infra.etl/iterative_reconf_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2310" in path] +filtered_paths = [path for path in paths if "report-iterative-2402" in path] out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_soak_rls2310.py b/csit.infra.etl/iterative_soak_rls2402.py similarity index 98% rename from csit.infra.etl/iterative_soak_rls2310.py rename to csit.infra.etl/iterative_soak_rls2402.py index cfe733ba9a..55c6eb494d 100644 --- a/csit.infra.etl/iterative_soak_rls2310.py +++ b/csit.infra.etl/iterative_soak_rls2402.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2310" in path] +filtered_paths = [path for path in paths if "report-iterative-2402" in path] out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2402", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2402.hcl.tftpl index 217197667a..cc0b1df8b5 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_device_rls2310.py" + "coverage_device_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2402.hcl.tftpl index 6aa5268df1..95d7a4c46e 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_hoststack_rls2310.py" + "coverage_hoststack_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2402.hcl.tftpl index 6358dbe690..3bab9264fa 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_mrr_rls2310.py" + "coverage_mrr_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2402.hcl.tftpl index 3bea4e77b4..6142219546 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_reconf_rls2310.py" + "coverage_ndrpdr_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2402.hcl.tftpl index c2400ad444..b474e75217 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_ndrpdr_rls2310.py" + "coverage_reconf_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2402.hcl.tftpl index 8ca898be37..0352e1e879 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_soak_rls2310.py" + "coverage_soak_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2402.hcl.tftpl index 285fcac9c0..74478c59f7 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_hoststack_rls2310.py" + "iterative_hoststack_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2402.hcl.tftpl index 764dfe7147..e6bd87b8ed 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_mrr_rls2310.py" + "iterative_mrr_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2402.hcl.tftpl index 5ea75a49d9..4a40321377 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_ndrpdr_rls2310.py" + "iterative_ndrpdr_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2402.hcl.tftpl index 17ad83e25a..670dd37a11 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_reconf_rls2310.py" + "iterative_reconf_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2402.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2402.hcl.tftpl index 69753701ce..c4ad363879 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2402.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_soak_rls2310.py" + "iterative_soak_rls2402.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf index 9dd43c681f..6b5edb79b7 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf @@ -76,7 +76,7 @@ module "etl-trending-ndrpdr" { memory = 60000 } -module "etl-iterative-hoststack-rls2310" { +module "etl-iterative-hoststack-rls2402" { providers = { nomad = nomad.yul1 } @@ -90,10 +90,10 @@ module "etl-iterative-hoststack-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-iterative-hoststack-rls2310" + job_name = "etl-iterative-hoststack-rls2402" } -module "etl-iterative-mrr-rls2310" { +module "etl-iterative-mrr-rls2402" { providers = { nomad = nomad.yul1 } @@ -107,10 +107,10 @@ module "etl-iterative-mrr-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-iterative-mrr-rls2310" + job_name = "etl-iterative-mrr-rls2402" } -module "etl-iterative-ndrpdr-rls2310" { +module "etl-iterative-ndrpdr-rls2402" { providers = { nomad = nomad.yul1 } @@ -124,10 +124,10 @@ module "etl-iterative-ndrpdr-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-iterative-ndrpdr-rls2310" + job_name = "etl-iterative-ndrpdr-rls2402" } -module "etl-iterative-reconf-rls2310" { +module "etl-iterative-reconf-rls2402" { providers = { nomad = nomad.yul1 } @@ -141,10 +141,10 @@ module "etl-iterative-reconf-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-iterative-reconf-rls2310" + job_name = "etl-iterative-reconf-rls2402" } -module "etl-iterative-soak-rls2310" { +module "etl-iterative-soak-rls2402" { providers = { nomad = nomad.yul1 } @@ -158,10 +158,10 @@ module "etl-iterative-soak-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-iterative-soak-rls2310" + job_name = "etl-iterative-soak-rls2402" } -module "etl-coverage-device-rls2310" { +module "etl-coverage-device-rls2402" { providers = { nomad = nomad.yul1 } @@ -175,10 +175,10 @@ module "etl-coverage-device-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-coverage-device-rls2310" + job_name = "etl-coverage-device-rls2402" } -module "etl-coverage-hoststack-rls2310" { +module "etl-coverage-hoststack-rls2402" { providers = { nomad = nomad.yul1 } @@ -192,10 +192,10 @@ module "etl-coverage-hoststack-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-coverage-hoststack-rls2310" + job_name = "etl-coverage-hoststack-rls2402" } -module "etl-coverage-mrr-rls2310" { +module "etl-coverage-mrr-rls2402" { providers = { nomad = nomad.yul1 } @@ -209,10 +209,10 @@ module "etl-coverage-mrr-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-coverage-mrr-rls2310" + job_name = "etl-coverage-mrr-rls2402" } -module "etl-coverage-ndrpdr-rls2310" { +module "etl-coverage-ndrpdr-rls2402" { providers = { nomad = nomad.yul1 } @@ -226,10 +226,10 @@ module "etl-coverage-ndrpdr-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-coverage-ndrpdr-rls2310" + job_name = "etl-coverage-ndrpdr-rls2402" } -module "etl-coverage-reconf-rls2310" { +module "etl-coverage-reconf-rls2402" { providers = { nomad = nomad.yul1 } @@ -243,10 +243,10 @@ module "etl-coverage-reconf-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-coverage-reconf-rls2310" + job_name = "etl-coverage-reconf-rls2402" } -module "etl-coverage-soak-rls2310" { +module "etl-coverage-soak-rls2402" { providers = { nomad = nomad.yul1 } @@ -260,5 +260,5 @@ module "etl-coverage-soak-rls2310" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "0 30 0 * * * *" datacenters = ["yul1"] - job_name = "etl-coverage-soak-rls2310" + job_name = "etl-coverage-soak-rls2402" } -- 2.16.6