From b0b7f8dfdd37d45de0f3fcf758798e3bfbb9381d Mon Sep 17 00:00:00 2001 From: pmikus Date: Thu, 17 Aug 2023 12:26:28 +0000 Subject: [PATCH] feat(terraform): ETL for 2310 Signed-off-by: pmikus Change-Id: I3013b8be6a7f06d2f1f3b8320e7cb6f057a47491 --- ...evice_rls2306.py => coverage_device_rls2310.py} | 4 +- ...ck_rls2306.py => coverage_hoststack_rls2310.py} | 4 +- ...rage_mrr_rls2306.py => coverage_mrr_rls2310.py} | 4 +- ...drpdr_rls2306.py => coverage_ndrpdr_rls2310.py} | 4 +- ...econf_rls2306.py => coverage_reconf_rls2310.py} | 4 +- ...ge_soak_rls2306.py => coverage_soak_rls2310.py} | 4 +- ...k_rls2306.py => iterative_hoststack_rls2310.py} | 4 +- ...ive_mrr_rls2306.py => iterative_mrr_rls2310.py} | 4 +- ...rpdr_rls2306.py => iterative_ndrpdr_rls2310.py} | 4 +- ...conf_rls2306.py => iterative_reconf_rls2310.py} | 4 +- ...e_soak_rls2306.py => iterative_soak_rls2310.py} | 4 +- .../terraform-nomad-pyspark-etl/README.md | 10 ++--- ...tftpl => etl-coverage-device-rls2310.hcl.tftpl} | 2 +- ...pl => etl-coverage-hoststack-rls2310.hcl.tftpl} | 2 +- ...cl.tftpl => etl-coverage-mrr-rls2310.hcl.tftpl} | 2 +- ...tftpl => etl-coverage-ndrpdr-rls2310.hcl.tftpl} | 2 +- ...tftpl => etl-coverage-reconf-rls2310.hcl.tftpl} | 2 +- ...l.tftpl => etl-coverage-soak-rls2310.hcl.tftpl} | 2 +- ...l => etl-iterative-hoststack-rls2310.hcl.tftpl} | 2 +- ...l.tftpl => etl-iterative-mrr-rls2310.hcl.tftpl} | 2 +- ...ftpl => etl-iterative-ndrpdr-rls2310.hcl.tftpl} | 2 +- ...ftpl => etl-iterative-reconf-rls2310.hcl.tftpl} | 2 +- ....tftpl => etl-iterative-soak-rls2310.hcl.tftpl} | 2 +- .../terraform-nomad-pyspark-etl/fdio/main.tf | 44 +++++++++++----------- .../terraform-nomad-pyspark-etl/fdio/versions.tf | 4 +- .../terraform-nomad-pyspark-etl/versions.tf | 4 +- 26 files changed, 64 insertions(+), 64 deletions(-) rename csit.infra.etl/{coverage_device_rls2306.py => coverage_device_rls2310.py} (98%) rename csit.infra.etl/{coverage_hoststack_rls2306.py => coverage_hoststack_rls2310.py} (98%) rename csit.infra.etl/{coverage_mrr_rls2306.py => coverage_mrr_rls2310.py} (98%) rename csit.infra.etl/{coverage_ndrpdr_rls2306.py => coverage_ndrpdr_rls2310.py} (98%) rename csit.infra.etl/{coverage_reconf_rls2306.py => coverage_reconf_rls2310.py} (98%) rename csit.infra.etl/{coverage_soak_rls2306.py => coverage_soak_rls2310.py} (98%) rename csit.infra.etl/{iterative_hoststack_rls2306.py => iterative_hoststack_rls2310.py} (99%) rename csit.infra.etl/{iterative_mrr_rls2306.py => iterative_mrr_rls2310.py} (99%) rename csit.infra.etl/{iterative_ndrpdr_rls2306.py => iterative_ndrpdr_rls2310.py} (99%) rename csit.infra.etl/{iterative_reconf_rls2306.py => iterative_reconf_rls2310.py} (99%) rename csit.infra.etl/{iterative_soak_rls2306.py => iterative_soak_rls2310.py} (99%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-device-rls2306.hcl.tftpl => etl-coverage-device-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-hoststack-rls2306.hcl.tftpl => etl-coverage-hoststack-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-mrr-rls2306.hcl.tftpl => etl-coverage-mrr-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-reconf-rls2306.hcl.tftpl => etl-coverage-ndrpdr-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-ndrpdr-rls2306.hcl.tftpl => etl-coverage-reconf-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-coverage-soak-rls2306.hcl.tftpl => etl-coverage-soak-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-hoststack-rls2306.hcl.tftpl => etl-iterative-hoststack-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-mrr-rls2306.hcl.tftpl => etl-iterative-mrr-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-ndrpdr-rls2306.hcl.tftpl => etl-iterative-ndrpdr-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-reconf-rls2306.hcl.tftpl => etl-iterative-reconf-rls2310.hcl.tftpl} (97%) rename fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/{etl-iterative-soak-rls2306.hcl.tftpl => etl-iterative-soak-rls2310.hcl.tftpl} (97%) diff --git a/csit.infra.etl/coverage_device_rls2306.py b/csit.infra.etl/coverage_device_rls2310.py similarity index 98% rename from csit.infra.etl/coverage_device_rls2306.py rename to csit.infra.etl/coverage_device_rls2310.py index 2d4f59b29c..509a73f0de 100644 --- a/csit.infra.etl/coverage_device_rls2306.py +++ b/csit.infra.etl/coverage_device_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2306" in path] +filtered_paths = [path for path in paths if "report-coverage-2310" in path] out_sdf = process_json_to_dataframe("device", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_hoststack_rls2306.py b/csit.infra.etl/coverage_hoststack_rls2310.py similarity index 98% rename from csit.infra.etl/coverage_hoststack_rls2306.py rename to csit.infra.etl/coverage_hoststack_rls2310.py index 9a5f7a5bc9..7c9e950bf7 100644 --- a/csit.infra.etl/coverage_hoststack_rls2306.py +++ b/csit.infra.etl/coverage_hoststack_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2306" in path] +filtered_paths = [path for path in paths if "report-coverage-2310" in path] out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_mrr_rls2306.py b/csit.infra.etl/coverage_mrr_rls2310.py similarity index 98% rename from csit.infra.etl/coverage_mrr_rls2306.py rename to csit.infra.etl/coverage_mrr_rls2310.py index 02ea40ef2b..4f8b5c8799 100644 --- a/csit.infra.etl/coverage_mrr_rls2306.py +++ b/csit.infra.etl/coverage_mrr_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2306" in path] +filtered_paths = [path for path in paths if "report-coverage-2310" in path] out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_ndrpdr_rls2306.py b/csit.infra.etl/coverage_ndrpdr_rls2310.py similarity index 98% rename from csit.infra.etl/coverage_ndrpdr_rls2306.py rename to csit.infra.etl/coverage_ndrpdr_rls2310.py index 5987e3eaa1..f2b30ed4fa 100644 --- a/csit.infra.etl/coverage_ndrpdr_rls2306.py +++ b/csit.infra.etl/coverage_ndrpdr_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2306" in path] +filtered_paths = [path for path in paths if "report-coverage-2310" in path] out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_reconf_rls2306.py b/csit.infra.etl/coverage_reconf_rls2310.py similarity index 98% rename from csit.infra.etl/coverage_reconf_rls2306.py rename to csit.infra.etl/coverage_reconf_rls2310.py index 32885744d7..aff458c30a 100644 --- a/csit.infra.etl/coverage_reconf_rls2306.py +++ b/csit.infra.etl/coverage_reconf_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2306" in path] +filtered_paths = [path for path in paths if "report-coverage-2310" in path] out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_soak_rls2306.py b/csit.infra.etl/coverage_soak_rls2310.py similarity index 98% rename from csit.infra.etl/coverage_soak_rls2306.py rename to csit.infra.etl/coverage_soak_rls2310.py index fd4b9fed35..2e1a739d74 100644 --- a/csit.infra.etl/coverage_soak_rls2306.py +++ b/csit.infra.etl/coverage_soak_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2306" in path] +filtered_paths = [path for path in paths if "report-coverage-2310" in path] out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_hoststack_rls2306.py b/csit.infra.etl/iterative_hoststack_rls2310.py similarity index 99% rename from csit.infra.etl/iterative_hoststack_rls2306.py rename to csit.infra.etl/iterative_hoststack_rls2310.py index 0b179dea88..7b2984694a 100644 --- a/csit.infra.etl/iterative_hoststack_rls2306.py +++ b/csit.infra.etl/iterative_hoststack_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2306" in path] +filtered_paths = [path for path in paths if "report-iterative-2310" in path] out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_mrr_rls2306.py b/csit.infra.etl/iterative_mrr_rls2310.py similarity index 99% rename from csit.infra.etl/iterative_mrr_rls2306.py rename to csit.infra.etl/iterative_mrr_rls2310.py index 33ffff7ce4..12a4c26fb6 100644 --- a/csit.infra.etl/iterative_mrr_rls2306.py +++ b/csit.infra.etl/iterative_mrr_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2306" in path] +filtered_paths = [path for path in paths if "report-iterative-2310" in path] out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_ndrpdr_rls2306.py b/csit.infra.etl/iterative_ndrpdr_rls2310.py similarity index 99% rename from csit.infra.etl/iterative_ndrpdr_rls2306.py rename to csit.infra.etl/iterative_ndrpdr_rls2310.py index 9b3a36be18..d368c4a8fc 100644 --- a/csit.infra.etl/iterative_ndrpdr_rls2306.py +++ b/csit.infra.etl/iterative_ndrpdr_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2306" in path] +filtered_paths = [path for path in paths if "report-iterative-2310" in path] out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_reconf_rls2306.py b/csit.infra.etl/iterative_reconf_rls2310.py similarity index 99% rename from csit.infra.etl/iterative_reconf_rls2306.py rename to csit.infra.etl/iterative_reconf_rls2310.py index a45ac66a83..ef5e604af3 100644 --- a/csit.infra.etl/iterative_reconf_rls2306.py +++ b/csit.infra.etl/iterative_reconf_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2306" in path] +filtered_paths = [path for path in paths if "report-iterative-2310" in path] out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_soak_rls2306.py b/csit.infra.etl/iterative_soak_rls2310.py similarity index 99% rename from csit.infra.etl/iterative_soak_rls2306.py rename to csit.infra.etl/iterative_soak_rls2310.py index ccc5012199..cfe733ba9a 100644 --- a/csit.infra.etl/iterative_soak_rls2306.py +++ b/csit.infra.etl/iterative_soak_rls2310.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2306" in path] +filtered_paths = [path for path in paths if "report-iterative-2310" in path] out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/README.md b/fdio.infra.terraform/terraform-nomad-pyspark-etl/README.md index 1147ddb16a..d61c8778d4 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/README.md +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/README.md @@ -3,14 +3,14 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.1.4 | -| [nomad](#requirement\_nomad) | >= 1.4.16 | +| [terraform](#requirement\_terraform) | >= 1.5.4 | +| [nomad](#requirement\_nomad) | >= 1.4.20 | ## Providers | Name | Version | |------|---------| -| [nomad](#provider\_nomad) | >= 1.4.16 | +| [nomad](#provider\_nomad) | >= 1.4.20 | ## Modules @@ -20,7 +20,7 @@ No modules. | Name | Type | |------|------| -| [nomad_job.nomad_job_etl](https://registry.terraform.io/providers/hashicorp/nomad/latest/docs/resources/job) | resource | +| [nomad_job.nomad_job](https://registry.terraform.io/providers/hashicorp/nomad/latest/docs/resources/job) | resource | ## Inputs @@ -35,7 +35,7 @@ No modules. | [envs](#input\_envs) | Specifies ETL environment variables. | `list(string)` | `[]` | no | | [image](#input\_image) | Specifies the Docker image to run. | `string` | `"pmikus/docker-ubuntu-focal-aws-glue:latest"` | no | | [job\_name](#input\_job\_name) | Specifies a name for the job. | `string` | `"etl"` | no | -| [memory](#input\_memory) | Specifies the memory required in MB. | `number` | `20000` | no | +| [memory](#input\_memory) | Specifies the memory required in MB. | `number` | `50000` | no | | [out\_aws\_access\_key\_id](#input\_out\_aws\_access\_key\_id) | AWS access key. | `string` | `"aws"` | no | | [out\_aws\_default\_region](#input\_out\_aws\_default\_region) | AWS region | `string` | `"aws"` | no | | [out\_aws\_secret\_access\_key](#input\_out\_aws\_secret\_access\_key) | AWS secret key | `string` | `"aws"` | no | diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl index c769659bc6..217197667a 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_device_rls2306.py" + "coverage_device_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl index 3567f0258e..6aa5268df1 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_hoststack_rls2306.py" + "coverage_hoststack_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl index edfd995980..6358dbe690 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_mrr_rls2306.py" + "coverage_mrr_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl index 82630ba64d..c2400ad444 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_reconf_rls2306.py" + "coverage_ndrpdr_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl index 298853b15e..3bea4e77b4 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_ndrpdr_rls2306.py" + "coverage_reconf_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl index f32a7cf024..8ca898be37 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_soak_rls2306.py" + "coverage_soak_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl index c4ebf032d8..285fcac9c0 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_hoststack_rls2306.py" + "iterative_hoststack_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl index d4e0f9598f..764dfe7147 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_mrr_rls2306.py" + "iterative_mrr_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl index e18ebbc6e2..5ea75a49d9 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_ndrpdr_rls2306.py" + "iterative_ndrpdr_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl index 46b5ba2c80..17ad83e25a 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_reconf_rls2306.py" + "iterative_reconf_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl similarity index 97% rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl index 7df5aef88f..69753701ce 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl @@ -33,7 +33,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_soak_rls2306.py" + "iterative_soak_rls2310.py" ] work_dir = "/local/csit/csit.infra.etl" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf index ec1a38d375..cb6b3a0021 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf @@ -74,7 +74,7 @@ module "etl-trending-ndrpdr" { job_name = "etl-trending-ndrpdr" } -module "etl-iterative-hoststack-rls2306" { +module "etl-iterative-hoststack-rls2310" { providers = { nomad = nomad.yul1 } @@ -88,10 +88,10 @@ module "etl-iterative-hoststack-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-iterative-hoststack-rls2306" + job_name = "etl-iterative-hoststack-rls2310" } -module "etl-iterative-mrr-rls2306" { +module "etl-iterative-mrr-rls2310" { providers = { nomad = nomad.yul1 } @@ -105,10 +105,10 @@ module "etl-iterative-mrr-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-iterative-mrr-rls2306" + job_name = "etl-iterative-mrr-rls2310" } -module "etl-iterative-ndrpdr-rls2306" { +module "etl-iterative-ndrpdr-rls2310" { providers = { nomad = nomad.yul1 } @@ -122,10 +122,10 @@ module "etl-iterative-ndrpdr-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-iterative-ndrpdr-rls2306" + job_name = "etl-iterative-ndrpdr-rls2310" } -module "etl-iterative-reconf-rls2306" { +module "etl-iterative-reconf-rls2310" { providers = { nomad = nomad.yul1 } @@ -139,10 +139,10 @@ module "etl-iterative-reconf-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-iterative-reconf-rls2306" + job_name = "etl-iterative-reconf-rls2310" } -module "etl-iterative-soak-rls2306" { +module "etl-iterative-soak-rls2310" { providers = { nomad = nomad.yul1 } @@ -156,10 +156,10 @@ module "etl-iterative-soak-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-iterative-soak-rls2306" + job_name = "etl-iterative-soak-rls2310" } -module "etl-coverage-device-rls2306" { +module "etl-coverage-device-rls2310" { providers = { nomad = nomad.yul1 } @@ -173,10 +173,10 @@ module "etl-coverage-device-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-coverage-device-rls2306" + job_name = "etl-coverage-device-rls2310" } -module "etl-coverage-hoststack-rls2306" { +module "etl-coverage-hoststack-rls2310" { providers = { nomad = nomad.yul1 } @@ -190,10 +190,10 @@ module "etl-coverage-hoststack-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-coverage-hoststack-rls2306" + job_name = "etl-coverage-hoststack-rls2310" } -module "etl-coverage-mrr-rls2306" { +module "etl-coverage-mrr-rls2310" { providers = { nomad = nomad.yul1 } @@ -207,10 +207,10 @@ module "etl-coverage-mrr-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-coverage-mrr-rls2306" + job_name = "etl-coverage-mrr-rls2310" } -module "etl-coverage-ndrpdr-rls2306" { +module "etl-coverage-ndrpdr-rls2310" { providers = { nomad = nomad.yul1 } @@ -224,10 +224,10 @@ module "etl-coverage-ndrpdr-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-coverage-ndrpdr-rls2306" + job_name = "etl-coverage-ndrpdr-rls2310" } -module "etl-coverage-reconf-rls2306" { +module "etl-coverage-reconf-rls2310" { providers = { nomad = nomad.yul1 } @@ -241,10 +241,10 @@ module "etl-coverage-reconf-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-coverage-reconf-rls2306" + job_name = "etl-coverage-reconf-rls2310" } -module "etl-coverage-soak-rls2306" { +module "etl-coverage-soak-rls2310" { providers = { nomad = nomad.yul1 } @@ -258,5 +258,5 @@ module "etl-coverage-soak-rls2306" { out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] cron = "@daily" datacenters = ["yul1"] - job_name = "etl-coverage-soak-rls2306" + job_name = "etl-coverage-soak-rls2310" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf index 2a5c83f29f..0c05e76d65 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf @@ -7,11 +7,11 @@ terraform { required_providers { nomad = { source = "hashicorp/nomad" - version = ">= 1.4.19" + version = ">= 1.4.20" } vault = { version = ">= 3.12.0" } } - required_version = ">= 1.3.7" + required_version = ">= 1.5.4" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf index a319c35908..f40435fe77 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf @@ -2,8 +2,8 @@ terraform { required_providers { nomad = { source = "hashicorp/nomad" - version = ">= 1.4.19" + version = ">= 1.4.20" } } - required_version = ">= 1.3.7" + required_version = ">= 1.5.4" } -- 2.16.6