feat(terraform): ETL for 2310

Signed-off-by: pmikus <peter.mikus@protonmail.ch>
Change-Id: I3013b8be6a7f06d2f1f3b8320e7cb6f057a47491

diff --git a/csit.infra.etl/coverage_device_rls2306.py b/csit.infra.etl/coverage_device_rls2310.py
similarity index 98%
rename from csit.infra.etl/coverage_device_rls2306.py
rename to csit.infra.etl/coverage_device_rls2310.py
index 2d4f59b..509a73f 100644 (file)
--- a/csit.infra.etl/coverage_device_rls2306.py
+++ b/csit.infra.etl/coverage_device_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2306" in path]
+filtered_paths = [path for path in paths if "report-coverage-2310" in path]
 
 out_sdf = process_json_to_dataframe("device", filtered_paths)
 out_sdf.printSchema()
@@ -154,7 +154,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_hoststack_rls2306.py b/csit.infra.etl/coverage_hoststack_rls2310.py
similarity index 98%
rename from csit.infra.etl/coverage_hoststack_rls2306.py
rename to csit.infra.etl/coverage_hoststack_rls2310.py
index 9a5f7a5..7c9e950 100644 (file)
--- a/csit.infra.etl/coverage_hoststack_rls2306.py
+++ b/csit.infra.etl/coverage_hoststack_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2306" in path]
+filtered_paths = [path for path in paths if "report-coverage-2310" in path]
 
 out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.show(truncate=False)
@@ -155,7 +155,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_mrr_rls2306.py b/csit.infra.etl/coverage_mrr_rls2310.py
similarity index 98%
rename from csit.infra.etl/coverage_mrr_rls2306.py
rename to csit.infra.etl/coverage_mrr_rls2310.py
index 02ea40e..4f8b5c8 100644 (file)
--- a/csit.infra.etl/coverage_mrr_rls2306.py
+++ b/csit.infra.etl/coverage_mrr_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2306" in path]
+filtered_paths = [path for path in paths if "report-coverage-2310" in path]
 
 out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
@@ -154,7 +154,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_ndrpdr_rls2306.py b/csit.infra.etl/coverage_ndrpdr_rls2310.py
similarity index 98%
rename from csit.infra.etl/coverage_ndrpdr_rls2306.py
rename to csit.infra.etl/coverage_ndrpdr_rls2310.py
index 5987e3e..f2b30ed 100644 (file)
--- a/csit.infra.etl/coverage_ndrpdr_rls2306.py
+++ b/csit.infra.etl/coverage_ndrpdr_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2306" in path]
+filtered_paths = [path for path in paths if "report-coverage-2310" in path]
 
 out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
@@ -154,7 +154,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_reconf_rls2306.py b/csit.infra.etl/coverage_reconf_rls2310.py
similarity index 98%
rename from csit.infra.etl/coverage_reconf_rls2306.py
rename to csit.infra.etl/coverage_reconf_rls2310.py
index 3288574..aff458c 100644 (file)
--- a/csit.infra.etl/coverage_reconf_rls2306.py
+++ b/csit.infra.etl/coverage_reconf_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2306" in path]
+filtered_paths = [path for path in paths if "report-coverage-2310" in path]
 
 out_sdf = process_json_to_dataframe("reconf", filtered_paths)
 out_sdf.show(truncate=False)
@@ -155,7 +155,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/coverage_soak_rls2306.py b/csit.infra.etl/coverage_soak_rls2310.py
similarity index 98%
rename from csit.infra.etl/coverage_soak_rls2306.py
rename to csit.infra.etl/coverage_soak_rls2310.py
index fd4b9fe..2e1a739 100644 (file)
--- a/csit.infra.etl/coverage_soak_rls2306.py
+++ b/csit.infra.etl/coverage_soak_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2306" in path]
+filtered_paths = [path for path in paths if "report-coverage-2310" in path]
 
 out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
@@ -154,7 +154,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_hoststack_rls2306.py b/csit.infra.etl/iterative_hoststack_rls2310.py
similarity index 99%
rename from csit.infra.etl/iterative_hoststack_rls2306.py
rename to csit.infra.etl/iterative_hoststack_rls2310.py
index 0b179de..7b29846 100644 (file)
--- a/csit.infra.etl/iterative_hoststack_rls2306.py
+++ b/csit.infra.etl/iterative_hoststack_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2306" in path]
+filtered_paths = [path for path in paths if "report-iterative-2310" in path]
 
 out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.show(truncate=False)
@@ -155,7 +155,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_mrr_rls2306.py b/csit.infra.etl/iterative_mrr_rls2310.py
similarity index 99%
rename from csit.infra.etl/iterative_mrr_rls2306.py
rename to csit.infra.etl/iterative_mrr_rls2310.py
index 33ffff7..12a4c26 100644 (file)
--- a/csit.infra.etl/iterative_mrr_rls2306.py
+++ b/csit.infra.etl/iterative_mrr_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2306" in path]
+filtered_paths = [path for path in paths if "report-iterative-2310" in path]
 
 out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
@@ -154,7 +154,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_ndrpdr_rls2306.py b/csit.infra.etl/iterative_ndrpdr_rls2310.py
similarity index 99%
rename from csit.infra.etl/iterative_ndrpdr_rls2306.py
rename to csit.infra.etl/iterative_ndrpdr_rls2310.py
index 9b3a36b..d368c4a 100644 (file)
--- a/csit.infra.etl/iterative_ndrpdr_rls2306.py
+++ b/csit.infra.etl/iterative_ndrpdr_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2306" in path]
+filtered_paths = [path for path in paths if "report-iterative-2310" in path]
 
 out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
@@ -154,7 +154,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_reconf_rls2306.py b/csit.infra.etl/iterative_reconf_rls2310.py
similarity index 99%
rename from csit.infra.etl/iterative_reconf_rls2306.py
rename to csit.infra.etl/iterative_reconf_rls2310.py
index a45ac66..ef5e604 100644 (file)
--- a/csit.infra.etl/iterative_reconf_rls2306.py
+++ b/csit.infra.etl/iterative_reconf_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2306" in path]
+filtered_paths = [path for path in paths if "report-iterative-2310" in path]
 
 out_sdf = process_json_to_dataframe("reconf", filtered_paths)
 out_sdf.show(truncate=False)
@@ -155,7 +155,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/csit.infra.etl/iterative_soak_rls2306.py b/csit.infra.etl/iterative_soak_rls2310.py
similarity index 99%
rename from csit.infra.etl/iterative_soak_rls2306.py
rename to csit.infra.etl/iterative_soak_rls2310.py
index ccc5012..cfe733b 100644 (file)
--- a/csit.infra.etl/iterative_soak_rls2306.py
+++ b/csit.infra.etl/iterative_soak_rls2310.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2306" in path]
+filtered_paths = [path for path in paths if "report-iterative-2310" in path]
 
 out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
@@ -154,7 +154,7 @@ out_sdf = out_sdf \
 try:
     wr.s3.to_parquet(
         df=out_sdf.toPandas(),
-        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
+        path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2310",
         dataset=True,
         partition_cols=["test_type", "year", "month", "day"],
         compression="snappy",

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/README.md b/fdio.infra.terraform/terraform-nomad-pyspark-etl/README.md
index 1147ddb..d61c877 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/README.md
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/README.md
@@ -3,14 +3,14 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.4 |
-| <a name="requirement_nomad"></a> [nomad](#requirement\_nomad) | >= 1.4.16 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.4 |
+| <a name="requirement_nomad"></a> [nomad](#requirement\_nomad) | >= 1.4.20 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_nomad"></a> [nomad](#provider\_nomad) | >= 1.4.16 |
+| <a name="provider_nomad"></a> [nomad](#provider\_nomad) | >= 1.4.20 |
 
 ## Modules
 
@@ -20,7 +20,7 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [nomad_job.nomad_job_etl](https://registry.terraform.io/providers/hashicorp/nomad/latest/docs/resources/job) | resource |
+| [nomad_job.nomad_job](https://registry.terraform.io/providers/hashicorp/nomad/latest/docs/resources/job) | resource |
 
 ## Inputs
 
@@ -35,7 +35,7 @@ No modules.
 | <a name="input_envs"></a> [envs](#input\_envs) | Specifies ETL environment variables. | `list(string)` | `[]` | no |
 | <a name="input_image"></a> [image](#input\_image) | Specifies the Docker image to run. | `string` | `"pmikus/docker-ubuntu-focal-aws-glue:latest"` | no |
 | <a name="input_job_name"></a> [job\_name](#input\_job\_name) | Specifies a name for the job. | `string` | `"etl"` | no |
-| <a name="input_memory"></a> [memory](#input\_memory) | Specifies the memory required in MB. | `number` | `20000` | no |
+| <a name="input_memory"></a> [memory](#input\_memory) | Specifies the memory required in MB. | `number` | `50000` | no |
 | <a name="input_out_aws_access_key_id"></a> [out\_aws\_access\_key\_id](#input\_out\_aws\_access\_key\_id) | AWS access key. | `string` | `"aws"` | no |
 | <a name="input_out_aws_default_region"></a> [out\_aws\_default\_region](#input\_out\_aws\_default\_region) | AWS region | `string` | `"aws"` | no |
 | <a name="input_out_aws_secret_access_key"></a> [out\_aws\_secret\_access\_key](#input\_out\_aws\_secret\_access\_key) | AWS secret key | `string` | `"aws"` | no |

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl
index c769659..2171976 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_device_rls2306.py"
+          "coverage_device_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl
index 3567f02..6aa5268 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_hoststack_rls2306.py"
+          "coverage_hoststack_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl
index edfd995..6358dbe 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_mrr_rls2306.py"
+          "coverage_mrr_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl
index 82630ba..c2400ad 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_reconf_rls2306.py"
+          "coverage_ndrpdr_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl
index 298853b..3bea4e7 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_ndrpdr_rls2306.py"
+          "coverage_reconf_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl
index f32a7cf..8ca898b 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_soak_rls2306.py"
+          "coverage_soak_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl
index c4ebf03..285fcac 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_hoststack_rls2306.py"
+          "iterative_hoststack_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl
index d4e0f95..764dfe7 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_mrr_rls2306.py"
+          "iterative_mrr_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl
index e18ebbc..5ea75a4 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_ndrpdr_rls2306.py"
+          "iterative_ndrpdr_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl
index 46b5ba2..17ad83e 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_reconf_rls2306.py"
+          "iterative_reconf_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl
similarity index 97%
rename from fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl
rename to fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl
index 7df5aef..6975370 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl
@@ -33,7 +33,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_soak_rls2306.py"
+          "iterative_soak_rls2310.py"
         ]
         work_dir = "/local/csit/csit.infra.etl"
       }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf
index ec1a38d..cb6b3a0 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf
@@ -74,7 +74,7 @@ module "etl-trending-ndrpdr" {
   job_name                  = "etl-trending-ndrpdr"
 }
 
-module "etl-iterative-hoststack-rls2306" {
+module "etl-iterative-hoststack-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -88,10 +88,10 @@ module "etl-iterative-hoststack-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-hoststack-rls2306"
+  job_name                  = "etl-iterative-hoststack-rls2310"
 }
 
-module "etl-iterative-mrr-rls2306" {
+module "etl-iterative-mrr-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -105,10 +105,10 @@ module "etl-iterative-mrr-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-mrr-rls2306"
+  job_name                  = "etl-iterative-mrr-rls2310"
 }
 
-module "etl-iterative-ndrpdr-rls2306" {
+module "etl-iterative-ndrpdr-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -122,10 +122,10 @@ module "etl-iterative-ndrpdr-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-ndrpdr-rls2306"
+  job_name                  = "etl-iterative-ndrpdr-rls2310"
 }
 
-module "etl-iterative-reconf-rls2306" {
+module "etl-iterative-reconf-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -139,10 +139,10 @@ module "etl-iterative-reconf-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-reconf-rls2306"
+  job_name                  = "etl-iterative-reconf-rls2310"
 }
 
-module "etl-iterative-soak-rls2306" {
+module "etl-iterative-soak-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -156,10 +156,10 @@ module "etl-iterative-soak-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-iterative-soak-rls2306"
+  job_name                  = "etl-iterative-soak-rls2310"
 }
 
-module "etl-coverage-device-rls2306" {
+module "etl-coverage-device-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -173,10 +173,10 @@ module "etl-coverage-device-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-device-rls2306"
+  job_name                  = "etl-coverage-device-rls2310"
 }
 
-module "etl-coverage-hoststack-rls2306" {
+module "etl-coverage-hoststack-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -190,10 +190,10 @@ module "etl-coverage-hoststack-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-hoststack-rls2306"
+  job_name                  = "etl-coverage-hoststack-rls2310"
 }
 
-module "etl-coverage-mrr-rls2306" {
+module "etl-coverage-mrr-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -207,10 +207,10 @@ module "etl-coverage-mrr-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-mrr-rls2306"
+  job_name                  = "etl-coverage-mrr-rls2310"
 }
 
-module "etl-coverage-ndrpdr-rls2306" {
+module "etl-coverage-ndrpdr-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -224,10 +224,10 @@ module "etl-coverage-ndrpdr-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-ndrpdr-rls2306"
+  job_name                  = "etl-coverage-ndrpdr-rls2310"
 }
 
-module "etl-coverage-reconf-rls2306" {
+module "etl-coverage-reconf-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -241,10 +241,10 @@ module "etl-coverage-reconf-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-reconf-rls2306"
+  job_name                  = "etl-coverage-reconf-rls2310"
 }
 
-module "etl-coverage-soak-rls2306" {
+module "etl-coverage-soak-rls2310" {
   providers = {
     nomad = nomad.yul1
   }
@@ -258,5 +258,5 @@ module "etl-coverage-soak-rls2306" {
   out_aws_default_region    = data.vault_generic_secret.fdio_docs.data["region"]
   cron                      = "@daily"
   datacenters               = ["yul1"]
-  job_name                  = "etl-coverage-soak-rls2306"
+  job_name                  = "etl-coverage-soak-rls2310"
 }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf
index 2a5c83f..0c05e76 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf
@@ -7,11 +7,11 @@ terraform {
   required_providers {
     nomad = {
       source  = "hashicorp/nomad"
-      version = ">= 1.4.19"
+      version = ">= 1.4.20"
     }
     vault = {
       version = ">= 3.12.0"
     }
   }
-  required_version = ">= 1.3.7"
+  required_version = ">= 1.5.4"
 }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf
index a319c35..f40435f 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf
@@ -2,8 +2,8 @@ terraform {
   required_providers {
     nomad = {
       source  = "hashicorp/nomad"
-      version = ">= 1.4.19"
+      version = ">= 1.4.20"
     }
   }
-  required_version = ">= 1.3.7"
+  required_version = ">= 1.5.4"
 }