fix(etl): Split memory usage

Signed-off-by: pmikus <peter.mikus@protonmail.ch>
Change-Id: Icd6b6bbff65ef70fd85bf3595c1d2a1a06655416

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl
index 241af4d..b5b43e5 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl
@@ -6,7 +6,7 @@ job "${job_name}" {
     prohibit_overlap = "${prohibit_overlap}"
     time_zone        = "${time_zone}"
   }
-  group "${job_name}-master" {
+  group "${job_name}-trending" {
     restart {
       mode = "fail"
     }
@@ -169,6 +169,20 @@ job "${job_name}" {
         memory = ${memory}
       }
     }
+  }
+  group "${job_name}-stats" {
+    restart {
+      mode = "fail"
+    }
+    constraint {
+      attribute       = "$${attr.cpu.arch}"
+      operator        = "!="
+      value           = "arm64"
+    }
+    constraint {
+      attribute      = "$${node.class}"
+      value          = "builder"
+    }
     task "${job_name}-stats" {
       artifact {
         source      = "git::https://github.com/FDio/csit"
@@ -200,7 +214,7 @@ job "${job_name}" {
       }
     }
   }
-  group "${job_name}-rls2302" {
+  group "${job_name}-coverage-rls2302" {
     restart {
       mode = "fail"
     }
@@ -405,6 +419,20 @@ job "${job_name}" {
         memory = ${memory}
       }
     }
+  }
+  group "${job_name}-iterative-rls2302" {
+    restart {
+      mode = "fail"
+    }
+    constraint {
+      attribute = "$${attr.cpu.arch}"
+      operator  = "!="
+      value     = "arm64"
+    }
+    constraint {
+      attribute = "$${node.class}"
+      value     = "builder"
+    }
     task "${job_name}-iterative-hoststack" {
       artifact {
         source      = "git::https://github.com/FDio/csit"

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf
index 24e7c94..b6e0011 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf
@@ -7,11 +7,11 @@ terraform {
   required_providers {
     nomad = {
       source  = "hashicorp/nomad"
-      version = ">= 1.4.16"
+      version = ">= 1.4.19"
     }
     vault = {
-      version = ">= 3.2.1"
+      version = ">= 3.12.0"
     }
   }
-  required_version = ">= 1.1.4"
+  required_version = ">= 1.3.7"
 }

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
index 9357c09..f6d318e 100644 (file)
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
@@ -27,7 +27,7 @@ variable "aws_default_region" {
 variable "cpu" {
   description = "Specifies the CPU required to run this task in MHz."
   type        = number
-  default     = 20000
+  default     = 10000
 }
 
 variable "cron" {
@@ -57,7 +57,7 @@ variable "job_name" {
 variable "memory" {
   description = "Specifies the memory required in MB."
   type        = number
-  default     = 60000
+  default     = 50000
 }
 
 variable "out_aws_access_key_id" {