From d6fd6ef312e88aca67d41c293baaf9bd4f5f2ae5 Mon Sep 17 00:00:00 2001 From: pmikus Date: Mon, 21 Aug 2023 07:46:49 +0000 Subject: [PATCH] fix(etl): Cron timings Signed-off-by: pmikus Change-Id: Iefe50300c31c999739a92d5fb578b9e45c47e84a --- .../app/.ebextensions/cron-linux.config | 2 +- .../conf/nomad/etl-trending-ndrpdr.hcl.tftpl | 2 ++ .../terraform-nomad-pyspark-etl/fdio/main.tf | 32 ++++++++++++---------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/csit.infra.dash/app/.ebextensions/cron-linux.config b/csit.infra.dash/app/.ebextensions/cron-linux.config index ae8c33c814..eb114d072d 100644 --- a/csit.infra.dash/app/.ebextensions/cron-linux.config +++ b/csit.infra.dash/app/.ebextensions/cron-linux.config @@ -7,7 +7,7 @@ files: SHELL=/bin/bash PATH=/sbin:/bin:/usr/sbin:/usr/bin MAILTO=root - 0 6 * * * root /bin/echo 'c' > /tmp/masterfifo + 30 2 * * * root /bin/echo 'c' > /tmp/masterfifo commands: remove_old_cron: diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl index 797de65944..8cd40f537e 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl @@ -31,6 +31,8 @@ job "${job_name}" { args = [ "--driver-memory", "30g", "--executor-memory", "30g", + "--executor-cores", "2", + "--master", "local[2]", "trending_ndrpdr.py" ] work_dir = "/local/csit/csit.infra.etl" diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf index cb6b3a0021..9dd43c681f 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf @@ -18,7 +18,7 @@ module "etl-stats" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-stats" } @@ -35,7 +35,7 @@ module "etl-trending-hoststack" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-trending-hoststack" } @@ -52,9 +52,10 @@ module "etl-trending-mrr" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-trending-mrr" + memory = 60000 } module "etl-trending-ndrpdr" { @@ -69,9 +70,10 @@ module "etl-trending-ndrpdr" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-trending-ndrpdr" + memory = 60000 } module "etl-iterative-hoststack-rls2310" { @@ -86,7 +88,7 @@ module "etl-iterative-hoststack-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-iterative-hoststack-rls2310" } @@ -103,7 +105,7 @@ module "etl-iterative-mrr-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-iterative-mrr-rls2310" } @@ -120,7 +122,7 @@ module "etl-iterative-ndrpdr-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-iterative-ndrpdr-rls2310" } @@ -137,7 +139,7 @@ module "etl-iterative-reconf-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-iterative-reconf-rls2310" } @@ -154,7 +156,7 @@ module "etl-iterative-soak-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-iterative-soak-rls2310" } @@ -171,7 +173,7 @@ module "etl-coverage-device-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-coverage-device-rls2310" } @@ -188,7 +190,7 @@ module "etl-coverage-hoststack-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-coverage-hoststack-rls2310" } @@ -205,7 +207,7 @@ module "etl-coverage-mrr-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-coverage-mrr-rls2310" } @@ -222,7 +224,7 @@ module "etl-coverage-ndrpdr-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-coverage-ndrpdr-rls2310" } @@ -239,7 +241,7 @@ module "etl-coverage-reconf-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-coverage-reconf-rls2310" } @@ -256,7 +258,7 @@ module "etl-coverage-soak-rls2310" { out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "@daily" + cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = "etl-coverage-soak-rls2310" } -- 2.16.6