From: Peter Mikus Date: Wed, 27 Apr 2022 13:28:42 +0000 (+0200) Subject: feat(infra): Terraform updates X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=373375dbbc35bbcb5c5a3d4ab3342efc4870bb51;ds=sidebyside feat(infra): Terraform updates Signed-off-by: Peter Mikus Change-Id: I3d7ecad9125e3ff55c0b4e1da5cda5a1ff22d690 --- diff --git a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/main.tf b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/main.tf index 1de8c391b1..7ecca06394 100644 --- a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/main.tf +++ b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/main.tf @@ -1,3 +1,8 @@ +locals { + name = "fdio-csit-dash-app" + version = "fdio-csit-dash-app-${var.version}" +} + data "vault_aws_access_credentials" "creds" { backend = "${var.vault_name}-path" role = "${var.vault_name}-role" @@ -6,6 +11,6 @@ data "vault_aws_access_credentials" "creds" { module "elastic_beanstalk_application_version" { source = "../terraform-aws-elastic-beanstalk-application-version" application_description = "FD.io CSIT Results Dashboard" - application_name = "fdio-csit-dash-app" - application_version_name = "fdio-csit-dash-app-base" + application_name = local.name + application_version_name = local.version } diff --git a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf index 7147227213..cf89f28fcb 100644 --- a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf +++ b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf @@ -26,3 +26,9 @@ variable "vault_name" { type = string default = "dynamic-aws-creds-vault-fdio-csit-jenkins" } + +variable "version" { + description = "Application version string." + type = number + default = 1 +} diff --git a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/versions.tf b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/versions.tf index 4afbbc00a7..4b05b16b27 100644 --- a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/versions.tf +++ b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/versions.tf @@ -2,7 +2,7 @@ terraform { backend "consul" { address = "10.32.8.14:8500" scheme = "http" - path = "terraform/dash" + path = "terraform/app-base" } required_providers { aws = { diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl index 208fb0a59f..e0b56c1843 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl @@ -172,147 +172,147 @@ job "${job_name}" { } } } - group "${job_name}-rls2202" { - # The restart stanza configures a tasks's behavior on task failure. Restarts - # happen on the client that is running the task. - # - # https://www.nomadproject.io/docs/job-specification/restart - # - restart { - mode = "fail" - } - - # The constraint allows restricting the set of eligible nodes. Constraints - # may filter on attributes or client metadata. - # - # For more information and examples on the "volume" stanza, please see - # the online documentation at: - # - # https://www.nomadproject.io/docs/job-specification/constraint - # - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - - constraint { - attribute = "$${node.class}" - value = "builder" - } - - # The "task" stanza creates an individual unit of work, such as a Docker - # container, web application, or batch processing. - # - # https://www.nomadproject.io/docs/job-specification/task.html - # - task "${job_name}-coverage" { - # The artifact stanza instructs Nomad to fetch and unpack a remote - # resource, such as a file, tarball, or binary. - # - # https://www.nomadproject.io/docs/job-specification/artifact - # - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - - # The "driver" parameter specifies the task driver that should be used to - # run the task. - driver = "docker" - - # The "config" stanza specifies the driver configuration, which is passed - # directly to the driver to start the task. The details of configurations - # are specific to each driver, so please see specific driver - # documentation for more information. - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "coverage_rls2202.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - - # The env stanza configures a list of environment variables to populate - # the task's environment before starting. - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - - # The "resources" stanza describes the requirements a task needs to - # execute. Resource requirements include memory, network, cpu, and more. - # This ensures the task will execute on a machine that contains enough - # resource capacity. - # - # https://www.nomadproject.io/docs/job-specification/resources - # - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-iterative" { - # The artifact stanza instructs Nomad to fetch and unpack a remote - # resource, such as a file, tarball, or binary. - # - # https://www.nomadproject.io/docs/job-specification/artifact - # - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - - # The "driver" parameter specifies the task driver that should be used to - # run the task. - driver = "docker" - - # The "config" stanza specifies the driver configuration, which is passed - # directly to the driver to start the task. The details of configurations - # are specific to each driver, so please see specific driver - # documentation for more information. - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "iterative_rls2202.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - - # The env stanza configures a list of environment variables to populate - # the task's environment before starting. - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - - # The "resources" stanza describes the requirements a task needs to - # execute. Resource requirements include memory, network, cpu, and more. - # This ensures the task will execute on a machine that contains enough - # resource capacity. - # - # https://www.nomadproject.io/docs/job-specification/resources - # - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } +# group "${job_name}-rls2202" { +# # The restart stanza configures a tasks's behavior on task failure. Restarts +# # happen on the client that is running the task. +# # +# # https://www.nomadproject.io/docs/job-specification/restart +# # +# restart { +# mode = "fail" +# } +# +# # The constraint allows restricting the set of eligible nodes. Constraints +# # may filter on attributes or client metadata. +# # +# # For more information and examples on the "volume" stanza, please see +# # the online documentation at: +# # +# # https://www.nomadproject.io/docs/job-specification/constraint +# # +# constraint { +# attribute = "$${attr.cpu.arch}" +# operator = "!=" +# value = "arm64" +# } +# +# constraint { +# attribute = "$${node.class}" +# value = "builder" +# } +# +# # The "task" stanza creates an individual unit of work, such as a Docker +# # container, web application, or batch processing. +# # +# # https://www.nomadproject.io/docs/job-specification/task.html +# # +# task "${job_name}-coverage" { +# # The artifact stanza instructs Nomad to fetch and unpack a remote +# # resource, such as a file, tarball, or binary. +# # +# # https://www.nomadproject.io/docs/job-specification/artifact +# # +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# +# # The "driver" parameter specifies the task driver that should be used to +# # run the task. +# driver = "docker" +# +# # The "config" stanza specifies the driver configuration, which is passed +# # directly to the driver to start the task. The details of configurations +# # are specific to each driver, so please see specific driver +# # documentation for more information. +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "coverage_rls2202.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# +# # The env stanza configures a list of environment variables to populate +# # the task's environment before starting. +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# +# # The "resources" stanza describes the requirements a task needs to +# # execute. Resource requirements include memory, network, cpu, and more. +# # This ensures the task will execute on a machine that contains enough +# # resource capacity. +# # +# # https://www.nomadproject.io/docs/job-specification/resources +# # +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-iterative" { +# # The artifact stanza instructs Nomad to fetch and unpack a remote +# # resource, such as a file, tarball, or binary. +# # +# # https://www.nomadproject.io/docs/job-specification/artifact +# # +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# +# # The "driver" parameter specifies the task driver that should be used to +# # run the task. +# driver = "docker" +# +# # The "config" stanza specifies the driver configuration, which is passed +# # directly to the driver to start the task. The details of configurations +# # are specific to each driver, so please see specific driver +# # documentation for more information. +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "iterative_rls2202.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# +# # The env stanza configures a list of environment variables to populate +# # the task's environment before starting. +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# +# # The "resources" stanza describes the requirements a task needs to +# # execute. Resource requirements include memory, network, cpu, and more. +# # This ensures the task will execute on a machine that contains enough +# # resource capacity. +# # +# # https://www.nomadproject.io/docs/job-specification/resources +# # +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# } } diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf b/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf index ec03c7c9ee..2b62d8d1ee 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf +++ b/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf @@ -2,7 +2,7 @@ terraform { backend "consul" { address = "consul.service.consul:8500" scheme = "http" - path = "fdio/terraform/1n/nomad" + path = "terraform/aws-secret-backend" } required_providers { vault = {