From 8018da98e0f362bc69fc9600fac222a86fd46b5e Mon Sep 17 00:00:00 2001 From: Tomas Alexy Date: Thu, 1 Apr 2021 15:44:14 +0200 Subject: [PATCH] Infra: AWS Update to Ubuntu 20.04 - Change AMI image to Ubuntu 20.04 - Add calibration role - Add AWS vfio-patch for kernel 5.8+ - Prepare root module's main.tf to be used with environment variables Signed-off-by: Tomas Alexy Change-Id: I7db3f28ba573a5a8a1dc07179ef78ef34ce9ebf3 --- .../2n_aws_c5n/deploy/variables.tf | 17 +++++------- fdio.infra.terraform/2n_aws_c5n/main.tf | 15 +++++------ fdio.infra.terraform/2n_aws_c5n/variables.tf | 31 ++++++++++++++++++++++ .../3n_aws_c5n/deploy/variables.tf | 17 +++++------- fdio.infra.terraform/3n_aws_c5n/main.tf | 15 +++++------ fdio.infra.terraform/3n_aws_c5n/variables.tf | 31 ++++++++++++++++++++++ .../ansible/roles/aws/defaults/main.yaml | 1 - .../ansible/roles/aws/tasks/main.yaml | 30 +++++++++++++-------- .../ansible/roles/aws/tasks/ubuntu_bionic.yaml | 2 +- .../ansible/roles/aws/tasks/ubuntu_focal.yaml | 10 +++++++ .../ansible/roles/calibration/defaults/main.yaml | 2 +- .../ansible/roles/calibration/tasks/main.yaml | 2 ++ resources/tools/testbed-setup/ansible/sut.yaml | 6 ++--- resources/tools/testbed-setup/ansible/tg.yaml | 6 ++--- 14 files changed, 125 insertions(+), 60 deletions(-) create mode 100644 fdio.infra.terraform/2n_aws_c5n/variables.tf create mode 100644 fdio.infra.terraform/3n_aws_c5n/variables.tf create mode 100644 resources/tools/testbed-setup/ansible/roles/aws/tasks/ubuntu_focal.yaml diff --git a/fdio.infra.terraform/2n_aws_c5n/deploy/variables.tf b/fdio.infra.terraform/2n_aws_c5n/deploy/variables.tf index d1ff1d6575..53efe6fb1d 100644 --- a/fdio.infra.terraform/2n_aws_c5n/deploy/variables.tf +++ b/fdio.infra.terraform/2n_aws_c5n/deploy/variables.tf @@ -1,37 +1,32 @@ variable "region" { description = "AWS Region" type = string - default = "eu-central-1" } variable "ami_image" { description = "AWS AMI image name" type = string - default = "ami-0b418580298265d5c" } variable "testbed_name" { description = "Testbed name" type = string - default = "testbed1" -} - -variable "topology_name" { - description = "Prefix used when creating a topology file" - type = string - default = "2n_aws_c5n" } variable "instance_type" { description = "AWS instance type" type = string - default = "c5n.4xlarge" } variable "avail_zone" { description = "AWS availability zone" type = string - default = "eu-central-1a" +} + +variable "topology_name" { + description = "Prefix used when creating a topology file" + type = string + default = "2n_aws_c5n" } variable "environment_name" { diff --git a/fdio.infra.terraform/2n_aws_c5n/main.tf b/fdio.infra.terraform/2n_aws_c5n/main.tf index a73288d9e8..7a090675e7 100644 --- a/fdio.infra.terraform/2n_aws_c5n/main.tf +++ b/fdio.infra.terraform/2n_aws_c5n/main.tf @@ -1,19 +1,18 @@ module "deploy" { source = "./deploy" - # TODO: Use ENV variable for testbed_name for dynamic deployment - testbed_name = "testbed1" + # Parameters starting with var. can be set using "TF_VAR_*" environment variables + # or -var parameter when running "terraform apply", for default values see ./variables.tf + testbed_name = var.testbed_name topology_name = "2n_aws_c5n" environment_name = "CSIT-AWS" resources_name_prefix = "CSIT_2n_aws_c5n" # AWS general - region = "eu-central-1" - avail_zone = "eu-central-1a" - instance_type = "c5n.4xlarge" - ami_image = "ami-0b418580298265d5c" - # eu-central-1/bionic-18.04-amd64-hvm-ssd-20200112 - # kernel 4.15.0-1057-aws (~4.15.0-74) + region = var.region + avail_zone = var.avail_zone + instance_type = var.instance_type + ami_image = var.ami_image # AWS Network vpc_cidr_mgmt = "192.168.0.0/24" diff --git a/fdio.infra.terraform/2n_aws_c5n/variables.tf b/fdio.infra.terraform/2n_aws_c5n/variables.tf new file mode 100644 index 0000000000..15e718bdb0 --- /dev/null +++ b/fdio.infra.terraform/2n_aws_c5n/variables.tf @@ -0,0 +1,31 @@ +variable "region" { + description = "AWS Region" + type = string + default = "eu-central-1" +} + +variable "avail_zone" { + description = "AWS availability zone" + type = string + default = "eu-central-1a" +} + +variable "ami_image" { + # eu-central-1/focal-20.04-amd64-hvm-ssd-20210119.1 + # kernel 5.4.0-1035-aws (~5.4.0-65) + description = "AWS AMI image ID" + type = string + default = "ami-0a875db8a031a9efb" +} + +variable "instance_type" { + description = "AWS instance type" + type = string + default = "c5n.4xlarge" +} + +variable "testbed_name" { + description = "Testbed name" + type = string + default = "testbed1" +} diff --git a/fdio.infra.terraform/3n_aws_c5n/deploy/variables.tf b/fdio.infra.terraform/3n_aws_c5n/deploy/variables.tf index f6fa83122f..38a9133eeb 100644 --- a/fdio.infra.terraform/3n_aws_c5n/deploy/variables.tf +++ b/fdio.infra.terraform/3n_aws_c5n/deploy/variables.tf @@ -1,37 +1,32 @@ variable "region" { description = "AWS Region" type = string - default = "eu-central-1" } variable "ami_image" { description = "AWS AMI image name" type = string - default = "ami-0b418580298265d5c" } variable "testbed_name" { description = "Testbed name" type = string - default = "testbed1" -} - -variable "topology_name" { - description = "Prefix used when creating a topology file" - type = string - default = "3n_aws_c5n" } variable "instance_type" { description = "AWS instance type" type = string - default = "c5n.4xlarge" } variable "avail_zone" { description = "AWS availability zone" type = string - default = "eu-central-1a" +} + +variable "topology_name" { + description = "Prefix used when creating a topology file" + type = string + default = "3n_aws_c5n" } variable "environment_name" { diff --git a/fdio.infra.terraform/3n_aws_c5n/main.tf b/fdio.infra.terraform/3n_aws_c5n/main.tf index 0e757b89bb..0aca7af05a 100644 --- a/fdio.infra.terraform/3n_aws_c5n/main.tf +++ b/fdio.infra.terraform/3n_aws_c5n/main.tf @@ -1,19 +1,18 @@ module "deploy" { source = "./deploy" - # TODO: Use ENV variable for testbed_name for dynamic deployment - testbed_name = "testbed1" + # Parameters starting with var. can be set using "TF_VAR_*" environment variables + # or -var parameter when running "terraform apply", for default values see ./variables.tf + testbed_name = var.testbed_name topology_name = "3n_aws_c5n" environment_name = "CSIT-AWS" resources_name_prefix = "CSIT_3n_aws_c5n" # AWS general - region = "eu-central-1" - avail_zone = "eu-central-1a" - instance_type = "c5n.4xlarge" - ami_image = "ami-0b418580298265d5c" - # eu-central-1/bionic-18.04-amd64-hvm-ssd-20200112 - # kernel 4.15.0-1057-aws (~4.15.0-74) + region = var.region + avail_zone = var.avail_zone + instance_type = var.instance_type + ami_image = var.ami_image # AWS Network vpc_cidr_mgmt = "192.168.0.0/24" diff --git a/fdio.infra.terraform/3n_aws_c5n/variables.tf b/fdio.infra.terraform/3n_aws_c5n/variables.tf new file mode 100644 index 0000000000..15e718bdb0 --- /dev/null +++ b/fdio.infra.terraform/3n_aws_c5n/variables.tf @@ -0,0 +1,31 @@ +variable "region" { + description = "AWS Region" + type = string + default = "eu-central-1" +} + +variable "avail_zone" { + description = "AWS availability zone" + type = string + default = "eu-central-1a" +} + +variable "ami_image" { + # eu-central-1/focal-20.04-amd64-hvm-ssd-20210119.1 + # kernel 5.4.0-1035-aws (~5.4.0-65) + description = "AWS AMI image ID" + type = string + default = "ami-0a875db8a031a9efb" +} + +variable "instance_type" { + description = "AWS instance type" + type = string + default = "c5n.4xlarge" +} + +variable "testbed_name" { + description = "Testbed name" + type = string + default = "testbed1" +} diff --git a/resources/tools/testbed-setup/ansible/roles/aws/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/aws/defaults/main.yaml index 706e552f96..d4ea91afd4 100644 --- a/resources/tools/testbed-setup/ansible/roles/aws/defaults/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/aws/defaults/main.yaml @@ -1,3 +1,2 @@ --- # file: roles/aws/defaults/main.yaml - diff --git a/resources/tools/testbed-setup/ansible/roles/aws/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/aws/tasks/main.yaml index deca0670ec..2d209762c3 100644 --- a/resources/tools/testbed-setup/ansible/roles/aws/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/aws/tasks/main.yaml @@ -4,7 +4,7 @@ - name: Edit repositories include_tasks: "{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml" tags: - - edit-repo + - aws-edit-repo - name: Get vfio-pci With WC Patcher get_url: @@ -12,27 +12,35 @@ dest: "/opt/get-vfio-with-wc.sh" mode: "744" tags: - - vfio-aws-patch + - aws-vfio-patch - name: Create vfio-pci Patch Directory file: path: "/opt/patches/" state: "directory" tags: - - vfio-aws-patch + - aws-vfio-patch -- name: Get vfio-pci WC Patch +- name: Get vfio-pci WC Patch >=4.10 get_url: url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/linux-4.10-vfio-wc.patch" dest: "/opt/patches/linux-4.10-vfio-wc.patch" mode: "744" tags: - - vfio-aws-patch + - aws-vfio-patch + +- name: Get vfio-pci WC Patch >=5.8 + get_url: + url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/linux-5.8-vfio-wc.patch" + dest: "/opt/patches/linux-5.8-vfio-wc.patch" + mode: "744" + tags: + - aws-vfio-patch - name: Compile vfio-pci With WC Patch shell: "/bin/bash /opt/get-vfio-with-wc.sh" tags: - - vfio-aws-patch + - aws-vfio-patch - name: Load Kernel Modules By Default lineinfile: @@ -43,7 +51,7 @@ - "vfio-pci" - "igb_uio" tags: - - load-kernel-modules + - aws-load-kernel-modules - name: Add Kernel Modules Options (igb_uio) lineinfile: @@ -54,7 +62,7 @@ with_items: - "options igb_uio wc_activate=1" tags: - - load-kernel-modules + - aws-load-kernel-modules - name: Add Kernel Modules Options (vfio-pci) lineinfile: @@ -65,14 +73,14 @@ with_items: - "options vfio enable_unsafe_noiommu_mode=1" tags: - - load-kernel-modules + - aws-load-kernel-modules - name: Reload systemd-modules systemd: name: "systemd-modules-load" state: "restarted" tags: - - reload-systemd-modules + - aws-reload-systemd-modules - name: Performance Tuning - Adjust nr_hugepages sysctl: @@ -82,4 +90,4 @@ sysctl_file: "/etc/sysctl.d/90-csit.conf" reload: "yes" tags: - - set-sysctl + - aws-set-hugepages diff --git a/resources/tools/testbed-setup/ansible/roles/aws/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/aws/tasks/ubuntu_bionic.yaml index ba6e107e45..bca1cf5095 100644 --- a/resources/tools/testbed-setup/ansible/roles/aws/tasks/ubuntu_bionic.yaml +++ b/resources/tools/testbed-setup/ansible/roles/aws/tasks/ubuntu_bionic.yaml @@ -7,4 +7,4 @@ state: "present" update_cache: yes tags: - - enable-src-repo \ No newline at end of file + - aws-enable-src-repo diff --git a/resources/tools/testbed-setup/ansible/roles/aws/tasks/ubuntu_focal.yaml b/resources/tools/testbed-setup/ansible/roles/aws/tasks/ubuntu_focal.yaml new file mode 100644 index 0000000000..a8cc56c0c4 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/aws/tasks/ubuntu_focal.yaml @@ -0,0 +1,10 @@ +--- +# file: roles/aws/tasks/ubuntu_focal.yaml.yaml + +- name: Enable deb-src APT Repository + apt_repository: + repo: "deb-src http://archive.ubuntu.com/ubuntu focal main" + state: "present" + update_cache: yes + tags: + - aws-enable-src-repo diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/defaults/main.yaml index 5eb945328c..020c0119b1 100644 --- a/resources/tools/testbed-setup/ansible/roles/calibration/defaults/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/calibration/defaults/main.yaml @@ -38,7 +38,7 @@ kernel_version_by_distro_by_arch: x86_64: - "5.4.0-65-generic" - "5.3.0-1020-azure" - - "5.3.0-1017-aws" + - "5.4.0-1035-aws" aarch64: - "5.4.0-65-generic" diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml index 5d0c3b18ff..696f1c9265 100644 --- a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml @@ -40,6 +40,8 @@ fail_msg: "Kernel parameters!" success_msg: "Kernel parameters match." loop: "{{ grub.keys()|sort }}" + when: + - grub is defined tags: - check-kernel-params diff --git a/resources/tools/testbed-setup/ansible/sut.yaml b/resources/tools/testbed-setup/ansible/sut.yaml index abc25a279c..37a96c28d0 100644 --- a/resources/tools/testbed-setup/ansible/sut.yaml +++ b/resources/tools/testbed-setup/ansible/sut.yaml @@ -64,10 +64,8 @@ tags: aws - role: cleanup tags: cleanup - # TODO: 'Check Kernel Parameters' failing in - # resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml - # - role: calibration - # tags: calibration + - role: calibration + tags: calibration - hosts: sut_azure become: yes diff --git a/resources/tools/testbed-setup/ansible/tg.yaml b/resources/tools/testbed-setup/ansible/tg.yaml index 69728a694e..3d5137f35e 100644 --- a/resources/tools/testbed-setup/ansible/tg.yaml +++ b/resources/tools/testbed-setup/ansible/tg.yaml @@ -70,10 +70,8 @@ tags: aws - role: cleanup tags: cleanup - # TODO: 'Check Kernel Parameters' failing in - # resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml - # - role: calibration - # tags: calibration + - role: calibration + tags: calibration - hosts: tg_azure become: yes -- 2.16.6