hostname: "t1-tg1"
isolcpus: "1-17,19-35"
+cfs_cpus: "0,18"
hostname: "t1-sut1"
isolcpus: "1-17,19-35"
+cfs_cpus: "0,18"
hostname: "t1-sut2"
isolcpus: "1-17,19-35"
+cfs_cpus: "0,18"
hostname: "t2-tg1"
isolcpus: "1-17,19-35"
+cfs_cpus: "0,18"
hostname: "t2-sut1"
isolcpus: "1-17,19-35"
+cfs_cpus: "0,18"
hostname: "t2-sut2"
isolcpus: "1-17,19-35"
+cfs_cpus: "0,18"
hostname: "t3-tg1"
isolcpus: "1-17,19-35"
+cfs_cpus: "0,18"
hostname: "t3-sut1"
isolcpus: "1-17,19-35"
+cfs_cpus: "0,18"
hostname: "t3-sut2"
isolcpus: "1-17,19-35"
+cfs_cpus: "0,18"
--- /dev/null
+---
+# file: host_vars/10.30.51.36.yaml
+
+hostname: "s17-t33-sut1"
+isolcpus: "1-15,17-31,33-47,49-63"
+cfs_cpus: "0,16,32,48"
+taishan_workaround: True
--- /dev/null
+---
+# file: host_vars/10.30.51.37.yaml
+
+hostname: "s18-t33-sut2"
+isolcpus: "1-15,17-31,33-47,49-63"
+cfs_cpus: "0,16,32,48"
+taishan_workaround: True
hostname: "s3-t21-sut1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s4-t21-tg1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s11-t31-sut1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s12-t31-sut2"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s13-t31-tg1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
---
# file: host_vars/10.30.51.49.yaml
-hostname: "s17-t33-tg1"
+hostname: "s19-t33t34-tg1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s1-t11-sut1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s2-t12-sut1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s5-t22-sut1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s6-t22-tg1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s7-t23-sut1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s8-t23-tg1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s9-t24-sut1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s10-t24-tg1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s14-t32-sut1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s15-t32-sut2"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hostname: "s16-t32-tg1"
isolcpus: "1-27,29-55,57-83,85-111"
+cfs_cpus: "0,28,56,84"
hosts:
10.30.51.45: null #s4-t21-tg1
10.30.51.48: null #s13-t31-tg1
- 10.30.51.49: null #s17-t33-tg1
+ 10.30.51.49: null #s19-t33t34-tg1
10.30.51.53: null #s6-t22-tg1
10.30.51.55: null #s8-t23-tg1
10.30.51.57: null #s10-t24-tg1
hosts:
10.30.51.50: null #s1-t11-sut1
10.30.51.51: null #s2-t12-sut1
-# arm:
-# children:
+ taishan:
+ children:
# tg:
# hosts:
-# sut:
-# hosts:
+ sut:
+ hosts:
+ 10.30.51.36: null #s17-t33-sut1
+ 10.30.51.37: null #s18-t33-sut2
--- /dev/null
+deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic main restricted
+deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic main restricted
+
+## Major bug fix updates produced after the final release of the
+## distribution.
+deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted
+deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted
+
+## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
+## team. Also, please note that software in universe WILL NOT receive any
+## review or updates from the Ubuntu security team.
+deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic universe
+deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic universe
+deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates universe
+deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates universe
+
+## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
+## team, and may not be under a free licence. Please satisfy yourself as to
+## your rights to use the software. Also, please note that software in
+## multiverse WILL NOT receive any review or updates from the Ubuntu
+## security team.
+deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic multiverse
+deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic multiverse
+deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates multiverse
+deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates multiverse
+
+## N.B. software from this repository may not have been tested as
+## extensively as that contained in the main release, although it includes
+## newer versions of some applications which may provide useful features.
+## Also, please note that software in backports WILL NOT receive any review
+## or updates from the Ubuntu security team.
+deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted universe multiverse
+deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted universe multiverse
+
+deb http://ports.ubuntu.com/ubuntu-ports bionic-security main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports bionic-security main restricted
+deb http://ports.ubuntu.com/ubuntu-ports bionic-security universe
+deb-src http://ports.ubuntu.com/ubuntu-ports bionic-security universe
+deb http://ports.ubuntu.com/ubuntu-ports bionic-security multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports bionic-security multiverse
+
+## Uncomment the following two lines to add software from Canonical's
+## 'partner' repository.
+## This software is not part of Ubuntu, but is offered by Canonical and the
+## respective vendors as a service to Ubuntu users.
+# deb http://archive.canonical.com/ubuntu bionic partner
+# deb-src http://archive.canonical.com/ubuntu bionic partner
# file: roles/common/tasks/main.yaml
- name: Ubuntu specific
- import_tasks: ubuntu_x86_64.yaml
- when: ansible_distribution|lower == 'ubuntu' and ansible_machine == 'x86_64'
+ import_tasks: ubuntu.yaml
+ when: ansible_distribution|lower == 'ubuntu'
- name: Set hostname
hostname:
line: '{{ ansible_default_ipv4.address }} {{ hostname }}.linuxfoundation.org'
tags: set-hostname
-- name: Set sudoers
+- name: Set sudoers admin
lineinfile:
path: '/etc/sudoers'
state: 'present'
validate: '/usr/sbin/visudo -cf %s'
tags: set-sudoers
-- name: Set sudoers
+- name: Set sudoers sudo
lineinfile:
path: '/etc/sudoers'
state: 'present'
---
-# file: roles/common/tasks/ubuntu_x86_64.yaml
+# file: roles/common/tasks/ubuntu.yaml
-- name: Copy apt sources file
+- name: Copy aarch64 apt sources file
+ template:
+ src: 'files/apt-sources-arm.list'
+ dest: '/etc/apt/sources.list'
+ tags: copy-apt-sources
+ when: ansible_machine == 'aarch64'
+
+- name: Copy x86_64 apt sources file
template:
src: 'files/apt-sources.list'
dest: '/etc/apt/sources.list'
tags: copy-apt-sources
+ when: ansible_machine == 'x86_64'
- name: Install python-apt
apt:
- name: Install qemu
apt:
- name: 'qemu-system-x86'
+ name: 'qemu-system'
state: 'present'
update_cache: True
tags: install-qemu
--- /dev/null
+[Unit]
+Description=Numa node workaround configuration
+
+[Service]
+ExecStart=/bin/sh -c 'echo 1 > /sys/bus/pci/devices/000a:11:00.0/numa_node'
+ExecStart=/bin/sh -c 'echo 1 > /sys/bus/pci/devices/000a:11:00.1/numa_node'
+Type=oneshot
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
# file: roles/sut/tasks/main.yaml
- name: Ubuntu specific
- import_tasks: ubuntu_x86_64.yaml
- when: ansible_distribution|lower == 'ubuntu' and ansible_machine == 'x86_64'
+ import_tasks: ubuntu.yaml
+ when: ansible_distribution|lower == 'ubuntu'
+
+- name: Taishan aarch64 specific
+ import_tasks: taishan_workaround.yaml
+ when: taishan_workaround is defined
- name: Copy 80-vpp.conf
file:
--- /dev/null
+---
+# file: roles/sut/tasks/taishan_workaround.yaml
+
+- name: Ensure systemd directory exists
+ file:
+ path: "/etc/systemd/system"
+ state: "directory"
+ owner: "root"
+ group: "root"
+ mode: "0755"
+ tags: taishan-workaround
+
+- name: Copy systemd numa config unit file
+ template:
+ src: "files/taishan_workaround.service"
+ dest: "/etc/systemd/system/nic-numa-config.service"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ register: numa_config_service
+ tags: taishan-workaround
+
+- name: Reload systemd daemon
+ command: "systemctl daemon-reload"
+ when: (numa_config_service and numa_config_service is changed)
+ tags: taishan-workaround
+
+- name: Enable numa config service
+ command: "systemctl enable nic-numa-config.service"
+ tags: taishan-workaround
---
-# file: roles/sut/tasks/ubuntu_x86_64.yaml
+# file: roles/sut/tasks/ubuntu.yaml
- name: Install DKMS
apt:
# - "NO_PROXY=localhost,127.0.0.1"
docker_daemon_environment: []
-docker_apt_key: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88"
-docker_repository: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} {{ docker_channel }}"
+docker_repository: "deb https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} {{ docker_channel }}"
docker_apt_package_name: "{{ docker_version }}~{{ docker_edition }}~3-0~{{ ansible_distribution | lower }}"
apt_cache_time: 86400
kubernetes_channel: "main"
kubernetes_version: "1.11.0-00"
-kubernetes_apt_key: "54A647F9048D5688D7DA2ABE6A030B21BA07F4FB"
kubernetes_repository: "deb http://apt.kubernetes.io/ kubernetes-xenial {{ kubernetes_channel }}"
kubernetes_apt_package_name: "{{ kubernetes_version }}"
# typically this value would only be touched in the nohz_full case
# to re-enable cores that by default were not running the watchdog,
# if a kernel lockup was suspected on those cores.
-kernel.watchdog_cpumask=0,18
+kernel.watchdog_cpumask={{ cfs_cpus }}
+++ /dev/null
-# This file describes the network interfaces available on your system
-# and how to activate them. For more information, see interfaces(5).
-
-# The loopback network interface
-auto lo
-iface lo inet loopback
-
-# The primary network interface
-auto {{ ansible_default_ipv4["interface"] }}
-iface {{ ansible_default_ipv4["interface"] }} inet static
- address {{ ansible_default_ipv4["address"] }}
- netmask {{ ansible_default_ipv4["netmask"] }}
- gateway {{ ansible_default_ipv4["gateway"] }}
- dns-nameservers 199.204.44.24 199.204.47.54
--- /dev/null
+# This file describes the network interfaces available on your system
+# For more information, see netplan(5).
+network:
+ version: 2
+ renderer: networkd
+ ethernets:
+ {{ ansible_default_ipv4["interface"] }}:
+ addresses: [ {{ (ansible_default_ipv4.address + '/' + ansible_default_ipv4.netmask) | ipaddr('host/prefix') }} ]
+ gateway4: {{ ansible_default_ipv4["gateway"] }}
+ nameservers:
+ addresses: [ 199.204.44.24, 199.204.47.54 ]
--- /dev/null
+---
+# file: roles/tg_sut/tasks/aarch64.yaml
+
+- name: Configure aarch64 kernel parameters
+ lineinfile:
+ path: '/etc/default/grub'
+ state: 'present'
+ regexp: '^GRUB_CMDLINE_LINUX='
+ line: 'GRUB_CMDLINE_LINUX="isolcpus={{ isolcpus }} nohz_full={{ isolcpus }} rcu_nocbs={{ isolcpus }} intel_iommu=on nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1"'
+ notify: ['Update GRUB']
+ tags: set-grub
+
# file: roles/tg_sut/tasks/main.yaml
- name: Ubuntu specific
- import_tasks: ubuntu_x86_64.yaml
- when: ansible_distribution|lower == 'ubuntu' and ansible_machine == 'x86_64'
+ import_tasks: ubuntu.yaml
+ when: ansible_distribution|lower == 'ubuntu'
-- name: Copy interfaces file
+- name: x86 specific
+ import_tasks: x86_64.yaml
+ when: ansible_machine == 'x86_64'
+
+- name: skylake specific
+ import_tasks: skylake.yaml
+ when: ("skylake" in groups) and inventory_hostname in groups['skylake']
+
+- name: aarch specific
+ import_tasks: aarch64.yaml
+ when: ansible_machine == 'aarch64'
+
+- name: Copy netplan network config file
template:
- src: 'files/interfaces_physical'
- dest: '/etc/network/interfaces'
+ src: 'files/netplan_config'
+ dest: '/etc/netplan/01-netcfg.yaml'
owner: 'root'
group: 'root'
mode: '0644'
tags: copy-interface-file
-- name: Copy sysctl file
+- name: Copy CSIT sysctl file
template:
src: 'files/90-csit'
dest: '/etc/sysctl.d/90-csit.conf'
dest: '/tmp/requirements.txt'
tags: copy-pip
-- name: Set isolcpus and pstate parameter
- lineinfile:
- path: '/etc/default/grub'
- state: 'present'
- regexp: '^GRUB_CMDLINE_LINUX='
- line: 'GRUB_CMDLINE_LINUX="isolcpus={{ isolcpus }} nohz_full={{ isolcpus }} rcu_nocbs={{ isolcpus }} numa_balancing=disable intel_pstate=disable intel_iommu=on iommu=pt nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1 intel_idle.max_cstate=1 hpet=disable tsc=reliable mce=off"'
- notify: ['Update GRUB']
- tags: set-grub
-
- name: Set ondemand service to disable
service:
name: 'ondemand'
state: 'present'
line: 'vfio-pci'
tags: load-vfio-pci
-
-- name: Load msr by default
- lineinfile:
- path: '/etc/modules'
- state: 'present'
- line: 'msr'
- tags: disable-turbo-boost
-
-- name: Disable Turbo-Boost by service
- copy:
- src: 'files/disable-turbo-boost.service'
- dest: '/etc/systemd/system/disable-turbo-boost.service'
- owner: 'root'
- group: 'root'
- mode: '0644'
- when: >
- inventory_hostname in groups['skylake']
- tags: disable-turbo-boost
-
-- name: Disable Turbo-Boost by service on startup
- service:
- name: disable-turbo-boost
- enabled: yes
- when: >
- inventory_hostname in groups['skylake']
- tags: disable-turbo-boost
--- /dev/null
+---
+# file: roles/tg_sut/tasks/skylake.yaml
+
+- name: Disable Skylake Turbo-Boost by service
+ copy:
+ src: 'files/disable-turbo-boost.service'
+ dest: '/etc/systemd/system/disable-turbo-boost.service'
+ owner: 'root'
+ group: 'root'
+ mode: '0644'
+ tags: disable-turbo-boost
+
+- name: Disable Skylake Turbo-Boost by service on startup
+ service:
+ name: disable-turbo-boost
+ enabled: yes
+ tags: disable-turbo-boost
---
-# file: roles/tg_sut/tasks/ubuntu_x86_64.yaml
+# file: roles/tg_sut/tasks/ubuntu.yaml
- name: Install python-dev
apt:
- "cron"
tags: install-docker
-- name: Install upstream APT GPG key
+- name: Install Docker APT GPG key
apt_key:
- id: "{{ docker_apt_key }}"
- keyserver: "{{ ansible_local.core.keyserver
- if (ansible_local|d() and ansible_local.core|d() and
- ansible_local.core.keyserver)
- else 'hkp://pool.sks-keyservers.net' }}"
+ url: https://download.docker.com/linux/ubuntu/gpg
state: "present"
tags: install-docker
-- name: Install upstream APT repository
+- name: Install Docker APT repository
apt_repository:
repo: "{{ docker_repository }}"
state: "present"
cache_valid_time: "{{ apt_cache_time }}"
tags: install-docker
-- name: Remove Upstart config file
+- name: Remove Upstart docker config file
file:
path: "/etc/default/docker"
state: "absent"
mode: "0755"
tags: ensure-docker
-- name: Copy systemd unit file
+- name: Copy systemd docker unit file
template:
src: "templates/docker.service.j2"
dest: "/etc/systemd/system/docker.service"
when: docker_users
tags: set-docker
-- name: Install upstream APT GPG key
+- name: Install kubernetes APT GPG key
apt_key:
- id: "{{ kubernetes_apt_key }}"
- keyserver: "{{ ansible_local.core.keyserver
- if (ansible_local|d() and ansible_local.core|d() and
- ansible_local.core.keyserver)
- else 'hkp://pool.sks-keyservers.net' }}"
+ url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
state: "present"
tags: install-kubernetes
-- name: Install upstream APT repository
+- name: Install kubernetes APT repository
apt_repository:
repo: "{{ kubernetes_repository }}"
state: "present"
--- /dev/null
+---
+# file: roles/tg_sut/tasks/x86_64.yaml
+
+- name: Configure x86_64 kernel parameters
+ lineinfile:
+ path: '/etc/default/grub'
+ state: 'present'
+ regexp: '^GRUB_CMDLINE_LINUX='
+ line: 'GRUB_CMDLINE_LINUX="isolcpus={{ isolcpus }} nohz_full={{ isolcpus }} rcu_nocbs={{ isolcpus }} numa_balancing=disable intel_pstate=disable intel_iommu=on iommu=pt nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1 intel_idle.max_cstate=1 hpet=disable tsc=reliable mce=off"'
+ notify: ['Update GRUB']
+ tags: set-grub
+
+- name: Load msr by default
+ lineinfile:
+ path: '/etc/modules'
+ state: 'present'
+ line: 'msr'
+ tags: disable-turbo-boost