From 34ec0f34987ceaec1f77cb56c4c672f01aa4dca4 Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Mon, 10 Feb 2020 08:42:33 +0000 Subject: [PATCH] Ansible: Optimizations Signed-off-by: Peter Mikus Change-Id: I56fb45fb22b924eec6ee6b6b7ad7874a07f1555c --- .../inventories/lf_inventory/group_vars/all.yaml | 2 +- .../ansible/roles/calibration/tasks/main.yaml | 33 ++++-- .../ansible/roles/calibration/tasks/x86_64.yaml | 24 ++-- .../ansible/roles/performance_tuning/files/90-csit | 80 ------------- .../roles/performance_tuning/tasks/main.yaml | 127 ++++++++++++++++++--- .../performance_tuning/tasks/turbo_boost.yaml | 8 +- .../roles/tg/files/csit-initialize-docker-tg.sh | 2 + .../ansible/roles/wrk/tasks/main.yaml | 4 +- 8 files changed, 164 insertions(+), 116 deletions(-) delete mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml index f24c6a7d42..0756621eef 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml @@ -2,4 +2,4 @@ # file: lf_inventory/group_vars/all.yaml # Ansible interpreter (for PIP) -ansible_python_interpreter: "python3" +ansible_python_interpreter: "/usr/bin/python3" diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml index 462f72788c..cf0f2450bb 100644 --- a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml @@ -9,18 +9,35 @@ tags: - install-dependencies -- name: Calibration - Run Spectre Meltdown Checker - raw: "wget -qO - https://meltdown.ovh | sudo bash -s - --no-color || true" - register: spectre_meltdown_output +- name: Calibration - Get Spectre Meltdown Checker + get_url: + url: "https://meltdown.ovh" + dest: "/opt/spectre-meltdown-checker.sh" + mode: "744" tags: - - run-spectre-meltdown-checker + - check-spectre-meltdown -- debug: var=spectre_meltdown_output.stdout_lines +- name: Calibration - Run Spectre Meltdown Checker + shell: "/opt/spectre-meltdown-checker.sh --no-color || true" + async: 60 + poll: 0 + register: spectre_meltdown_async tags: - - run-spectre-meltdown-checker + - check-spectre-meltdown - name: Calibration - {{ ansible_machine }} Specific include_tasks: "{{ ansible_machine }}.yaml" - when: ansible_machine == "x86_64" tags: - - run-jitter-tool + - check-machine-specific + - check-jitter-tool + +- name: Check sync status + async_status: + jid: "{{ spectre_meltdown_async.ansible_job_id }}" + register: "spectre_meltdown_poll_results" + until: spectre_meltdown_poll_results.finished + retries: 30 + +- debug: var=spectre_meltdown_poll_results.stdout_lines + tags: + - check-spectre-meltdown diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/x86_64.yaml index 1d9ed5ea08..90b1c954b5 100644 --- a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/x86_64.yaml +++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/x86_64.yaml @@ -6,20 +6,30 @@ repo: "https://gerrit.fd.io/r/pma_tools" dest: "{{ pma_directory }}" tags: - - run-jitter-tool + - check-jitter-tool - name: Calibration - Compile PMA Tool raw: "cd {{ pma_directory }}/jitter && make" tags: - - run-jitter-tool + - check-jitter-tool - name: Calibration - Run Jitter Tool - raw: "{{ pma_directory }}/jitter/jitter -c {{ jitter_core }} -i {{ jitter_iterations }} -f" + shell: "{{ pma_directory }}/jitter/jitter -c {{ jitter_core }} -i {{ jitter_iterations }} -f" become: yes - register: jitter_output + async: 60 + poll: 0 + ignore_errors: yes + register: jitter_async tags: - - run-jitter-tool + - check-jitter-tool -- debug: var=jitter_output.stdout_lines +- name: Check sync status + async_status: + jid: "{{ jitter_async.ansible_job_id }}" + register: "jitter_poll_results" + until: jitter_poll_results.finished + retries: 30 + +- debug: var=jitter_poll_results.stdout_lines tags: - - run-jitter-tool + - check-jitter-tool diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit deleted file mode 100644 index a0404c20d7..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit +++ /dev/null @@ -1,80 +0,0 @@ -# change the minimum size of the hugepage pool. -# 2G VPP, 16x2G for Fullbox VM, 2G reserve -vm.nr_hugepages={{ sysctl.vm.nr_hugepages }} - -# this file contains the maximum number of memory map areas a process -# may have. memory map areas are used as a side-effect of calling -# malloc, directly by mmap and mprotect, and also when loading shared -# libraries. -# -# while most applications need less than a thousand maps, certain -# programs, particularly malloc debuggers, may consume lots of them, -# e.g., up to one or two maps per allocation. -# must be greater than or equal to (2 * vm.nr_hugepages). -vm.max_map_count={{ sysctl.vm.max_map_count }} - -# hugetlb_shm_group contains group id that is allowed to create sysv -# shared memory segment using hugetlb page. -vm.hugetlb_shm_group=1000 - -# this control is used to define how aggressive the kernel will swap -# memory pages. higher values will increase agressiveness, lower values -# decrease the amount of swap. a value of 0 instructs the kernel not to -# initiate swap until the amount of free and file-backed pages is less -# than the high water mark in a zone. -vm.swappiness=0 - -# this parameter can be used to control the nmi watchdog -# (i.e. the hard lockup detector) on x86 systems. -# -# 0 - disable the hard lockup detector -# 1 - enable the hard lockup detector -# -# the hard lockup detector monitors each cpu for its ability to respond to -# timer interrupts. the mechanism utilizes cpu performance counter registers -# that are programmed to generate non-maskable interrupts (nmis) periodically -# while a cpu is busy. hence, the alternative name 'nmi watchdog'. -# -# the nmi watchdog is disabled by default if the kernel is running as a guest -# in a kvm virtual machine. this default can be overridden by adding -#kernel. nmi_watchdog=1 - -# shared memory max must be greator or equal to the total size of hugepages. -# for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024 -# if the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax) -# is greater than the calculated totalhugepagesize then set this parameter -# to current shmmax value. -kernel.shmmax=8589934592 - -# this option can be used to select the type of process address -# space randomization that is used in the system, for architectures -# that support this feature. -# 0 - turn the process address space randomization off. this is the -# default for architectures that do not support this feature anyways, -# and kernels that are booted with the "norandmaps" parameter. -kernel.randomize_va_space=0 - -# this parameter can be used to control the soft lockup detector. -# -# 0 - disable the soft lockup detector -# 1 - enable the soft lockup detector -# -# the soft lockup detector monitors cpus for threads that are hogging the cpus -# without rescheduling voluntarily, and thus prevent the 'watchdog/n' threads -# from running. the mechanism depends on the cpus ability to respond to timer -# interrupts which are needed for the 'watchdog/n' threads to be woken up by -# the watchdog timer function, otherwise the nmi watchdog - if enabled - can -# detect a hard lockup condition. -#kernel.soft_watchdog=0 - -# this value can be used to control on which cpus the watchdog may run. -# the default cpumask is all possible cores, but if no_hz_full is -# enabled in the kernel config, and cores are specified with the -# nohz_full= boot argument, those cores are excluded by default. -# offline cores can be included in this mask, and if the core is later -# brought online, the watchdog will be started based on the mask value. -# -# typically this value would only be touched in the nohz_full case -# to re-enable cores that by default were not running the watchdog, -# if a kernel lockup was suspected on those cores. -kernel.watchdog_cpumask={{ sysctl.kernel.watchdog_cpumask }} diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml index 77b2683124..9fedd27160 100644 --- a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml @@ -9,19 +9,19 @@ tags: - install-dependencies -- name: Performance Tuning - Configure {{ ansible_machine }} kernel parameters +- name: Performance Tuning - Configure {{ ansible_machine }} Kernel Parameters lineinfile: path: "/etc/default/grub" state: "present" regexp: "^GRUB_CMDLINE_LINUX=" - line: {{ grub_cmdline_linux[ansible_machine] }} + line: "{{ grub_cmdline_linux[ansible_machine] }}" notify: - "Update GRUB" - "Reboot server" tags: - set-grub -- name: Performance Tuning - Turbo boost +- name: Performance Tuning - Turbo Boost import_tasks: turbo_boost.yaml when: > cpu_microarchitecture == "skylake" or @@ -29,17 +29,116 @@ tags: - turbo-boost -- name: Performance Tuning - Copy CSIT sysctl file - template: - src: "files/90-csit" - dest: "/etc/sysctl.d/90-csit.conf" - owner: "root" - group: "root" - mode: "0644" +- name: Performance Tuning - Adjust nr_hugepages + # change the minimum size of the hugepage pool. + # 2G VPP, 4GB per VNF/CNF, 2G reserve + sysctl: + name: "vm.nr_hugepages" + value: "{{ sysctl.vm.nr_hugepages }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: Performance Tuning - Adjust max_map_count + # this file contains the maximum number of memory map areas a process + # may have. memory map areas are used as a side-effect of calling + # malloc, directly by mmap and mprotect, and also when loading shared + # libraries. + # + # while most applications need less than a thousand maps, certain + # programs, particularly malloc debuggers, may consume lots of them, + # e.g., up to one or two maps per allocation. + # must be greater than or equal to (2 * vm.nr_hugepages). + sysctl: + name: "vm.max_map_count" + value: "{{ sysctl.vm.nr_hugepages * 4 }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: Performance Tuning - Adjust hugetlb_shm_group + # hugetlb_shm_group contains group id that is allowed to create sysv + # shared memory segment using hugetlb page. + sysctl: + name: "vm.hugetlb_shm_group" + value: "1000" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: Performance Tuning - Adjust swappiness + # this control is used to define how aggressive the kernel will swap + # memory pages. higher values will increase agressiveness, lower values + # decrease the amount of swap. a value of 0 instructs the kernel not to + # initiate swap until the amount of free and file-backed pages is less + # than the high water mark in a zone. + sysctl: + name: "vm.swappiness" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: Performance Tuning - Adjust shmmax + # shared memory max must be greator or equal to the total size of hugepages. + # for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024 + # if the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax) + # is greater than the calculated totalhugepagesize then set this parameter + # to current shmmax value. + sysctl: + name: "kernel.shmmax" + value: "{{ sysctl.vm.nr_hugepages * 2 * 1024 * 1024 }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: Performance Tuning - Adjust watchdog_cpumask + # this value can be used to control on which cpus the watchdog may run. + # the default cpumask is all possible cores, but if no_hz_full is + # enabled in the kernel config, and cores are specified with the + # nohz_full= boot argument, those cores are excluded by default. + # offline cores can be included in this mask, and if the core is later + # brought online, the watchdog will be started based on the mask value. + # + # typically this value would only be touched in the nohz_full case + # to re-enable cores that by default were not running the watchdog, + # if a kernel lockup was suspected on those cores. + sysctl: + name: "kernel.watchdog_cpumask" + value: "{{ sysctl.kernel.watchdog_cpumask }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: Performance Tuning - Adjust randomize_va_space + # this option can be used to select the type of process address + # space randomization that is used in the system, for architectures + # that support this feature. + # 0 - turn the process address space randomization off. this is the + # default for architectures that do not support this feature anyways, + # and kernels that are booted with the "norandmaps" parameter. + sysctl: + name: "kernel.randomize_va_space" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" tags: - - copy-90-csit + - set-sysctl -- name: Performance Tuning - Copy cpufrequtils file +- name: Performance Tuning - Copy Cpufrequtils File copy: src: "files/cpufrequtils" dest: "/etc/default/cpufrequtils" @@ -49,14 +148,14 @@ tags: - copy-cpufrequtils -- name: Performance Tuning - Set ondemand service to disable +- name: Performance Tuning - Set Ondemand Service To Disable service: name: "ondemand" enabled: "no" tags: - set-ondemand -- name: Performance Tuning - Load kernel modules by default +- name: Performance Tuning - Load Kernel Modules By Default lineinfile: path: "/etc/modules" state: "present" diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml index f6633f5e82..310803ca5b 100644 --- a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml @@ -1,7 +1,7 @@ --- # file: roles/performance_tuning/tasks/turbo_boost.yaml -- name: Turbo boost - Install msr-tools +- name: Turbo Boost - Install msr-tools package: name: - "msr-tools" @@ -10,7 +10,7 @@ tags: - turbo-boost -- name: Turbo boost - Load msr by default +- name: Turbo Boost - Load msr By Default lineinfile: path: "/etc/modules" state: "present" @@ -18,7 +18,7 @@ tags: - turbo-boost -- name: Turbo boost - Custom startup service hook +- name: Turbo Boost - Custom Startup Service Hook copy: src: "files/disable-turbo-boost.service" dest: "/etc/systemd/system/disable-turbo-boost.service" @@ -28,7 +28,7 @@ tags: - turbo-boost -- name: Turbo boost - Custom startup service hook enable +- name: Turbo Boost - Custom Startup Service Hook Enable service: name: "disable-turbo-boost" enabled: yes diff --git a/resources/tools/testbed-setup/ansible/roles/tg/files/csit-initialize-docker-tg.sh b/resources/tools/testbed-setup/ansible/roles/tg/files/csit-initialize-docker-tg.sh index 7b90d20bda..e9f19fd899 100755 --- a/resources/tools/testbed-setup/ansible/roles/tg/files/csit-initialize-docker-tg.sh +++ b/resources/tools/testbed-setup/ansible/roles/tg/files/csit-initialize-docker-tg.sh @@ -44,6 +44,8 @@ case "${1:-start}" in dcr_stc_params+="--volume /dev:/dev " # Mount /opt/boot/ where VM kernel and initrd are located. dcr_stc_params+="--volume /opt:/opt " + # Mount /usr/local/bin/wrk where WRK is located. + dcr_stc_params+="--volume /usr/local/bin/wrk:/usr/local/bin/wrk " # Mount host hugepages for VMs. dcr_stc_params+="--volume /dev/hugepages:/dev/hugepages " diff --git a/resources/tools/testbed-setup/ansible/roles/wrk/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/wrk/tasks/main.yaml index d8d06145db..c2e42aa3f1 100644 --- a/resources/tools/testbed-setup/ansible/roles/wrk/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/wrk/tasks/main.yaml @@ -46,8 +46,8 @@ tags: - install-wrk -- name: WRK Install - Move Binary - command: "mv {{ wrk_target_dir }}/wrk-{{ wrk_version }}/wrk /usr/local/bin/" +- name: WRK Install - Copy Binary + command: "cp {{ wrk_target_dir }}/wrk-{{ wrk_version }}/wrk /usr/local/bin/" when: wrk_compiled tags: - install-wrk -- 2.16.6