From: Peter Mikus Date: Fri, 24 Jan 2020 14:56:57 +0000 (+0000) Subject: Ansible: Cleanup and speedup X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=2cf0755933dac3e95c10928347996bad3cbe0a42 Ansible: Cleanup and speedup Signed-off-by: Peter Mikus Change-Id: Ia2ce482798204ad426f93f334c97e28eb51139fd --- diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh index 2329beba70..2098f789cf 100644 --- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh +++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh @@ -44,7 +44,6 @@ activate_virtualenv || die generate_tests || die archive_tests || die reserve_and_cleanup_testbed || die -ansible_hosts "calibration" || die select_tags || die compose_pybot_arguments || die run_pybot || die diff --git a/resources/libraries/bash/function/ansible.sh b/resources/libraries/bash/function/ansible.sh index 431acc7c5d..56665dfa1c 100644 --- a/resources/libraries/bash/function/ansible.sh +++ b/resources/libraries/bash/function/ansible.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,10 +16,9 @@ set -exuo pipefail -function ansible_hosts () { +function ansible_adhoc () { - # Run ansible playbook on hosts in working topology file. Ansible scope is - # determined by tags passed as parameters to this function. + # Run ansible ad-hoc command module on hosts in working topology file. # # Variable read: # - ${WORKING_TOPOLOGY} - Reserved working topology. @@ -31,9 +30,33 @@ function ansible_hosts () { die "Please install sshpass!" fi - if ! installed ansible-playbook; then - # TODO: Consider moving to requirements.txt? - pip install ansible==2.7.8 || die "Install ansible via PIP failed!" + hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || { + die "Failed to read hosts from working topology!" + } + pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!" + ANSIBLE_STDOUT_CALLBACK=yaml ansible \ + --vault-password-file=vault_pass \ + --extra-vars '@vault.yml' \ + --inventory inventories/lf_inventory/hosts site.yaml \ + --limit "$(echo ${hosts[@]//\"})" \ + --module-name shell \ + --args \"$(echo $@)\" || die "Failed to run ansible on host!" + popd || die "Popd failed!" +} + +function ansible_playbook () { + + # Run ansible playbook on hosts in working topology file. Ansible scope is + # determined by tags passed as parameters to this function. + # + # Variable read: + # - ${WORKING_TOPOLOGY} - Reserved working topology. + # - ${TOOLS_DIR} - CSIT tools directory, where testbed-setup is located. + + set -exuo pipefail + + if ! installed sshpass; then + die "Please install sshpass!" fi hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || { @@ -49,6 +72,7 @@ function ansible_hosts () { popd || die "Popd failed!" } + function installed () { # Check if the given utility is installed. Fail if not installed. diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index 38ba60116a..f89f71dc65 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -561,7 +561,7 @@ function reserve_and_cleanup_testbed () { # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. # Functions called: # - die - Print to stderr and exit. - # - ansible_hosts - Perform an action using ansible, see ansible.sh + # - ansible_playbook - Perform an action using ansible, see ansible.sh # Traps registered: # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}. @@ -588,9 +588,9 @@ function reserve_and_cleanup_testbed () { } die "Trap attempt failed, unreserve succeeded. Aborting." } - # Cleanup check. + # Cleanup + calibration checks. set +e - ansible_hosts "cleanup" + ansible_playbook "cleanup, calibration" result="$?" set -e if [[ "${result}" == "0" ]]; then @@ -1033,7 +1033,7 @@ function untrap_and_unreserve_testbed () { # - EXIT - Failure to untrap is reported, but ignored otherwise. # Functions called: # - die - Print to stderr and exit. - # - ansible_hosts - Perform an action using ansible, see ansible.sh + # - ansible_playbook - Perform an action using ansible, see ansible.sh set -xo pipefail set +eu # We do not want to exit early in a "teardown" function. @@ -1043,7 +1043,7 @@ function untrap_and_unreserve_testbed () { set -eu warn "Testbed looks unreserved already. Trap removal failed before?" else - ansible_hosts "cleanup" || true + ansible_playbook "cleanup" || true python3 "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || { die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2 } diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml new file mode 100644 index 0000000000..ca4e75d268 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml @@ -0,0 +1,2 @@ +--- +# file: roles/calibration/tasks/aarch64.yaml diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml index 34c987bdcb..c3c96ac499 100644 --- a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml @@ -9,6 +9,6 @@ - debug: var=spectre_meltdown_output.stdout_lines tags: run-spectre-meltdown-checker -- name: x86 specific - import_tasks: x86_64.yaml - when: ansible_machine == 'x86_64' +- name: Machine specifics + include_tasks: '{{ ansible_machine }}.yaml' + tags: run-jitter-tool diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml index 20306100e6..1cd64351a8 100644 --- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml @@ -1,24 +1,29 @@ --- # file: roles/cleanup/tasks/kill_containers.yaml -- name: Kill container - Get running Docker containers - shell: "docker ps -aq" - register: running_containers - changed_when: no - tags: kill-containers +- name: Kill containers + block: + - name: Kill container - Get running Docker containers + shell: "docker ps -aq" + register: running_containers + changed_when: no + tags: kill-containers -- name: Kill container - Remove all Docker containers - shell: "docker rm --force {{ item }}" - with_items: "{{ running_containers.stdout_lines }}" - tags: kill-containers + - name: Kill container - Remove all Docker containers + shell: "docker rm --force {{ item }}" + with_items: "{{ running_containers.stdout_lines }}" + tags: kill-containers -- name: Kill container - Get running LXC containers - shell: "lxc-ls" - register: running_containers - changed_when: no - tags: kill-containers + - name: Kill container - Get running LXC containers + shell: "lxc-ls" + register: running_containers + changed_when: no + tags: kill-containers -- name: Kill container - Remove all LXC containers - shell: "lxc-destroy --force -n {{ item }}" - with_items: "{{ running_containers.stdout_lines }}" - tags: kill-containers + - name: Kill container - Remove all LXC containers + shell: "lxc-destroy --force -n {{ item }}" + with_items: "{{ running_containers.stdout_lines }}" + tags: kill-containers + rescue: + - fail: + msg: "Kill containers failed!" diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml index 4a1180b77f..a593fc7616 100644 --- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml @@ -1,27 +1,32 @@ --- # file: roles/cleanup/tasks/kill_process.yaml -- name: Kill process - Get pid of {{ process }} - shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'" - when: > - process is defined and process != "" - register: running_processes - tags: kill-process +- name: Kill process - {{ process }} + block: + - name: Kill process - Get pid of {{ process }} + shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'" + when: > + process is defined and process != "" + register: running_processes + tags: kill-process -- name: Kill process - Safe kill {{ process }} - shell: "kill {{ item }}" - with_items: "{{ running_processes.stdout_lines }}" - tags: kill-process + - name: Kill process - Safe kill {{ process }} + shell: "kill {{ item }}" + with_items: "{{ running_processes.stdout_lines }}" + tags: kill-process -- wait_for: - path: "/proc/{{ item }}/status" - state: absent - with_items: "{{ running_processes.stdout_lines }}" - ignore_errors: yes - register: killed_processes - tags: kill-process + - wait_for: + path: "/proc/{{ item }}/status" + state: absent + with_items: "{{ running_processes.stdout_lines }}" + ignore_errors: yes + register: killed_processes + tags: kill-process -- name: Kill process - Force kill {{ process }} - shell: "kill -9 {{ item }}" - with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}" - tags: kill-process + - name: Kill process - Force kill {{ process }} + shell: "kill -9 {{ item }}" + with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}" + tags: kill-process + rescue: + - fail: + msg: "Kill process {{ process }} failed!" diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml index b3f8fffea2..c24b5e6a7f 100644 --- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml @@ -1,58 +1,60 @@ --- # file: roles/cleanup/tasks/sut.yaml -- name: Kill processes - qemu - import_tasks: kill_process.yaml - vars: - process: "qemu" - tags: kill-process - -- name: Kill processes - l3fwd - import_tasks: kill_process.yaml - vars: - process: "l3fwd" - tags: kill-process - -- name: Kill processes - testpmd - import_tasks: kill_process.yaml - vars: - process: "testpmd" - tags: kill-process - -- name: Kill processes - iperf3 - import_tasks: kill_process.yaml - vars: - process: "iperf3" - tags: kill-process - -- name: Kill processes - vpp_echo - import_tasks: kill_process.yaml - vars: - process: "vpp_echo" - tags: kill-process - -- name: Remove file or dir - Core zip file - file: - state: absent - path: "/tmp/*tar.lzo.lrz.xz*" - tags: remove-file-dir - -- name: Remove file or dir - Core dump file - file: - state: absent - path: "/tmp/*core*" - tags: remove-file-dir - -- name: Kill containers - Remove all containers - import_tasks: kill_containers.yaml - tags: kill-containers - -- name: Kubernetes - Reset - raw: 'kubeadm reset --force' - tags: kill-kubernetes - -- name: Remove packages - Remove VPP - import_tasks: remove_package.yaml - vars: - package: "*vpp*" - tags: remove-package +- name: Host cleanup + block: + - name: Kill processes - qemu + import_tasks: kill_process.yaml + vars: + process: "qemu" + tags: kill-process + + - name: Kill processes - l3fwd + import_tasks: kill_process.yaml + vars: + process: "l3fwd" + tags: kill-process + + - name: Kill processes - testpmd + import_tasks: kill_process.yaml + vars: + process: "testpmd" + tags: kill-process + + - name: Kill processes - iperf3 + import_tasks: kill_process.yaml + vars: + process: "iperf3" + tags: kill-process + + - name: Kill processes - vpp_echo + import_tasks: kill_process.yaml + vars: + process: "vpp_echo" + tags: kill-process + + - name: Remove file or dir - Core zip file + file: + state: absent + path: "/tmp/*tar.lzo.lrz.xz*" + tags: remove-file-dir + + - name: Remove file or dir - Core dump file + file: + state: absent + path: "/tmp/*core*" + tags: remove-file-dir + + - name: Kill containers - Remove all containers + import_tasks: kill_containers.yaml + tags: kill-containers + + - name: Kubernetes - Reset + raw: 'kubeadm reset --force' + tags: kill-kubernetes + + - name: Remove packages - Remove VPP + import_tasks: remove_package.yaml + vars: + package: "*vpp*" + tags: remove-package