From d01411c3c4af6c724a3800c621804ea979818d6d Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Thu, 10 Oct 2019 15:31:28 +0000 Subject: [PATCH] Cleanup via Ansible + Remove dependency on topo_ scripts that depends on custom SSH() that depends on framework itself. This way the cleanup is independent of failure in our SSH libs. + Simple ansible command can do cleanup of a machine: ansible-playbook --inventory inventories/lf_inventory/hosts site.yaml \ --limit '10.32.8.18' --tags 'cleanup' + Add vpp_device reset and cleanup. + Remove historical scripts. - Still in testing beta phase. - Need to add SRIOV cleanup. Signed-off-by: Peter Mikus Change-Id: I68e23304c7ad01041f51263c328c6e8d9b555cb7 --- resources/libraries/bash/function/ansible.sh | 3 +- resources/libraries/bash/function/common.sh | 4 +- resources/tools/scripts/rename_robot_keywords.py | 243 --------------------- resources/tools/scripts/robot_output_parser.py | 208 ------------------ resources/tools/scripts/topo_container_copy.py | 137 ------------ resources/tools/scripts/topo_installation.py | 171 --------------- .../ansible/roles/cleanup/files/reset_vppdevice.sh | 113 ++++++++++ .../roles/cleanup/tasks/kill_containers.yaml | 28 +++ .../ansible/roles/cleanup/tasks/kill_process.yaml | 27 +++ .../ansible/roles/cleanup/tasks/main.yaml | 31 +++ .../roles/cleanup/tasks/remove_package.yaml | 31 +++ .../ansible/roles/cleanup/tasks/sut.yaml | 52 +++++ .../ansible/roles/cleanup/tasks/tg.yaml | 14 ++ .../ansible/roles/cleanup/tasks/vpp_device.yaml | 15 ++ resources/tools/testbed-setup/ansible/sut.yaml | 2 + resources/tools/testbed-setup/ansible/tg.yaml | 2 + .../tools/testbed-setup/ansible/vpp_device.yaml | 2 + 17 files changed, 320 insertions(+), 763 deletions(-) delete mode 100755 resources/tools/scripts/rename_robot_keywords.py delete mode 100755 resources/tools/scripts/robot_output_parser.py delete mode 100644 resources/tools/scripts/topo_container_copy.py delete mode 100755 resources/tools/scripts/topo_installation.py create mode 100644 resources/tools/testbed-setup/ansible/roles/cleanup/files/reset_vppdevice.sh create mode 100644 resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/cleanup/tasks/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/cleanup/tasks/remove_package.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/cleanup/tasks/vpp_device.yaml diff --git a/resources/libraries/bash/function/ansible.sh b/resources/libraries/bash/function/ansible.sh index 5bf122e4b0..431acc7c5d 100644 --- a/resources/libraries/bash/function/ansible.sh +++ b/resources/libraries/bash/function/ansible.sh @@ -28,8 +28,7 @@ function ansible_hosts () { set -exuo pipefail if ! installed sshpass; then - sudo apt-get update -y || die "apt-get update failed!" - sudo apt-get install -y sshpass || die "Install sshpass failed!" + die "Please install sshpass!" fi if ! installed ansible-playbook; then diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index f9c9e2ea91..f5e1111a17 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -604,7 +604,7 @@ function reserve_and_cleanup_testbed () { } # Cleanup check. set +e - cleanup_topo + ansible_hosts "cleanup" result="$?" set -e if [[ "${result}" == "0" ]]; then @@ -1040,7 +1040,7 @@ function untrap_and_unreserve_testbed () { set -eu warn "Testbed looks unreserved already. Trap removal failed before?" else - cleanup_topo || true + ansible_hosts "cleanup" || true python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || { die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2 } diff --git a/resources/tools/scripts/rename_robot_keywords.py b/resources/tools/scripts/rename_robot_keywords.py deleted file mode 100755 index 9f27b4aaec..0000000000 --- a/resources/tools/scripts/rename_robot_keywords.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2017 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This script renames the given robot keywords in the given directory -recursively. - -Example: - - ./rename_robot_keywords.py -i kws.csv -s ";" -d ~/ws/vpp/git/csit/ -vvv - - Input file "kws.csv" is CSV file exported from e.g. MS Excel. Its structure - must be: - - - - One keyword per line. - -""" - -import argparse -import sys -import re -from os import walk, rename -from os.path import join - - -def time_interval(func): - """Decorator function to measure the time spent by the decorated function. - - :param func: Decorated function. - :type func: Callable object. - :returns: Wrapper function. - :rtype: Callable object. - """ - - import time - - def wrapper(*args, **kwargs): - start = time.clock() - result = func(*args, **kwargs) - stop = time.clock() - print("\nRenaming done in {:.5g} seconds\n". - format(stop - start)) - return result - return wrapper - - -def get_files(path, extension): - """Generates the list of files to process. - - :param path: Path to files. - :param extension: Extension of files to process. If it is the empty string, - all files will be processed. - :type path: str - :type extension: str - :returns: List of files to process. - :rtype: list - """ - - file_list = list() - for root, dirs, files in walk(path): - for filename in files: - if extension: - if filename.endswith(extension): - file_list.append(join(root, filename)) - else: - file_list.append(join(root, filename)) - - return file_list - - -def read_keywords(args): - """This function reads the keywords from the input file and creates: - - - a dictionary where the key is the old name and the value is the new name, - these keywords will be further processed. - - a list of keywords which will not be processed, typically keywords with - argument(s) in its names. - - a list of duplicates - duplicated keyword names or names which are parts - of another keyword name, they will not be processed. - - :param args: Parsed arguments. - :type args: ArgumentParser - :returns: keyword names - dictionary where the key is the old name and the - value is the new name; ignored keyword names - list of keywords which will - not be processed; duplicates - duplicated keyword names or names which are - parts of another keyword name, they will not be processed. - :rtype: tuple(dict, list, list) - """ - - kw_names = dict() - ignored_kw_names = list() - duplicates = list() - - for line in args.input: - old_name, new_name = line.split(args.separator) - if '$' in old_name: - ignored_kw_names.append((old_name, new_name[:-1])) - elif old_name in kw_names.keys(): - duplicates.append((old_name, new_name[:-1])) - else: - kw_names[old_name] = new_name[:-1] - - # Remove duplicates: - for old_name, _ in duplicates: - new_name = kw_names.pop(old_name, None) - if new_name: - duplicates.append((old_name, new_name)) - - # Find KW names which are parts of other KW names: - for old_name in kw_names.keys(): - count = 0 - for key in kw_names.keys(): - if old_name in key: - count += 1 - if old_name in kw_names[key]: - if old_name != key: - count += 1 - if count > 1: - duplicates.append((old_name, kw_names[old_name])) - kw_names.pop(old_name) - - return kw_names, ignored_kw_names, duplicates - - -def rename_keywords(file_list, kw_names, args): - """Rename the keywords in specified files. - - :param file_list: List of files to be processed. - :param kw_names: Dictionary where the key is the old name and the value is - the new name - :type file_list: list - :type kw_names: dict - """ - - kw_not_found = list() - - for old_name, new_name in kw_names.items(): - kw_found = False - if args.verbosity > 0: - print("\nFrom: {}\n To: {}\n".format(old_name, new_name)) - for file_name in file_list: - tmp_file_name = file_name + ".new" - with open(file_name) as file_read: - file_write = open(tmp_file_name, 'w') - occurrences = 0 - for line in file_read: - new_line = re.sub(old_name, new_name, line) - file_write.write(new_line) - if new_line != line: - occurrences += 1 - if occurrences: - kw_found = True - if args.verbosity > 1: - print(" {:3d}: {}".format(occurrences, file_name)) - file_write.close() - rename(tmp_file_name, file_name) - if not kw_found: - kw_not_found.append(old_name) - - if args.verbosity > 0: - print("\nKeywords not found:") - for item in kw_not_found: - print(" {}".format(item)) - - -def parse_args(): - """Parse arguments from command line. - - :returns: Parsed arguments. - :rtype: ArgumentParser - """ - - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse. - RawDescriptionHelpFormatter) - parser.add_argument("-i", "--input", - required=True, - type=argparse.FileType('r'), - help="Text file with the old keyword name and the new " - "keyword name separated by separator per line.") - parser.add_argument("-s", "--separator", - default=";", - type=str, - help="Separator which separates the old and the new " - "keyword name.") - parser.add_argument("-d", "--dir", - required=True, - type=str, - help="Directory with robot files where the keywords " - "should be recursively searched.") - parser.add_argument("-v", "--verbosity", action="count", - help="Set the output verbosity.") - return parser.parse_args() - - -@time_interval -def main(): - """Main function.""" - - args = parse_args() - - kw_names, ignored_kw_names, duplicates = read_keywords(args) - - file_list = get_files(args.dir, "robot") - - if args.verbosity > 2: - print("\nList of files to be processed:") - for item in file_list: - print(" {}".format(item)) - print("\n{} files to be processed.\n".format(len(file_list))) - - print("\nList of keywords to be renamed:") - for item in kw_names: - print(" {}".format(item)) - print("\n{} keywords to be renamed.\n".format(len(kw_names))) - - rename_keywords(file_list, kw_names, args) - - if args.verbosity >= 0: - print("\nIgnored keywords: ({})".format(len(ignored_kw_names))) - for old, new in ignored_kw_names: - print(" From: {}\n To: {}\n".format(old, new)) - - print("\nIgnored duplicates ({}):".format(len(duplicates))) - for old, new in duplicates: - print(" From: {}\n To: {}\n".format(old, new)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/resources/tools/scripts/robot_output_parser.py b/resources/tools/scripts/robot_output_parser.py deleted file mode 100755 index b9ad8f8aa9..0000000000 --- a/resources/tools/scripts/robot_output_parser.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2016 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Script parses the data taken by robot framework (output.xml) and dumps -interested values into XML output file.""" - -import argparse -import re -import sys -import xml.etree.ElementTree as ET - -from robot.api import ExecutionResult, ResultVisitor - - -class ExecutionChecker(ResultVisitor): - """Iterates through test cases.""" - - tc_regexp = re.compile(ur'^tc\d+-((\d+)B|IMIX)-(\d)t(\d)c-(.*)') - rate_regexp = re.compile(ur'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)[\D\d]*') - lat_regexp = re.compile(ur'^[\D\d]*'\ - ur'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','\ - ur'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]\s\n'\ - ur'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','\ - ur'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]\s\n'\ - ur'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','\ - ur'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]') - - def __init__(self, args): - self.root = ET.Element('build', - attrib={'vdevice': args.vdevice}) - - def visit_suite(self, suite): - """Implements traversing through the suite and its direct children. - - :param suite: Suite to process. - :type suite: Suite - :return: Nothing. - """ - if self.start_suite(suite) is not False: - suite.suites.visit(self) - suite.tests.visit(self) - self.end_suite(suite) - - def start_suite(self, suite): - """Called when suite starts. - - :param suite: Suite to process. - :type suite: Suite - :return: Nothing. - """ - pass - - def end_suite(self, suite): - """Called when suite ends. - - :param suite: Suite to process. - :type suite: Suite - :return: Nothing. - """ - pass - - def visit_test(self, test): - """Implements traversing through the test. - - :param test: Test to process. - :type test: Test - :return: Nothing. - """ - if self.start_test(test) is not False: - self.end_test(test) - - def start_test(self, test): - """Called when test starts. - - :param test: Test to process. - :type test: Test - :return: Nothing. - """ - if any("NDRPDRDISC" in tag for tag in test.tags): - if test.status == 'PASS': - tags = [] - for tag in test.tags: - tags.append(tag) - - test_elem = ET.SubElement( - self.root, "S" + test.parent.name.replace(" ", "")) - test_elem.attrib['name'] = test.parent.name - test_elem.attrib['framesize'] = str(re.search( - self.tc_regexp, test.name).group(1)) - test_elem.attrib['threads'] = str(re.search( - self.tc_regexp, test.name).group(3)) - test_elem.attrib['cores'] = str(re.search( - self.tc_regexp, test.name).group(4)) - if any("NDRDISC" in tag for tag in test.tags): - try: - test_elem.attrib['lat_100'] = str(re.search( - self.lat_regexp, test.message).group(1)) + '/' +\ - str(re.search(self.lat_regexp, test.message). - group(2)) - except AttributeError: - test_elem.attrib['lat_100'] = "-1/-1/-1/-1/-1/-1" - try: - test_elem.attrib['lat_50'] = str(re.search( - self.lat_regexp, test.message).group(3)) + '/' +\ - str(re.search(self.lat_regexp, test.message). - group(4)) - except AttributeError: - test_elem.attrib['lat_50'] = "-1/-1/-1/-1/-1/-1" - try: - test_elem.attrib['lat_10'] = str(re.search( - self.lat_regexp, test.message).group(5)) + '/' +\ - str(re.search(self.lat_regexp, test.message). - group(6)) - except AttributeError: - test_elem.attrib['lat_10'] = "-1/-1/-1/-1/-1/-1" - test_elem.attrib['tags'] = ', '.join(tags) - try: - test_elem.text = str(re.search( - self.rate_regexp, test.message).group(1)) - except AttributeError: - test_elem.text = "-1" - - def end_test(self, test): - """Called when test ends. - - :param test: Test to process. - :type test: Test - :return: Nothing. - """ - pass - - -def parse_tests(args): - """Process data from robot output.xml file and return XML data. - - :param args: Parsed arguments. - :type args: ArgumentParser - - :return: XML formatted output. - :rtype: ElementTree - """ - - result = ExecutionResult(args.input) - checker = ExecutionChecker(args) - result.visit(checker) - - return checker.root - - -def print_error(msg): - """Print error message on stderr. - - :param msg: Error message to print. - :type msg: str - :return: nothing - """ - - sys.stderr.write(msg + '\n') - - -def parse_args(): - """Parse arguments from cmd line. - - :return: Parsed arguments. - :rtype ArgumentParser - """ - - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input", - required=True, - type=argparse.FileType('r'), - help="Robot XML log file") - parser.add_argument("-o", "--output", - required=True, - type=argparse.FileType('w'), - help="XML output file") - parser.add_argument("-v", "--vdevice", - required=False, - default="", - type=str, - help="VPP version") - - return parser.parse_args() - - -def main(): - """Main function.""" - - args = parse_args() - - root = parse_tests(args) - ET.ElementTree.write(ET.ElementTree(root), args.output) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/resources/tools/scripts/topo_container_copy.py b/resources/tools/scripts/topo_container_copy.py deleted file mode 100644 index 83599b4444..0000000000 --- a/resources/tools/scripts/topo_container_copy.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2017 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This script provides copy and load of Docker container images. - As destinations are used all DUT nodes from the topology file.""" - -import sys -import argparse -from yaml import load - -from resources.libraries.python.ssh import SSH - - -def ssh_no_error(ssh, cmd, sudo=False): - """Execute a command over ssh channel, and log and exit if the command - fails. - - :param ssh: SSH() object connected to a node. - :param cmd: Command line to execute on remote node. - :param sudo: Run command with sudo privileges. - :type ssh: SSH() object - :type cmd: str - :type sudo: bool - :returns: stdout from the SSH command. - :rtype: str - :raises RuntimeError: In case of unexpected ssh command failure - """ - if sudo: - ret, stdo, stde = ssh.exec_command_sudo(cmd, timeout=60) - else: - ret, stdo, stde = ssh.exec_command(cmd, timeout=60) - - if ret != 0: - print('Command execution failed: "{}"'.format(cmd)) - print('stdout: {0}'.format(stdo)) - print('stderr: {0}'.format(stde)) - raise RuntimeError('Unexpected ssh command failure') - - return stdo - - -def ssh_ignore_error(ssh, cmd, sudo=False): - """Execute a command over ssh channel, ignore errors. - - :param ssh: SSH() object connected to a node. - :param cmd: Command line to execute on remote node. - :param sudo: Run command with sudo privileges. - :type ssh: SSH() object - :type cmd: str - :type sudo: bool - :returns: stdout from the SSH command. - :rtype: str - """ - if sudo: - ret, stdo, stde = ssh.exec_command_sudo(cmd) - else: - ret, stdo, stde = ssh.exec_command(cmd) - - if ret != 0: - print('Command execution failed: "{}"'.format(cmd)) - print('stdout: {0}'.format(stdo)) - print('stderr: {0}'.format(stde)) - - return stdo - - -def main(): - """Copy and load of Docker image.""" - parser = argparse.ArgumentParser() - parser.add_argument("-t", "--topo", required=True, - help="Topology file") - parser.add_argument("-d", "--directory", required=True, - help="Destination directory") - parser.add_argument("-i", "--images", required=False, nargs='+', - help="Images paths to copy") - parser.add_argument("-c", "--cancel", help="Cancel all", - action="store_true") - - args = parser.parse_args() - topology_file = args.topo - images = args.images - directory = args.directory - cancel_all = args.cancel - - work_file = open(topology_file) - topology = load(work_file.read())['nodes'] - - ssh = SSH() - for node in topology: - if topology[node]['type'] == "DUT": - print("###TI host: {host}".format(host=topology[node]['host'])) - ssh.connect(topology[node]) - - if cancel_all: - # Remove destination directory on DUT - cmd = "rm -r {directory}".format(directory=directory) - stdout = ssh_ignore_error(ssh, cmd) - print("###TI {stdout}".format(stdout=stdout)) - - else: - # Create installation directory on DUT - cmd = "rm -r {directory}; mkdir {directory}"\ - .format(directory=directory) - stdout = ssh_no_error(ssh, cmd) - print("###TI {stdout}".format(stdout=stdout)) - - # Copy images from local path to destination dir - for image in images: - print("###TI scp: {}".format(image)) - ssh.scp(local_path=image, remote_path=directory) - - # Load image to Docker. - cmd = "for f in {directory}/*.tar.gz; do "\ - "sudo docker load -i $f; done".format(directory=directory) - stdout = ssh_no_error(ssh, cmd) - print("###TI {}".format(stdout)) - - # Remove images from Docker. - cmd = "docker rmi $(sudo docker images -f 'dangling=true' -q)" - stdout = ssh_ignore_error(ssh, cmd, sudo=True) - print("###TI {}".format(stdout)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/resources/tools/scripts/topo_installation.py b/resources/tools/scripts/topo_installation.py deleted file mode 100755 index 5c91abbd0f..0000000000 --- a/resources/tools/scripts/topo_installation.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This script provides copy and installation of VPP build deb packages. - As destinations are used all DUT nodes from the topology file.""" - -import sys -import argparse -from yaml import load - -from resources.libraries.python.ssh import SSH - - -def ssh_no_error(ssh, cmd, sudo=False): - """Execute a command over ssh channel, and log and exit if the command - fails. - - :param ssh: SSH() object connected to a node. - :param cmd: Command line to execute on remote node. - :type ssh: SSH() object - :type cmd: str - :return: stdout from the SSH command. - :rtype: str - """ - - if sudo: - ret, stdo, stde = ssh.exec_command_sudo(cmd, timeout=60) - else: - ret, stdo, stde = ssh.exec_command(cmd, timeout=60) - - if ret != 0: - print 'Command execution failed: "{}"'.format(cmd) - print 'stdout: {0}'.format(stdo) - print 'stderr: {0}'.format(stde) - raise RuntimeError('Unexpected ssh command failure') - - return stdo - - -def ssh_ignore_error(ssh, cmd, sudo=False): - """Execute a command over ssh channel, ignore errors. - - :param ssh: SSH() object connected to a node. - :param cmd: Command line to execute on remote node. - :type ssh: SSH() object - :type cmd: str - :return: stdout from the SSH command. - :rtype: str - """ - - if sudo: - ret, stdo, stde = ssh.exec_command_sudo(cmd) - else: - ret, stdo, stde = ssh.exec_command(cmd) - - if ret != 0: - print 'Command execution failed: "{}"'.format(cmd) - print 'stdout: {0}'.format(stdo) - print 'stderr: {0}'.format(stde) - - return stdo - - -def main(): - """Copy and installation of VPP packages.""" - - parser = argparse.ArgumentParser() - parser.add_argument("-t", "--topo", required=True, - help="Topology file") - parser.add_argument("-d", "--directory", required=True, - help="Installation directory") - parser.add_argument("-p", "--packages", required=False, nargs='+', - help="Packages paths to copy") - parser.add_argument("-c", "--cancel", help="Cancel installation", - action="store_true") - parser.add_argument("-hc", "--honeycomb", help="Include Honeycomb package.", - required=False, default=False) - - args = parser.parse_args() - topology_file = args.topo - packages = args.packages - install_dir = args.directory - cancel_installation = args.cancel - honeycomb = args.honeycomb - - work_file = open(topology_file) - topology = load(work_file.read())['nodes'] - - def fix_interrupted(package): - """If there are interrupted installations, clean them up.""" - - cmd = "dpkg -l | grep {0}".format(package) - ret, _, _ = ssh.exec_command(cmd) - if ret == 0: - # Try to fix interrupted installations - cmd = 'dpkg --configure -a' - stdout = ssh_no_error(ssh, cmd, sudo=True) - print "###TI {}".format(stdout) - # Try to remove installed packages - cmd = 'apt-get purge -y "{0}.*"'.format(package) - stdout = ssh_no_error(ssh, cmd, sudo=True) - print "###TI {}".format(stdout) - - ssh = SSH() - for node in topology: - if topology[node]['type'] == "DUT": - print "###TI host: {}".format(topology[node]['host']) - ssh.connect(topology[node]) - - if cancel_installation: - # Remove installation directory on DUT - cmd = "rm -r {}".format(install_dir) - stdout = ssh_ignore_error(ssh, cmd) - print "###TI {}".format(stdout) - - if honeycomb: - fix_interrupted("honeycomb") - # remove HC logs - cmd = "rm -rf /var/log/honeycomb" - stdout = ssh_ignore_error(ssh, cmd, sudo=True) - print "###TI {}".format(stdout) - fix_interrupted("vpp") - - else: - # Create installation directory on DUT - cmd = "rm -r {0}; mkdir {0}".format(install_dir) - stdout = ssh_no_error(ssh, cmd) - print "###TI {}".format(stdout) - - if honeycomb: - smd = "ls ~/honeycomb | grep .deb" - stdout = ssh_ignore_error(ssh, smd) - if "honeycomb" in stdout: - # If custom honeycomb packages exist, use them - cmd = "cp ~/honeycomb/*.deb {0}".format(install_dir) - stdout = ssh_no_error(ssh, cmd) - print "###TI {}".format(stdout) - else: - # Copy packages from local path to installation dir - for deb in packages: - print "###TI scp: {}".format(deb) - ssh.scp(local_path=deb, remote_path=install_dir) - else: - # Copy packages from local path to installation dir - for deb in packages: - print "###TI scp: {}".format(deb) - ssh.scp(local_path=deb, remote_path=install_dir) - - if honeycomb: - fix_interrupted("honeycomb") - fix_interrupted("vpp") - - # Installation of deb packages - cmd = "dpkg -i --force-all {}/*.deb".format(install_dir) - stdout = ssh_no_error(ssh, cmd, sudo=True) - print "###TI {}".format(stdout) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/files/reset_vppdevice.sh b/resources/tools/testbed-setup/ansible/roles/cleanup/files/reset_vppdevice.sh new file mode 100644 index 0000000000..ede2db1273 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/files/reset_vppdevice.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +set -euo pipefail + +function die () { + # Print the message to standard error end exit with error code specified + # by the second argument. + # + # Hardcoded values: + # - The default error message. + # Arguments: + # - ${1} - The whole error message, be sure to quote. Optional + # - ${2} - the code to exit with, default: 1. + + set +eu + warn "${1:-Unspecified run-time error occurred!}" + exit "${2:-1}" +} + + +function set_eligibility_off { + # Set Nomad eligibility to ineligible for scheduling. Fail otherwise. + + set -euo pipefail + + node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die + node_status="$(nomad node status | grep $(hostname))" || die + + if [[ "${node_status}" != *"ineligible"* ]]; then + nomad node eligibility -disable "${node_id}" || die + node_status="$(nomad node status | grep $(hostname))" || die + if [[ "${node_status}" != *"ineligible"* ]]; then + die "Set eligibility off failed!" + fi + fi +} + + +function set_eligibility_on { + # Set Nomad eligibility to eligible for scheduling. Fail otherwise. + + set -euo pipefail + + node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die + node_status="$(nomad node status | grep $(hostname))" || die + + if [[ "${node_status}" == *"ineligible"* ]]; then + nomad node eligibility -enable "${node_id}" || die + node_status="$(nomad node status | grep $(hostname))" || die + if [[ "${node_status}" == *"ineligible"* ]]; then + die "Set eligibility on failed!" + fi + fi +} + + +function restart_vfs_service { + # Stop and start VF serice. This will reinitialize VFs and driver mappings. + + set -euo pipefail + + warn "Restarting VFs service (this may take few minutes)..." + sudo service csit-initialize-vfs stop || die "Failed to stop VFs service!" + sudo service csit-initialize-vfs start || die "Failed to start VFs service!" +} + + +function wait_for_pending_containers { + # Wait in loop for defined amount of time for pending containers to + # gracefully quit them. If parameter force is specified. Force kill them. + + # Arguments: + # - ${@} - Script parameters. + + set -euo pipefail + + retries=60 + wait_time=60 + containers=(docker ps --quiet --filter name=csit*) + + for i in $(seq 1 ${retries}); do + mapfile -t pending_containers < <( ${containers[@]} ) || die + warn "Waiting for pending containers [${pending_containers[@]}] ..." + if [ ${#pending_containers[@]} -eq 0 ]; then + break + fi + sleep "${wait_time}" || die + done + if [ ${#pending_containers[@]} -ne 0 ]; then + if [[ "${1-}" == "force" ]]; then + warn "Force killing [${pending_containers[@]}] ..." + docker rm --force ${pending_containers[@]} || die + else + die "Still few containers running!" + fi + fi +} + + +function warn () { + # Print the message to standard error. + # + # Arguments: + # - ${@} - The text of the message. + + echo "$@" >&2 +} + + +set_eligibility_off || die +wait_for_pending_containers "${@}" || die +restart_vfs_service || die +set_eligibility_on || die diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml new file mode 100644 index 0000000000..a61aa6ceee --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml @@ -0,0 +1,28 @@ +--- +# file: roles/cleanup/tasks/kill_containers.yaml + +- name: Kill container - Get running Docker containers + shell: "docker ps -aq" + register: running_containers + changed_when: no + tags: kill-containers + +- name: Kill container - Remove all Docker containers + docker_container: + name: "{{ item }}" + state: absent + with_items: "{{ running_containers.stdout_lines }}" + tags: kill-containers + +- name: Kill container - Get running LXC containers + shell: "lxc-ls" + register: running_containers + changed_when: no + tags: kill-containers + +- name: Kill container - Remove all LXC containers + lxc_container: + name: '{{ item }}' + state: absent + with_items: "{{ running_containers.stdout_lines }}" + tags: kill-containers diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml new file mode 100644 index 0000000000..4a1180b77f --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml @@ -0,0 +1,27 @@ +--- +# file: roles/cleanup/tasks/kill_process.yaml + +- name: Kill process - Get pid of {{ process }} + shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'" + when: > + process is defined and process != "" + register: running_processes + tags: kill-process + +- name: Kill process - Safe kill {{ process }} + shell: "kill {{ item }}" + with_items: "{{ running_processes.stdout_lines }}" + tags: kill-process + +- wait_for: + path: "/proc/{{ item }}/status" + state: absent + with_items: "{{ running_processes.stdout_lines }}" + ignore_errors: yes + register: killed_processes + tags: kill-process + +- name: Kill process - Force kill {{ process }} + shell: "kill -9 {{ item }}" + with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}" + tags: kill-process diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/main.yaml new file mode 100644 index 0000000000..64a55c4672 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/main.yaml @@ -0,0 +1,31 @@ +--- +# file: roles/cleanup/tasks/main.yaml +# purpose: Structured per server cleanup tasks. +# - main: +# - tg: +# - Run tasks on TG servers only. +# - Cleanup processes (T-Rex). +# - sut: +# - Run tasks on SUT servers only. +# - Cleanup file leftovers (logs). +# - Cleanup packages (VPP, Honeycomb). +# - Cleanup processes (qemu, l3fwd, testpmd, docker, kubernetes) +# - Cleanup interfaces. +# - vpp_device +# - Run tasks on vpp_device servers only. +# - Reset SRIOV + +- name: tg specific + include_tasks: tg.yaml + when: "'tg' in group_names" + tags: cleanup + +- name: sut specific + include_tasks: sut.yaml + when: "'sut' in group_names" + tags: cleanup + +- name: vpp_device specific + include_tasks: vpp_device.yaml + when: "'vpp_device' in group_names" + tags: cleanup diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/remove_package.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/remove_package.yaml new file mode 100644 index 0000000000..8f5ec8fefe --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/remove_package.yaml @@ -0,0 +1,31 @@ +--- +# file: roles/cleanup/tasks/remove_package.yaml + +- name: Remove package - Fix corrupted apt + shell: 'dpkg --configure -a' + when: > + ansible_distribution == 'Ubuntu' + tags: remove-package + +- name: Remove package - Check if {{ package }} is installed + shell: > + "dpkg-query -W -f='${Status}' {{ package }} | grep 'install ok installed'" + register: package_is_installed + failed_when: no + changed_when: no + when: > + ansible_distribution == 'Ubuntu' + tags: remove-package + +- name: Remove package - {{ package }} + apt: + name: '{{ package }}' + force: yes + purge: yes + state: absent + when: > + package is defined and + package != '' and + package_is_installed.rc == 0 and + ansible_distribution|lower == 'ubuntu' + tags: remove-package diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml new file mode 100644 index 0000000000..5083a96a29 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml @@ -0,0 +1,52 @@ +--- +# file: roles/cleanup/tasks/sut.yaml + +- name: Kill processes - qemu + import_tasks: kill_process.yaml + vars: + process: "qemu" + tags: kill-process + +- name: Kill processes - l3fwd + import_tasks: kill_process.yaml + vars: + process: "l3fwd" + tags: kill-process + +- name: Kill processes - testpmd + import_tasks: kill_process.yaml + vars: + process: "testpmd" + tags: kill-process + +- name: Remove file or dir - HoneyComb logs + file: + state: absent + path: "/var/log/honeycomb" + tags: remove-file-dir + +- name: Remove file or dir - Core zip file + file: + state: absent + path: "/tmp/*tar.lzo.lrz.xz*" + tags: remove-file-dir + +- name: Remove file or dir - Core dump file + file: + state: absent + path: "/tmp/*core*" + tags: remove-file-dir + +- name: Kill containers - Remove all containers + import_tasks: kill_containers.yaml + tags: kill-containers + +- name: Kubernetes - Reset + raw: 'kubeadm reset --force' + tags: kill-kubernetes + +- name: Remove packages - Remove VPP + import_tasks: remove_package.yaml + vars: + package: "*vpp*" + tags: remove-package diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml new file mode 100644 index 0000000000..f58cb59a1a --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml @@ -0,0 +1,14 @@ +--- +# file: roles/cleanup/tasks/tg.yaml + +- name: Kill processes - TRex + import_tasks: kill_process.yaml + vars: + process: "_t-rex" + tags: kill-process + +- name: Kill processes - WRK + import_tasks: kill_process.yaml + vars: + process: "wrk" + tags: kill-process diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/vpp_device.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/vpp_device.yaml new file mode 100644 index 0000000000..5b7713a554 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/vpp_device.yaml @@ -0,0 +1,15 @@ +--- +# file: roles/cleanup/tasks/vpp_device.yaml + +- name: Reset vpp_device binary + template: + src: 'files/reset_vppdevice.sh' + dest: '/usr/local/bin' + owner: 'root' + group: 'root' + mode: '644' + tags: reset-sriov + +- name: Reset vpp_device + raw: 'reset_vppdevice.sh --force' + tags: reset-sriov diff --git a/resources/tools/testbed-setup/ansible/sut.yaml b/resources/tools/testbed-setup/ansible/sut.yaml index d93d0b07cf..fa7adb73e2 100644 --- a/resources/tools/testbed-setup/ansible/sut.yaml +++ b/resources/tools/testbed-setup/ansible/sut.yaml @@ -16,5 +16,7 @@ tags: kubernetes - role: performance_tuning tags: performance_tuning + - role: cleanup + tags: cleanup - role: calibration tags: calibration diff --git a/resources/tools/testbed-setup/ansible/tg.yaml b/resources/tools/testbed-setup/ansible/tg.yaml index 4b5106652b..62ba9d53dd 100644 --- a/resources/tools/testbed-setup/ansible/tg.yaml +++ b/resources/tools/testbed-setup/ansible/tg.yaml @@ -14,5 +14,7 @@ tags: docker - role: performance_tuning tags: performance_tuning + - role: cleanup + tags: cleanup - role: calibration tags: calibration diff --git a/resources/tools/testbed-setup/ansible/vpp_device.yaml b/resources/tools/testbed-setup/ansible/vpp_device.yaml index 8353be03c2..5860a7dde7 100644 --- a/resources/tools/testbed-setup/ansible/vpp_device.yaml +++ b/resources/tools/testbed-setup/ansible/vpp_device.yaml @@ -12,3 +12,5 @@ tags: docker - role: vpp_device tags: vpp_device + - role: cleanup + tags: cleanup -- 2.16.6