set -exuo pipefail
if ! installed sshpass; then
- sudo apt-get update -y || die "apt-get update failed!"
- sudo apt-get install -y sshpass || die "Install sshpass failed!"
+ die "Please install sshpass!"
fi
if ! installed ansible-playbook; then
}
# Cleanup check.
set +e
- cleanup_topo
+ ansible_hosts "cleanup"
result="$?"
set -e
if [[ "${result}" == "0" ]]; then
set -eu
warn "Testbed looks unreserved already. Trap removal failed before?"
else
- cleanup_topo || true
+ ansible_hosts "cleanup" || true
python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || {
die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2
}
+++ /dev/null
-#!/usr/bin/python
-
-# Copyright (c) 2017 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""This script renames the given robot keywords in the given directory
-recursively.
-
-Example:
-
- ./rename_robot_keywords.py -i kws.csv -s ";" -d ~/ws/vpp/git/csit/ -vvv
-
- Input file "kws.csv" is CSV file exported from e.g. MS Excel. Its structure
- must be:
-
- <Old keyword name><separator><New keyword name>
-
- One keyword per line.
-
-"""
-
-import argparse
-import sys
-import re
-from os import walk, rename
-from os.path import join
-
-
-def time_interval(func):
- """Decorator function to measure the time spent by the decorated function.
-
- :param func: Decorated function.
- :type func: Callable object.
- :returns: Wrapper function.
- :rtype: Callable object.
- """
-
- import time
-
- def wrapper(*args, **kwargs):
- start = time.clock()
- result = func(*args, **kwargs)
- stop = time.clock()
- print("\nRenaming done in {:.5g} seconds\n".
- format(stop - start))
- return result
- return wrapper
-
-
-def get_files(path, extension):
- """Generates the list of files to process.
-
- :param path: Path to files.
- :param extension: Extension of files to process. If it is the empty string,
- all files will be processed.
- :type path: str
- :type extension: str
- :returns: List of files to process.
- :rtype: list
- """
-
- file_list = list()
- for root, dirs, files in walk(path):
- for filename in files:
- if extension:
- if filename.endswith(extension):
- file_list.append(join(root, filename))
- else:
- file_list.append(join(root, filename))
-
- return file_list
-
-
-def read_keywords(args):
- """This function reads the keywords from the input file and creates:
-
- - a dictionary where the key is the old name and the value is the new name,
- these keywords will be further processed.
- - a list of keywords which will not be processed, typically keywords with
- argument(s) in its names.
- - a list of duplicates - duplicated keyword names or names which are parts
- of another keyword name, they will not be processed.
-
- :param args: Parsed arguments.
- :type args: ArgumentParser
- :returns: keyword names - dictionary where the key is the old name and the
- value is the new name; ignored keyword names - list of keywords which will
- not be processed; duplicates - duplicated keyword names or names which are
- parts of another keyword name, they will not be processed.
- :rtype: tuple(dict, list, list)
- """
-
- kw_names = dict()
- ignored_kw_names = list()
- duplicates = list()
-
- for line in args.input:
- old_name, new_name = line.split(args.separator)
- if '$' in old_name:
- ignored_kw_names.append((old_name, new_name[:-1]))
- elif old_name in kw_names.keys():
- duplicates.append((old_name, new_name[:-1]))
- else:
- kw_names[old_name] = new_name[:-1]
-
- # Remove duplicates:
- for old_name, _ in duplicates:
- new_name = kw_names.pop(old_name, None)
- if new_name:
- duplicates.append((old_name, new_name))
-
- # Find KW names which are parts of other KW names:
- for old_name in kw_names.keys():
- count = 0
- for key in kw_names.keys():
- if old_name in key:
- count += 1
- if old_name in kw_names[key]:
- if old_name != key:
- count += 1
- if count > 1:
- duplicates.append((old_name, kw_names[old_name]))
- kw_names.pop(old_name)
-
- return kw_names, ignored_kw_names, duplicates
-
-
-def rename_keywords(file_list, kw_names, args):
- """Rename the keywords in specified files.
-
- :param file_list: List of files to be processed.
- :param kw_names: Dictionary where the key is the old name and the value is
- the new name
- :type file_list: list
- :type kw_names: dict
- """
-
- kw_not_found = list()
-
- for old_name, new_name in kw_names.items():
- kw_found = False
- if args.verbosity > 0:
- print("\nFrom: {}\n To: {}\n".format(old_name, new_name))
- for file_name in file_list:
- tmp_file_name = file_name + ".new"
- with open(file_name) as file_read:
- file_write = open(tmp_file_name, 'w')
- occurrences = 0
- for line in file_read:
- new_line = re.sub(old_name, new_name, line)
- file_write.write(new_line)
- if new_line != line:
- occurrences += 1
- if occurrences:
- kw_found = True
- if args.verbosity > 1:
- print(" {:3d}: {}".format(occurrences, file_name))
- file_write.close()
- rename(tmp_file_name, file_name)
- if not kw_found:
- kw_not_found.append(old_name)
-
- if args.verbosity > 0:
- print("\nKeywords not found:")
- for item in kw_not_found:
- print(" {}".format(item))
-
-
-def parse_args():
- """Parse arguments from command line.
-
- :returns: Parsed arguments.
- :rtype: ArgumentParser
- """
-
- parser = argparse.ArgumentParser(description=__doc__,
- formatter_class=argparse.
- RawDescriptionHelpFormatter)
- parser.add_argument("-i", "--input",
- required=True,
- type=argparse.FileType('r'),
- help="Text file with the old keyword name and the new "
- "keyword name separated by separator per line.")
- parser.add_argument("-s", "--separator",
- default=";",
- type=str,
- help="Separator which separates the old and the new "
- "keyword name.")
- parser.add_argument("-d", "--dir",
- required=True,
- type=str,
- help="Directory with robot files where the keywords "
- "should be recursively searched.")
- parser.add_argument("-v", "--verbosity", action="count",
- help="Set the output verbosity.")
- return parser.parse_args()
-
-
-@time_interval
-def main():
- """Main function."""
-
- args = parse_args()
-
- kw_names, ignored_kw_names, duplicates = read_keywords(args)
-
- file_list = get_files(args.dir, "robot")
-
- if args.verbosity > 2:
- print("\nList of files to be processed:")
- for item in file_list:
- print(" {}".format(item))
- print("\n{} files to be processed.\n".format(len(file_list)))
-
- print("\nList of keywords to be renamed:")
- for item in kw_names:
- print(" {}".format(item))
- print("\n{} keywords to be renamed.\n".format(len(kw_names)))
-
- rename_keywords(file_list, kw_names, args)
-
- if args.verbosity >= 0:
- print("\nIgnored keywords: ({})".format(len(ignored_kw_names)))
- for old, new in ignored_kw_names:
- print(" From: {}\n To: {}\n".format(old, new))
-
- print("\nIgnored duplicates ({}):".format(len(duplicates)))
- for old, new in duplicates:
- print(" From: {}\n To: {}\n".format(old, new))
-
-
-if __name__ == "__main__":
- sys.exit(main())
+++ /dev/null
-#!/usr/bin/python
-
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Script parses the data taken by robot framework (output.xml) and dumps
-interested values into XML output file."""
-
-import argparse
-import re
-import sys
-import xml.etree.ElementTree as ET
-
-from robot.api import ExecutionResult, ResultVisitor
-
-
-class ExecutionChecker(ResultVisitor):
- """Iterates through test cases."""
-
- tc_regexp = re.compile(ur'^tc\d+-((\d+)B|IMIX)-(\d)t(\d)c-(.*)')
- rate_regexp = re.compile(ur'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)[\D\d]*')
- lat_regexp = re.compile(ur'^[\D\d]*'\
- ur'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','\
- ur'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]\s\n'\
- ur'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','\
- ur'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]\s\n'\
- ur'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+\/-?\d+)\','\
- ur'\s\'(-?\d+\/-?\d+\/-?\d+)\'\]')
-
- def __init__(self, args):
- self.root = ET.Element('build',
- attrib={'vdevice': args.vdevice})
-
- def visit_suite(self, suite):
- """Implements traversing through the suite and its direct children.
-
- :param suite: Suite to process.
- :type suite: Suite
- :return: Nothing.
- """
- if self.start_suite(suite) is not False:
- suite.suites.visit(self)
- suite.tests.visit(self)
- self.end_suite(suite)
-
- def start_suite(self, suite):
- """Called when suite starts.
-
- :param suite: Suite to process.
- :type suite: Suite
- :return: Nothing.
- """
- pass
-
- def end_suite(self, suite):
- """Called when suite ends.
-
- :param suite: Suite to process.
- :type suite: Suite
- :return: Nothing.
- """
- pass
-
- def visit_test(self, test):
- """Implements traversing through the test.
-
- :param test: Test to process.
- :type test: Test
- :return: Nothing.
- """
- if self.start_test(test) is not False:
- self.end_test(test)
-
- def start_test(self, test):
- """Called when test starts.
-
- :param test: Test to process.
- :type test: Test
- :return: Nothing.
- """
- if any("NDRPDRDISC" in tag for tag in test.tags):
- if test.status == 'PASS':
- tags = []
- for tag in test.tags:
- tags.append(tag)
-
- test_elem = ET.SubElement(
- self.root, "S" + test.parent.name.replace(" ", ""))
- test_elem.attrib['name'] = test.parent.name
- test_elem.attrib['framesize'] = str(re.search(
- self.tc_regexp, test.name).group(1))
- test_elem.attrib['threads'] = str(re.search(
- self.tc_regexp, test.name).group(3))
- test_elem.attrib['cores'] = str(re.search(
- self.tc_regexp, test.name).group(4))
- if any("NDRDISC" in tag for tag in test.tags):
- try:
- test_elem.attrib['lat_100'] = str(re.search(
- self.lat_regexp, test.message).group(1)) + '/' +\
- str(re.search(self.lat_regexp, test.message).
- group(2))
- except AttributeError:
- test_elem.attrib['lat_100'] = "-1/-1/-1/-1/-1/-1"
- try:
- test_elem.attrib['lat_50'] = str(re.search(
- self.lat_regexp, test.message).group(3)) + '/' +\
- str(re.search(self.lat_regexp, test.message).
- group(4))
- except AttributeError:
- test_elem.attrib['lat_50'] = "-1/-1/-1/-1/-1/-1"
- try:
- test_elem.attrib['lat_10'] = str(re.search(
- self.lat_regexp, test.message).group(5)) + '/' +\
- str(re.search(self.lat_regexp, test.message).
- group(6))
- except AttributeError:
- test_elem.attrib['lat_10'] = "-1/-1/-1/-1/-1/-1"
- test_elem.attrib['tags'] = ', '.join(tags)
- try:
- test_elem.text = str(re.search(
- self.rate_regexp, test.message).group(1))
- except AttributeError:
- test_elem.text = "-1"
-
- def end_test(self, test):
- """Called when test ends.
-
- :param test: Test to process.
- :type test: Test
- :return: Nothing.
- """
- pass
-
-
-def parse_tests(args):
- """Process data from robot output.xml file and return XML data.
-
- :param args: Parsed arguments.
- :type args: ArgumentParser
-
- :return: XML formatted output.
- :rtype: ElementTree
- """
-
- result = ExecutionResult(args.input)
- checker = ExecutionChecker(args)
- result.visit(checker)
-
- return checker.root
-
-
-def print_error(msg):
- """Print error message on stderr.
-
- :param msg: Error message to print.
- :type msg: str
- :return: nothing
- """
-
- sys.stderr.write(msg + '\n')
-
-
-def parse_args():
- """Parse arguments from cmd line.
-
- :return: Parsed arguments.
- :rtype ArgumentParser
- """
-
- parser = argparse.ArgumentParser()
- parser.add_argument("-i", "--input",
- required=True,
- type=argparse.FileType('r'),
- help="Robot XML log file")
- parser.add_argument("-o", "--output",
- required=True,
- type=argparse.FileType('w'),
- help="XML output file")
- parser.add_argument("-v", "--vdevice",
- required=False,
- default="",
- type=str,
- help="VPP version")
-
- return parser.parse_args()
-
-
-def main():
- """Main function."""
-
- args = parse_args()
-
- root = parse_tests(args)
- ET.ElementTree.write(ET.ElementTree(root), args.output)
-
-
-if __name__ == "__main__":
- sys.exit(main())
+++ /dev/null
-#!/usr/bin/env python
-
-# Copyright (c) 2017 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""This script provides copy and load of Docker container images.
- As destinations are used all DUT nodes from the topology file."""
-
-import sys
-import argparse
-from yaml import load
-
-from resources.libraries.python.ssh import SSH
-
-
-def ssh_no_error(ssh, cmd, sudo=False):
- """Execute a command over ssh channel, and log and exit if the command
- fails.
-
- :param ssh: SSH() object connected to a node.
- :param cmd: Command line to execute on remote node.
- :param sudo: Run command with sudo privileges.
- :type ssh: SSH() object
- :type cmd: str
- :type sudo: bool
- :returns: stdout from the SSH command.
- :rtype: str
- :raises RuntimeError: In case of unexpected ssh command failure
- """
- if sudo:
- ret, stdo, stde = ssh.exec_command_sudo(cmd, timeout=60)
- else:
- ret, stdo, stde = ssh.exec_command(cmd, timeout=60)
-
- if ret != 0:
- print('Command execution failed: "{}"'.format(cmd))
- print('stdout: {0}'.format(stdo))
- print('stderr: {0}'.format(stde))
- raise RuntimeError('Unexpected ssh command failure')
-
- return stdo
-
-
-def ssh_ignore_error(ssh, cmd, sudo=False):
- """Execute a command over ssh channel, ignore errors.
-
- :param ssh: SSH() object connected to a node.
- :param cmd: Command line to execute on remote node.
- :param sudo: Run command with sudo privileges.
- :type ssh: SSH() object
- :type cmd: str
- :type sudo: bool
- :returns: stdout from the SSH command.
- :rtype: str
- """
- if sudo:
- ret, stdo, stde = ssh.exec_command_sudo(cmd)
- else:
- ret, stdo, stde = ssh.exec_command(cmd)
-
- if ret != 0:
- print('Command execution failed: "{}"'.format(cmd))
- print('stdout: {0}'.format(stdo))
- print('stderr: {0}'.format(stde))
-
- return stdo
-
-
-def main():
- """Copy and load of Docker image."""
- parser = argparse.ArgumentParser()
- parser.add_argument("-t", "--topo", required=True,
- help="Topology file")
- parser.add_argument("-d", "--directory", required=True,
- help="Destination directory")
- parser.add_argument("-i", "--images", required=False, nargs='+',
- help="Images paths to copy")
- parser.add_argument("-c", "--cancel", help="Cancel all",
- action="store_true")
-
- args = parser.parse_args()
- topology_file = args.topo
- images = args.images
- directory = args.directory
- cancel_all = args.cancel
-
- work_file = open(topology_file)
- topology = load(work_file.read())['nodes']
-
- ssh = SSH()
- for node in topology:
- if topology[node]['type'] == "DUT":
- print("###TI host: {host}".format(host=topology[node]['host']))
- ssh.connect(topology[node])
-
- if cancel_all:
- # Remove destination directory on DUT
- cmd = "rm -r {directory}".format(directory=directory)
- stdout = ssh_ignore_error(ssh, cmd)
- print("###TI {stdout}".format(stdout=stdout))
-
- else:
- # Create installation directory on DUT
- cmd = "rm -r {directory}; mkdir {directory}"\
- .format(directory=directory)
- stdout = ssh_no_error(ssh, cmd)
- print("###TI {stdout}".format(stdout=stdout))
-
- # Copy images from local path to destination dir
- for image in images:
- print("###TI scp: {}".format(image))
- ssh.scp(local_path=image, remote_path=directory)
-
- # Load image to Docker.
- cmd = "for f in {directory}/*.tar.gz; do "\
- "sudo docker load -i $f; done".format(directory=directory)
- stdout = ssh_no_error(ssh, cmd)
- print("###TI {}".format(stdout))
-
- # Remove <none> images from Docker.
- cmd = "docker rmi $(sudo docker images -f 'dangling=true' -q)"
- stdout = ssh_ignore_error(ssh, cmd, sudo=True)
- print("###TI {}".format(stdout))
-
-
-if __name__ == "__main__":
- sys.exit(main())
+++ /dev/null
-#!/usr/bin/env python
-
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""This script provides copy and installation of VPP build deb packages.
- As destinations are used all DUT nodes from the topology file."""
-
-import sys
-import argparse
-from yaml import load
-
-from resources.libraries.python.ssh import SSH
-
-
-def ssh_no_error(ssh, cmd, sudo=False):
- """Execute a command over ssh channel, and log and exit if the command
- fails.
-
- :param ssh: SSH() object connected to a node.
- :param cmd: Command line to execute on remote node.
- :type ssh: SSH() object
- :type cmd: str
- :return: stdout from the SSH command.
- :rtype: str
- """
-
- if sudo:
- ret, stdo, stde = ssh.exec_command_sudo(cmd, timeout=60)
- else:
- ret, stdo, stde = ssh.exec_command(cmd, timeout=60)
-
- if ret != 0:
- print 'Command execution failed: "{}"'.format(cmd)
- print 'stdout: {0}'.format(stdo)
- print 'stderr: {0}'.format(stde)
- raise RuntimeError('Unexpected ssh command failure')
-
- return stdo
-
-
-def ssh_ignore_error(ssh, cmd, sudo=False):
- """Execute a command over ssh channel, ignore errors.
-
- :param ssh: SSH() object connected to a node.
- :param cmd: Command line to execute on remote node.
- :type ssh: SSH() object
- :type cmd: str
- :return: stdout from the SSH command.
- :rtype: str
- """
-
- if sudo:
- ret, stdo, stde = ssh.exec_command_sudo(cmd)
- else:
- ret, stdo, stde = ssh.exec_command(cmd)
-
- if ret != 0:
- print 'Command execution failed: "{}"'.format(cmd)
- print 'stdout: {0}'.format(stdo)
- print 'stderr: {0}'.format(stde)
-
- return stdo
-
-
-def main():
- """Copy and installation of VPP packages."""
-
- parser = argparse.ArgumentParser()
- parser.add_argument("-t", "--topo", required=True,
- help="Topology file")
- parser.add_argument("-d", "--directory", required=True,
- help="Installation directory")
- parser.add_argument("-p", "--packages", required=False, nargs='+',
- help="Packages paths to copy")
- parser.add_argument("-c", "--cancel", help="Cancel installation",
- action="store_true")
- parser.add_argument("-hc", "--honeycomb", help="Include Honeycomb package.",
- required=False, default=False)
-
- args = parser.parse_args()
- topology_file = args.topo
- packages = args.packages
- install_dir = args.directory
- cancel_installation = args.cancel
- honeycomb = args.honeycomb
-
- work_file = open(topology_file)
- topology = load(work_file.read())['nodes']
-
- def fix_interrupted(package):
- """If there are interrupted installations, clean them up."""
-
- cmd = "dpkg -l | grep {0}".format(package)
- ret, _, _ = ssh.exec_command(cmd)
- if ret == 0:
- # Try to fix interrupted installations
- cmd = 'dpkg --configure -a'
- stdout = ssh_no_error(ssh, cmd, sudo=True)
- print "###TI {}".format(stdout)
- # Try to remove installed packages
- cmd = 'apt-get purge -y "{0}.*"'.format(package)
- stdout = ssh_no_error(ssh, cmd, sudo=True)
- print "###TI {}".format(stdout)
-
- ssh = SSH()
- for node in topology:
- if topology[node]['type'] == "DUT":
- print "###TI host: {}".format(topology[node]['host'])
- ssh.connect(topology[node])
-
- if cancel_installation:
- # Remove installation directory on DUT
- cmd = "rm -r {}".format(install_dir)
- stdout = ssh_ignore_error(ssh, cmd)
- print "###TI {}".format(stdout)
-
- if honeycomb:
- fix_interrupted("honeycomb")
- # remove HC logs
- cmd = "rm -rf /var/log/honeycomb"
- stdout = ssh_ignore_error(ssh, cmd, sudo=True)
- print "###TI {}".format(stdout)
- fix_interrupted("vpp")
-
- else:
- # Create installation directory on DUT
- cmd = "rm -r {0}; mkdir {0}".format(install_dir)
- stdout = ssh_no_error(ssh, cmd)
- print "###TI {}".format(stdout)
-
- if honeycomb:
- smd = "ls ~/honeycomb | grep .deb"
- stdout = ssh_ignore_error(ssh, smd)
- if "honeycomb" in stdout:
- # If custom honeycomb packages exist, use them
- cmd = "cp ~/honeycomb/*.deb {0}".format(install_dir)
- stdout = ssh_no_error(ssh, cmd)
- print "###TI {}".format(stdout)
- else:
- # Copy packages from local path to installation dir
- for deb in packages:
- print "###TI scp: {}".format(deb)
- ssh.scp(local_path=deb, remote_path=install_dir)
- else:
- # Copy packages from local path to installation dir
- for deb in packages:
- print "###TI scp: {}".format(deb)
- ssh.scp(local_path=deb, remote_path=install_dir)
-
- if honeycomb:
- fix_interrupted("honeycomb")
- fix_interrupted("vpp")
-
- # Installation of deb packages
- cmd = "dpkg -i --force-all {}/*.deb".format(install_dir)
- stdout = ssh_no_error(ssh, cmd, sudo=True)
- print "###TI {}".format(stdout)
-
-if __name__ == "__main__":
- sys.exit(main())
--- /dev/null
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+function die () {
+ # Print the message to standard error end exit with error code specified
+ # by the second argument.
+ #
+ # Hardcoded values:
+ # - The default error message.
+ # Arguments:
+ # - ${1} - The whole error message, be sure to quote. Optional
+ # - ${2} - the code to exit with, default: 1.
+
+ set +eu
+ warn "${1:-Unspecified run-time error occurred!}"
+ exit "${2:-1}"
+}
+
+
+function set_eligibility_off {
+ # Set Nomad eligibility to ineligible for scheduling. Fail otherwise.
+
+ set -euo pipefail
+
+ node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die
+ node_status="$(nomad node status | grep $(hostname))" || die
+
+ if [[ "${node_status}" != *"ineligible"* ]]; then
+ nomad node eligibility -disable "${node_id}" || die
+ node_status="$(nomad node status | grep $(hostname))" || die
+ if [[ "${node_status}" != *"ineligible"* ]]; then
+ die "Set eligibility off failed!"
+ fi
+ fi
+}
+
+
+function set_eligibility_on {
+ # Set Nomad eligibility to eligible for scheduling. Fail otherwise.
+
+ set -euo pipefail
+
+ node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die
+ node_status="$(nomad node status | grep $(hostname))" || die
+
+ if [[ "${node_status}" == *"ineligible"* ]]; then
+ nomad node eligibility -enable "${node_id}" || die
+ node_status="$(nomad node status | grep $(hostname))" || die
+ if [[ "${node_status}" == *"ineligible"* ]]; then
+ die "Set eligibility on failed!"
+ fi
+ fi
+}
+
+
+function restart_vfs_service {
+ # Stop and start VF serice. This will reinitialize VFs and driver mappings.
+
+ set -euo pipefail
+
+ warn "Restarting VFs service (this may take few minutes)..."
+ sudo service csit-initialize-vfs stop || die "Failed to stop VFs service!"
+ sudo service csit-initialize-vfs start || die "Failed to start VFs service!"
+}
+
+
+function wait_for_pending_containers {
+ # Wait in loop for defined amount of time for pending containers to
+ # gracefully quit them. If parameter force is specified. Force kill them.
+
+ # Arguments:
+ # - ${@} - Script parameters.
+
+ set -euo pipefail
+
+ retries=60
+ wait_time=60
+ containers=(docker ps --quiet --filter name=csit*)
+
+ for i in $(seq 1 ${retries}); do
+ mapfile -t pending_containers < <( ${containers[@]} ) || die
+ warn "Waiting for pending containers [${pending_containers[@]}] ..."
+ if [ ${#pending_containers[@]} -eq 0 ]; then
+ break
+ fi
+ sleep "${wait_time}" || die
+ done
+ if [ ${#pending_containers[@]} -ne 0 ]; then
+ if [[ "${1-}" == "force" ]]; then
+ warn "Force killing [${pending_containers[@]}] ..."
+ docker rm --force ${pending_containers[@]} || die
+ else
+ die "Still few containers running!"
+ fi
+ fi
+}
+
+
+function warn () {
+ # Print the message to standard error.
+ #
+ # Arguments:
+ # - ${@} - The text of the message.
+
+ echo "$@" >&2
+}
+
+
+set_eligibility_off || die
+wait_for_pending_containers "${@}" || die
+restart_vfs_service || die
+set_eligibility_on || die
--- /dev/null
+---
+# file: roles/cleanup/tasks/kill_containers.yaml
+
+- name: Kill container - Get running Docker containers
+ shell: "docker ps -aq"
+ register: running_containers
+ changed_when: no
+ tags: kill-containers
+
+- name: Kill container - Remove all Docker containers
+ docker_container:
+ name: "{{ item }}"
+ state: absent
+ with_items: "{{ running_containers.stdout_lines }}"
+ tags: kill-containers
+
+- name: Kill container - Get running LXC containers
+ shell: "lxc-ls"
+ register: running_containers
+ changed_when: no
+ tags: kill-containers
+
+- name: Kill container - Remove all LXC containers
+ lxc_container:
+ name: '{{ item }}'
+ state: absent
+ with_items: "{{ running_containers.stdout_lines }}"
+ tags: kill-containers
--- /dev/null
+---
+# file: roles/cleanup/tasks/kill_process.yaml
+
+- name: Kill process - Get pid of {{ process }}
+ shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'"
+ when: >
+ process is defined and process != ""
+ register: running_processes
+ tags: kill-process
+
+- name: Kill process - Safe kill {{ process }}
+ shell: "kill {{ item }}"
+ with_items: "{{ running_processes.stdout_lines }}"
+ tags: kill-process
+
+- wait_for:
+ path: "/proc/{{ item }}/status"
+ state: absent
+ with_items: "{{ running_processes.stdout_lines }}"
+ ignore_errors: yes
+ register: killed_processes
+ tags: kill-process
+
+- name: Kill process - Force kill {{ process }}
+ shell: "kill -9 {{ item }}"
+ with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}"
+ tags: kill-process
--- /dev/null
+---
+# file: roles/cleanup/tasks/main.yaml
+# purpose: Structured per server cleanup tasks.
+# - main:
+# - tg:
+# - Run tasks on TG servers only.
+# - Cleanup processes (T-Rex).
+# - sut:
+# - Run tasks on SUT servers only.
+# - Cleanup file leftovers (logs).
+# - Cleanup packages (VPP, Honeycomb).
+# - Cleanup processes (qemu, l3fwd, testpmd, docker, kubernetes)
+# - Cleanup interfaces.
+# - vpp_device
+# - Run tasks on vpp_device servers only.
+# - Reset SRIOV
+
+- name: tg specific
+ include_tasks: tg.yaml
+ when: "'tg' in group_names"
+ tags: cleanup
+
+- name: sut specific
+ include_tasks: sut.yaml
+ when: "'sut' in group_names"
+ tags: cleanup
+
+- name: vpp_device specific
+ include_tasks: vpp_device.yaml
+ when: "'vpp_device' in group_names"
+ tags: cleanup
--- /dev/null
+---
+# file: roles/cleanup/tasks/remove_package.yaml
+
+- name: Remove package - Fix corrupted apt
+ shell: 'dpkg --configure -a'
+ when: >
+ ansible_distribution == 'Ubuntu'
+ tags: remove-package
+
+- name: Remove package - Check if {{ package }} is installed
+ shell: >
+ "dpkg-query -W -f='${Status}' {{ package }} | grep 'install ok installed'"
+ register: package_is_installed
+ failed_when: no
+ changed_when: no
+ when: >
+ ansible_distribution == 'Ubuntu'
+ tags: remove-package
+
+- name: Remove package - {{ package }}
+ apt:
+ name: '{{ package }}'
+ force: yes
+ purge: yes
+ state: absent
+ when: >
+ package is defined and
+ package != '' and
+ package_is_installed.rc == 0 and
+ ansible_distribution|lower == 'ubuntu'
+ tags: remove-package
--- /dev/null
+---
+# file: roles/cleanup/tasks/sut.yaml
+
+- name: Kill processes - qemu
+ import_tasks: kill_process.yaml
+ vars:
+ process: "qemu"
+ tags: kill-process
+
+- name: Kill processes - l3fwd
+ import_tasks: kill_process.yaml
+ vars:
+ process: "l3fwd"
+ tags: kill-process
+
+- name: Kill processes - testpmd
+ import_tasks: kill_process.yaml
+ vars:
+ process: "testpmd"
+ tags: kill-process
+
+- name: Remove file or dir - HoneyComb logs
+ file:
+ state: absent
+ path: "/var/log/honeycomb"
+ tags: remove-file-dir
+
+- name: Remove file or dir - Core zip file
+ file:
+ state: absent
+ path: "/tmp/*tar.lzo.lrz.xz*"
+ tags: remove-file-dir
+
+- name: Remove file or dir - Core dump file
+ file:
+ state: absent
+ path: "/tmp/*core*"
+ tags: remove-file-dir
+
+- name: Kill containers - Remove all containers
+ import_tasks: kill_containers.yaml
+ tags: kill-containers
+
+- name: Kubernetes - Reset
+ raw: 'kubeadm reset --force'
+ tags: kill-kubernetes
+
+- name: Remove packages - Remove VPP
+ import_tasks: remove_package.yaml
+ vars:
+ package: "*vpp*"
+ tags: remove-package
--- /dev/null
+---
+# file: roles/cleanup/tasks/tg.yaml
+
+- name: Kill processes - TRex
+ import_tasks: kill_process.yaml
+ vars:
+ process: "_t-rex"
+ tags: kill-process
+
+- name: Kill processes - WRK
+ import_tasks: kill_process.yaml
+ vars:
+ process: "wrk"
+ tags: kill-process
--- /dev/null
+---
+# file: roles/cleanup/tasks/vpp_device.yaml
+
+- name: Reset vpp_device binary
+ template:
+ src: 'files/reset_vppdevice.sh'
+ dest: '/usr/local/bin'
+ owner: 'root'
+ group: 'root'
+ mode: '644'
+ tags: reset-sriov
+
+- name: Reset vpp_device
+ raw: 'reset_vppdevice.sh --force'
+ tags: reset-sriov
tags: kubernetes
- role: performance_tuning
tags: performance_tuning
+ - role: cleanup
+ tags: cleanup
- role: calibration
tags: calibration
tags: docker
- role: performance_tuning
tags: performance_tuning
+ - role: cleanup
+ tags: cleanup
- role: calibration
tags: calibration
tags: docker
- role: vpp_device
tags: vpp_device
+ - role: cleanup
+ tags: cleanup
Code Review / csit.git / commitdiff