X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2Fhoneycomb%2FHoneycombSetup.py;h=32992c03defa3af0d0e1a6ebeca450e51890eb9f;hp=8a05ad7583f1f93effbf1c0fa82493c98e51216c;hb=dc0288317e241f6cb18b0978993a1e2f396c793c;hpb=839c9c1a64524fe236ff435e6fbd3fe29eb97603 diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py index 8a05ad7583..32992c03de 100644 --- a/resources/libraries/python/honeycomb/HoneycombSetup.py +++ b/resources/libraries/python/honeycomb/HoneycombSetup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Cisco and/or its affiliates. +# Copyright (c) 2017 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -13,6 +13,8 @@ """Implementation of keywords for Honeycomb setup.""" +from ipaddress import IPv6Address, AddressValueError + from robot.api import logger from resources.libraries.python.HTTPRequest import HTTPRequest, HTTPCodes, \ @@ -56,7 +58,7 @@ class HoneycombSetup(object): HoneycombSetup.print_environment(nodes) - logger.console("\nStarting Honeycomb service ...") + logger.console("\n(re)Starting Honeycomb service ...") cmd = "sudo service honeycomb start" @@ -103,6 +105,34 @@ class HoneycombSetup(object): raise HoneycombError('Node(s) {0} failed to stop Honeycomb.'. format(errors)) + @staticmethod + def restart_honeycomb_on_dut(node): + """Restart Honeycomb on specified DUT nodes. + + This keyword restarts the Honeycomb service on specified DUTs. Use the + keyword "Check Honeycomb Startup State" to check if the Honeycomb is up + and running. + + :param node: Node to restart Honeycomb on. + :type node: dict + :raises HoneycombError: If Honeycomb fails to start. + """ + + logger.console("\n(re)Starting Honeycomb service ...") + + cmd = "sudo service honeycomb restart" + + ssh = SSH() + ssh.connect(node) + (ret_code, _, _) = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + raise HoneycombError('Node {0} failed to restart Honeycomb.'. + format(node['host'])) + else: + logger.info( + "Honeycomb service restart is in progress on node {0}".format( + node['host'])) + @staticmethod def check_honeycomb_startup_state(*nodes): """Check state of Honeycomb service during startup on specified nodes. @@ -127,8 +157,18 @@ class HoneycombSetup(object): for node in nodes: if node['type'] == NodeType.DUT: HoneycombSetup.print_ports(node) - status_code, _ = HTTPRequest.get(node, path, timeout=10, - enable_logging=False) + try: + status_code, _ = HTTPRequest.get(node, path, + enable_logging=False) + except HTTPRequestError: + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = ssh.exec_command_sudo( + "tail -n 100 /var/log/syslog") + if ret_code != 0: + # It's probably Centos + ssh.exec_command_sudo("tail -n 100 /var/log/messages") + raise if status_code == HTTPCodes.OK: logger.info("Honeycomb on node {0} is up and running". format(node['host'])) @@ -169,7 +209,6 @@ class HoneycombSetup(object): if node['type'] == NodeType.DUT: try: status_code, _ = HTTPRequest.get(node, '/index.html', - timeout=5, enable_logging=False) if status_code == HTTPCodes.OK: raise HoneycombError('Honeycomb on node {0} is still ' @@ -198,31 +237,58 @@ class HoneycombSetup(object): return True @staticmethod - def configure_unsecured_access(*nodes): - """Configure Honeycomb to allow restconf requests through insecure HTTP - used by tests. By default this is only allowed for localhost. + def configure_restconf_binding_address(node): + """Configure Honeycomb to accept restconf requests from all IP + addresses. IP version is determined by node data. - :param nodes: All nodes in test topology. - :type nodes: dict + :param node: Information about a DUT node. + :type node: dict :raises HoneycombError: If the configuration could not be changed. """ - # TODO: Modify tests to use HTTPS instead. find = "restconf-binding-address" - replace = '\\"restconf-binding-address\\": \\"0.0.0.0\\",' + try: + IPv6Address(unicode(node["host"])) + # if management IP of the node is in IPv6 format + replace = '\\"restconf-binding-address\\": \\"0::0\\",' + except (AttributeError, AddressValueError): + replace = '\\"restconf-binding-address\\": \\"0.0.0.0\\",' argument = '"/{0}/c\\ {1}"'.format(find, replace) - path = "{0}/config/honeycomb.json".format(Const.REMOTE_HC_DIR) + path = "{0}/config/restconf.json".format(Const.REMOTE_HC_DIR) command = "sed -i {0} {1}".format(argument, path) ssh = SSH() - for node in nodes: - if node['type'] == NodeType.DUT: - ssh.connect(node) - (ret_code, _, stderr) = ssh.exec_command_sudo(command) - if ret_code != 0: - raise HoneycombError("Failed to modify configuration on " - "node {0}, {1}".format(node, stderr)) + ssh.connect(node) + (ret_code, _, stderr) = ssh.exec_command_sudo(command) + if ret_code != 0: + raise HoneycombError("Failed to modify configuration on " + "node {0}, {1}".format(node, stderr)) + + @staticmethod + def configure_jvpp_timeout(node, timeout=10): + """Configure timeout value for Java API commands Honeycomb sends to VPP. + + :param node: Information about a DUT node. + :param timeout: Timeout value in seconds. + :type node: dict + :type timeout: int + :raises HoneycombError: If the configuration could not be changed. + """ + + find = "jvpp-request-timeout" + replace = '\\"jvpp-request-timeout\\": {0}'.format(timeout) + + argument = '"/{0}/c\\ {1}"'.format(find, replace) + path = "{0}/config/jvpp.json".format(Const.REMOTE_HC_DIR) + command = "sed -i {0} {1}".format(argument, path) + + ssh = SSH() + ssh.connect(node) + (ret_code, _, stderr) = ssh.exec_command_sudo(command) + if ret_code != 0: + raise HoneycombError("Failed to modify configuration on " + "node {0}, {1}".format(node, stderr)) @staticmethod def print_environment(nodes): @@ -243,8 +309,7 @@ class HoneycombSetup(object): "which java", "java -version", "dpkg --list | grep openjdk", - "ls -la /opt/honeycomb", - "ls -la /opt/honeycomb/v3po-karaf-1.0.0-SNAPSHOT") + "ls -la /opt/honeycomb") for node in nodes: if node['type'] == NodeType.DUT: @@ -274,3 +339,455 @@ class HoneycombSetup(object): ssh.connect(node) ssh.exec_command_sudo(cmd) + @staticmethod + def configure_log_level(node, level): + """Set Honeycomb logging to the specified level. + + :param node: Honeycomb node. + :param level: Log level (INFO, DEBUG, TRACE). + :type node: dict + :type level: str + """ + + find = 'logger name=\\"io.fd\\"' + replace = ''.format(level) + + argument = '"/{0}/c\\ {1}"'.format(find, replace) + path = "{0}/config/logback.xml".format(Const.REMOTE_HC_DIR) + command = "sed -i {0} {1}".format(argument, path) + + ssh = SSH() + ssh.connect(node) + (ret_code, _, stderr) = ssh.exec_command_sudo(command) + if ret_code != 0: + raise HoneycombError("Failed to modify configuration on " + "node {0}, {1}".format(node, stderr)) + + @staticmethod + def manage_honeycomb_features(node, feature, disable=False): + """Configure Honeycomb to use features that are disabled by default, or + disable previously enabled features. + + ..Note:: If the module is not enabled in VPP, Honeycomb will + be unable to establish VPP connection. + + :param node: Honeycomb node. + :param feature: Feature to enable. + :param disable: Disable the specified feature instead of enabling it. + :type node: dict + :type feature: string + :type disable: bool + :raises HoneycombError: If the configuration could not be changed. + """ + + disabled_features = { + "NSH": "io.fd.hc2vpp.vppnsh.impl.VppNshModule" + } + + ssh = SSH() + ssh.connect(node) + + if feature in disabled_features.keys(): + # uncomment by replacing the entire line + find = replace = "{0}".format(disabled_features[feature]) + if disable: + replace = "// {0}".format(find) + + argument = '"/{0}/c\\ {1}"'.format(find, replace) + path = "{0}/modules/*module-config"\ + .format(Const.REMOTE_HC_DIR) + command = "sed -i {0} {1}".format(argument, path) + + (ret_code, _, stderr) = ssh.exec_command_sudo(command) + if ret_code != 0: + raise HoneycombError("Failed to modify configuration on " + "node {0}, {1}".format(node, stderr)) + else: + raise HoneycombError( + "Unrecognized feature {0}.".format(feature)) + + @staticmethod + def copy_java_libraries(node): + """Copy Java libraries installed by vpp-api-java package to honeycomb + lib folder. + + This is a (temporary?) workaround for jvpp version mismatches. + + :param node: Honeycomb node + :type node: dict + """ + + ssh = SSH() + ssh.connect(node) + (_, stdout, _) = ssh.exec_command_sudo( + "ls /usr/share/java | grep ^jvpp-*") + + files = stdout.split("\n")[:-1] + for item in files: + # example filenames: + # jvpp-registry-17.04.jar + # jvpp-core-17.04.jar + + parts = item.split("-") + version = "{0}-SNAPSHOT".format(parts[2][:5]) + artifact_id = "{0}-{1}".format(parts[0], parts[1]) + + directory = "{0}/lib/io/fd/vpp/{1}/{2}".format( + Const.REMOTE_HC_DIR, artifact_id, version) + cmd = "sudo mkdir -p {0}; " \ + "sudo cp /usr/share/java/{1} {0}/{2}-{3}.jar".format( + directory, item, artifact_id, version) + + (ret_code, _, stderr) = ssh.exec_command(cmd) + if ret_code != 0: + raise HoneycombError("Failed to copy JVPP libraries on " + "node {0}, {1}".format(node, stderr)) + + @staticmethod + def copy_odl_client(node, odl_name, src_path, dst_path): + """Copy ODL Client from source path to destination path. + + :param node: Honeycomb node. + :param odl_name: Name of ODL client version to use. + :param src_path: Source Path where to find ODl client. + :param dst_path: Destination path. + :type node: dict + :type odl_name: str + :type src_path: str + :type dst_path: str + :raises HoneycombError: If the operation fails. + """ + + ssh = SSH() + ssh.connect(node) + + cmd = "cp -r {src}/*karaf_{odl_name}* {dst}".format( + src=src_path, odl_name=odl_name, dst=dst_path) + + ret_code, _, _ = ssh.exec_command(cmd, timeout=60) + if int(ret_code) != 0: + raise HoneycombError( + "Failed to copy ODL client on node {0}".format(node["host"])) + + @staticmethod + def setup_odl_client(node, path): + """Start ODL client on the specified node. + + Karaf should be located in the provided path, and VPP and Honeycomb + should already be running, otherwise the start will fail. + :param node: Node to start ODL client on. + :param path: Path to ODL client on node. + :type node: dict + :type path: str + :raises HoneycombError: If Honeycomb fails to start. + """ + + logger.console("\nStarting ODL client ...") + ssh = SSH() + ssh.connect(node) + + cmd = "{path}/*karaf*/bin/start clean".format(path=path) + ret_code, _, _ = ssh.exec_command_sudo(cmd) + + if int(ret_code) != 0: + raise HoneycombError('Node {0} failed to start ODL.'. + format(node['host'])) + else: + logger.info("Starting the ODL client on node {0} is " + "in progress ...".format(node['host'])) + + @staticmethod + def install_odl_features(node, path, *features): + """Install required features on a running ODL client. + + :param node: Honeycomb node. + :param path: Path to ODL client on node. + :param features: Optional, list of additional features to install. + :type node: dict + :type path: str + :type features: list + """ + + ssh = SSH() + ssh.connect(node) + + cmd = "{path}/*karaf*/bin/client -u karaf feature:install " \ + "odl-restconf-all odl-netconf-connector-all".format(path=path) + for feature in features: + cmd += " {0}".format(feature) + + ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=120) + + if int(ret_code) != 0: + raise HoneycombError("Feature install did not succeed.") + + @staticmethod + def check_odl_startup_state(node): + """Check the status of ODL client startup. + + :param node: Honeycomb node. + :param node: dict + :returns: True when ODL is started. + :rtype: bool + :raises HoneycombError: When the response is not code 200: OK. + """ + + path = HcUtil.read_path_from_url_file( + "odl_client/odl_netconf_connector") + expected_status_codes = (HTTPCodes.UNAUTHORIZED, + HTTPCodes.FORBIDDEN, + HTTPCodes.NOT_FOUND, + HTTPCodes.SERVICE_UNAVAILABLE, + HTTPCodes.INTERNAL_SERVER_ERROR) + + status_code, _ = HTTPRequest.get(node, path, timeout=10, + enable_logging=False) + if status_code == HTTPCodes.OK: + logger.info("ODL client on node {0} is up and running". + format(node['host'])) + elif status_code in expected_status_codes: + if status_code == HTTPCodes.UNAUTHORIZED: + logger.info('Unauthorized. If this triggers keyword ' + 'timeout, verify username and password.') + raise HoneycombError('ODL client on node {0} running but ' + 'not yet ready.'.format(node['host']), + enable_logging=False) + else: + raise HoneycombError('Unexpected return code: {0}.'. + format(status_code)) + return True + + @staticmethod + def check_odl_shutdown_state(node): + """Check the status of ODL client shutdown. + + :param node: Honeycomb node. + :type node: dict + :returns: True when ODL is stopped. + :rtype: bool + :raises HoneycombError: When the response is not code 200: OK. + """ + + cmd = "pgrep -f karaf" + path = HcUtil.read_path_from_url_file( + "odl_client/odl_netconf_connector") + + try: + status_code, _ = HTTPRequest.get(node, path, timeout=10, + enable_logging=False) + raise HoneycombError("ODL client is still running.") + except HTTPRequestError: + logger.debug("Connection refused, checking process state....") + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = ssh.exec_command(cmd) + if ret_code == 0: + raise HoneycombError("ODL client is still running.") + + return True + + @staticmethod + def mount_honeycomb_on_odl(node): + """Tell ODL client to mount Honeycomb instance over netconf. + + :param node: Honeycomb node. + :type node: dict + :raises HoneycombError: When the response is not code 200: OK. + """ + + path = HcUtil.read_path_from_url_file( + "odl_client/odl_netconf_connector") + + url_file = "{0}/{1}".format(Const.RESOURCES_TPL_HC, + "odl_client/mount_honeycomb.xml") + + with open(url_file) as template: + data = template.read() + + status_code, _ = HTTPRequest.post( + node, path, headers={"Content-Type": "application/xml"}, + payload=data, timeout=10, enable_logging=False) + + if status_code == HTTPCodes.OK: + logger.info("ODL mount point configured successfully.") + elif status_code == HTTPCodes.CONFLICT: + logger.info("ODL mount point was already configured.") + else: + raise HoneycombError('Mount point configuration not successful') + + @staticmethod + def stop_odl_client(node, path): + """Stop ODL client service on the specified node. + + :param node: Node to start ODL client on. + :param path: Path to ODL client. + :type node: dict + :type path: str + :raises HoneycombError: If ODL client fails to stop. + """ + + ssh = SSH() + ssh.connect(node) + + cmd = "{0}/*karaf*/bin/stop".format(path) + + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + logger.debug("ODL Client refused to shut down.") + cmd = "pkill -f 'karaf'" + (ret_code, _, _) = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + raise HoneycombError('Node {0} failed to stop ODL.'. + format(node['host'])) + + logger.info("ODL client service stopped.") + + @staticmethod + def stop_vpp_service(node): + """Stop VPP service on the specified node. + + :param node: VPP node. + :type node: dict + :raises RuntimeError: If VPP fails to stop. + """ + + ssh = SSH() + ssh.connect(node) + cmd = "service vpp stop" + ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=80) + if int(ret_code) != 0: + logger.debug("VPP service refused to shut down.") + +class HoneycombStartupConfig(object): + """Generator for Honeycomb startup configuration. + """ + def __init__(self): + """Initializer.""" + + self.template = """ + #!/bin/sh - + STATUS=100 + + while [ $STATUS -eq 100 ] + do + {java_call} -jar $(dirname $0)/{jar_filename} + STATUS=$? + echo "Honeycomb exited with status: $STATUS" + if [ $STATUS -eq 100 ] + then + echo "Restarting..." + fi + done + """ + + self.java_call = "{scheduler} {affinity} java {jit_mode} {params}" + + self.scheduler = "" + self.core_affinity = "" + self.jit_mode = "" + self.params = "" + self.numa = "" + + self.config = "" + self.ssh = SSH() + + def apply_config(self, node): + """Generate configuration file /opt/honeycomb/honeycomb on the specified + node. + + :param node: Honeycomb node. + :type node: dict + """ + + self.ssh.connect(node) + _, filename, _ = self.ssh.exec_command("ls /opt/honeycomb | grep .jar") + + java_call = self.java_call.format(scheduler=self.scheduler, + affinity=self.core_affinity, + jit_mode=self.jit_mode, + params=self.params) + self.config = self.template.format(java_call=java_call, + jar_filename=filename) + + self.ssh.connect(node) + cmd = "echo '{config}' > /tmp/honeycomb " \ + "&& chmod +x /tmp/honeycomb " \ + "&& sudo mv -f /tmp/honeycomb /opt/honeycomb".format( + config=self.config) + self.ssh.exec_command(cmd) + + def set_cpu_scheduler(self, scheduler="FIFO"): + """Use alternate CPU scheduler. + + Note: OTHER scheduler doesn't load-balance over isolcpus. + + :param scheduler: CPU scheduler to use. + :type scheduler: str + """ + + schedulers = {"FIFO": "-f 99", # First In, First Out + "RR": "-r 99", # Round Robin + "OTHER": "-o", # Ubuntu default + } + self.scheduler = "chrt {0}".format(schedulers[scheduler]) + + def set_cpu_core_affinity(self, low, high=None): + """Set core affinity for the honeycomb process and subprocesses. + + :param low: Lowest core ID number. + :param high: Highest core ID number. Leave empty to use a single core. + :type low: int + :type high: int + """ + + self.core_affinity = "taskset -c {low}-{high}".format( + low=low, high=high if high else low) + + def set_jit_compiler_mode(self, jit_mode): + """Set running mode for Java's JIT compiler. + + :param jit_mode: Desiret JIT mode. + :type jit_mode: str + """ + + modes = {"client": "-client", # Default + "server": "-server", # Higher performance but longer warmup + "classic": "-classic" # Disables JIT compiler + } + + self.jit_mode = modes[jit_mode] + + def set_memory_size(self, mem_min, mem_max=None): + """Set minimum and maximum memory use for the JVM. + + :param mem_min: Minimum amount of memory (MB). + :param mem_max: Maximum amount of memory (MB). Default is 4 times + minimum value. + :type mem_min: int + :type mem_max: int + """ + + self.params += " -Xms{min}m -Xmx{max}m".format( + min=mem_min, max=mem_max if mem_max else mem_min*4) + + def set_metaspace_size(self, mem_min, mem_max=None): + """Set minimum and maximum memory used for class metadata in the JVM. + + :param mem_min: Minimum metaspace size (MB). + :param mem_max: Maximum metaspace size (MB). Defailt is 4 times + minimum value. + :type mem_min: int + :type mem_max: int + """ + + self.params += " -XX:MetaspaceSize={min}m " \ + "-XX:MaxMetaspaceSize={max}m".format( + min=mem_min, max=mem_max if mem_max else mem_min*4) + + def set_numa_optimization(self): + """Use optimization of memory use and garbage collection for NUMA + architectures.""" + + self.params += " -XX:+UseNUMA -XX:+UseParallelGC"