X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2Fhoneycomb%2FHoneycombSetup.py;h=32992c03defa3af0d0e1a6ebeca450e51890eb9f;hp=50fd1c997d495129270bf7fbbd2766a3f2a2644e;hb=dc0288317e241f6cb18b0978993a1e2f396c793c;hpb=919b7cb359cc8fd06ca8cc7a651391ac6aa03f20 diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py index 50fd1c997d..32992c03de 100644 --- a/resources/libraries/python/honeycomb/HoneycombSetup.py +++ b/resources/libraries/python/honeycomb/HoneycombSetup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Cisco and/or its affiliates. +# Copyright (c) 2017 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -58,7 +58,7 @@ class HoneycombSetup(object): HoneycombSetup.print_environment(nodes) - logger.console("\nStarting Honeycomb service ...") + logger.console("\n(re)Starting Honeycomb service ...") cmd = "sudo service honeycomb start" @@ -106,34 +106,32 @@ class HoneycombSetup(object): format(errors)) @staticmethod - def restart_honeycomb_and_vpp_on_duts(*nodes): - """Restart the Honeycomb service on specified DUT nodes. + def restart_honeycomb_on_dut(node): + """Restart Honeycomb on specified DUT nodes. - Use the keyword "Check Honeycomb Startup State" to check when Honeycomb - is fully restarted. - :param nodes: List of nodes to restart Honeycomb on. - :type nodes: list - :raises HoneycombError: If Honeycomb failed to restart. + This keyword restarts the Honeycomb service on specified DUTs. Use the + keyword "Check Honeycomb Startup State" to check if the Honeycomb is up + and running. + + :param node: Node to restart Honeycomb on. + :type node: dict + :raises HoneycombError: If Honeycomb fails to start. """ - logger.console("\nRestarting Honeycomb service ...") - cmd = "sudo service honeycomb restart && sudo service vpp restart" - errors = [] + logger.console("\n(re)Starting Honeycomb service ...") - for node in nodes: - if node['type'] == NodeType.DUT: - ssh = SSH() - ssh.connect(node) - (ret_code, _, _) = ssh.exec_command_sudo(cmd) - if int(ret_code) != 0: - errors.append(node['host']) - else: - logger.info("Restart of Honeycomb and VPP on node {0} is " - "in progress ...".format(node['host'])) - if errors: - raise HoneycombError('Node(s) {0} failed to restart Honeycomb' - ' and/or VPP.'. - format(errors)) + cmd = "sudo service honeycomb restart" + + ssh = SSH() + ssh.connect(node) + (ret_code, _, _) = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + raise HoneycombError('Node {0} failed to restart Honeycomb.'. + format(node['host'])) + else: + logger.info( + "Honeycomb service restart is in progress on node {0}".format( + node['host'])) @staticmethod def check_honeycomb_startup_state(*nodes): @@ -159,8 +157,18 @@ class HoneycombSetup(object): for node in nodes: if node['type'] == NodeType.DUT: HoneycombSetup.print_ports(node) - status_code, _ = HTTPRequest.get(node, path, - enable_logging=False) + try: + status_code, _ = HTTPRequest.get(node, path, + enable_logging=False) + except HTTPRequestError: + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = ssh.exec_command_sudo( + "tail -n 100 /var/log/syslog") + if ret_code != 0: + # It's probably Centos + ssh.exec_command_sudo("tail -n 100 /var/log/messages") + raise if status_code == HTTPCodes.OK: logger.info("Honeycomb on node {0} is up and running". format(node['host'])) @@ -247,7 +255,7 @@ class HoneycombSetup(object): replace = '\\"restconf-binding-address\\": \\"0.0.0.0\\",' argument = '"/{0}/c\\ {1}"'.format(find, replace) - path = "{0}/config/honeycomb.json".format(Const.REMOTE_HC_DIR) + path = "{0}/config/restconf.json".format(Const.REMOTE_HC_DIR) command = "sed -i {0} {1}".format(argument, path) ssh = SSH() @@ -356,16 +364,19 @@ class HoneycombSetup(object): "node {0}, {1}".format(node, stderr)) @staticmethod - def enable_module_features(node, *features): - """Configure Honeycomb to use VPP modules that are disabled by default. + def manage_honeycomb_features(node, feature, disable=False): + """Configure Honeycomb to use features that are disabled by default, or + disable previously enabled features. ..Note:: If the module is not enabled in VPP, Honeycomb will be unable to establish VPP connection. :param node: Honeycomb node. - :param features: Features to enable. + :param feature: Feature to enable. + :param disable: Disable the specified feature instead of enabling it. :type node: dict - :type features: string + :type feature: string + :type disable: bool :raises HoneycombError: If the configuration could not be changed. """ @@ -376,23 +387,24 @@ class HoneycombSetup(object): ssh = SSH() ssh.connect(node) - for feature in features: - if feature in disabled_features.keys(): - # uncomment by replacing the entire line - find = replace = "{0}".format(disabled_features[feature]) - - argument = '"/{0}/c\\ {1}"'.format(find, replace) - path = "{0}/modules/*module-config"\ - .format(Const.REMOTE_HC_DIR) - command = "sed -i {0} {1}".format(argument, path) - - (ret_code, _, stderr) = ssh.exec_command_sudo(command) - if ret_code != 0: - raise HoneycombError("Failed to modify configuration on " - "node {0}, {1}".format(node, stderr)) - else: - raise HoneycombError( - "Unrecognized feature {0}.".format(feature)) + if feature in disabled_features.keys(): + # uncomment by replacing the entire line + find = replace = "{0}".format(disabled_features[feature]) + if disable: + replace = "// {0}".format(find) + + argument = '"/{0}/c\\ {1}"'.format(find, replace) + path = "{0}/modules/*module-config"\ + .format(Const.REMOTE_HC_DIR) + command = "sed -i {0} {1}".format(argument, path) + + (ret_code, _, stderr) = ssh.exec_command_sudo(command) + if ret_code != 0: + raise HoneycombError("Failed to modify configuration on " + "node {0}, {1}".format(node, stderr)) + else: + raise HoneycombError( + "Unrecognized feature {0}.".format(feature)) @staticmethod def copy_java_libraries(node): @@ -432,41 +444,51 @@ class HoneycombSetup(object): "node {0}, {1}".format(node, stderr)) @staticmethod - def find_odl_client(node): - """Check if there is a karaf directory in home. + def copy_odl_client(node, odl_name, src_path, dst_path): + """Copy ODL Client from source path to destination path. :param node: Honeycomb node. + :param odl_name: Name of ODL client version to use. + :param src_path: Source Path where to find ODl client. + :param dst_path: Destination path. :type node: dict - :returns: True if ODL client is present on node, else False. - :rtype: bool + :type odl_name: str + :type src_path: str + :type dst_path: str + :raises HoneycombError: If the operation fails. """ ssh = SSH() ssh.connect(node) - (ret_code, stdout, _) = ssh.exec_command_sudo( - "ls ~ | grep karaf") - logger.debug(stdout) - return not bool(ret_code) + cmd = "cp -r {src}/*karaf_{odl_name}* {dst}".format( + src=src_path, odl_name=odl_name, dst=dst_path) + + ret_code, _, _ = ssh.exec_command(cmd, timeout=60) + if int(ret_code) != 0: + raise HoneycombError( + "Failed to copy ODL client on node {0}".format(node["host"])) @staticmethod - def start_odl_client(node): + def setup_odl_client(node, path): """Start ODL client on the specified node. - karaf should be located in home directory, and VPP and Honeycomb should - already be running, otherwise the start will fail. - :param node: Nodes to start ODL client on. + Karaf should be located in the provided path, and VPP and Honeycomb + should already be running, otherwise the start will fail. + :param node: Node to start ODL client on. + :param path: Path to ODL client on node. :type node: dict + :type path: str :raises HoneycombError: If Honeycomb fails to start. """ logger.console("\nStarting ODL client ...") - - cmd = "~/*karaf*/bin/start" - ssh = SSH() ssh.connect(node) - (ret_code, _, _) = ssh.exec_command_sudo(cmd) + + cmd = "{path}/*karaf*/bin/start clean".format(path=path) + ret_code, _, _ = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: raise HoneycombError('Node {0} failed to start ODL.'. format(node['host'])) @@ -474,6 +496,31 @@ class HoneycombSetup(object): logger.info("Starting the ODL client on node {0} is " "in progress ...".format(node['host'])) + @staticmethod + def install_odl_features(node, path, *features): + """Install required features on a running ODL client. + + :param node: Honeycomb node. + :param path: Path to ODL client on node. + :param features: Optional, list of additional features to install. + :type node: dict + :type path: str + :type features: list + """ + + ssh = SSH() + ssh.connect(node) + + cmd = "{path}/*karaf*/bin/client -u karaf feature:install " \ + "odl-restconf-all odl-netconf-connector-all".format(path=path) + for feature in features: + cmd += " {0}".format(feature) + + ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=120) + + if int(ret_code) != 0: + raise HoneycombError("Feature install did not succeed.") + @staticmethod def check_odl_startup_state(node): """Check the status of ODL client startup. @@ -510,6 +557,35 @@ class HoneycombSetup(object): format(status_code)) return True + @staticmethod + def check_odl_shutdown_state(node): + """Check the status of ODL client shutdown. + + :param node: Honeycomb node. + :type node: dict + :returns: True when ODL is stopped. + :rtype: bool + :raises HoneycombError: When the response is not code 200: OK. + """ + + cmd = "pgrep -f karaf" + path = HcUtil.read_path_from_url_file( + "odl_client/odl_netconf_connector") + + try: + status_code, _ = HTTPRequest.get(node, path, timeout=10, + enable_logging=False) + raise HoneycombError("ODL client is still running.") + except HTTPRequestError: + logger.debug("Connection refused, checking process state....") + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = ssh.exec_command(cmd) + if ret_code == 0: + raise HoneycombError("ODL client is still running.") + + return True + @staticmethod def mount_honeycomb_on_odl(node): """Tell ODL client to mount Honeycomb instance over netconf. @@ -535,6 +611,183 @@ class HoneycombSetup(object): if status_code == HTTPCodes.OK: logger.info("ODL mount point configured successfully.") elif status_code == HTTPCodes.CONFLICT: - logger.warn("ODL mount point was already configured.") + logger.info("ODL mount point was already configured.") else: raise HoneycombError('Mount point configuration not successful') + + @staticmethod + def stop_odl_client(node, path): + """Stop ODL client service on the specified node. + + :param node: Node to start ODL client on. + :param path: Path to ODL client. + :type node: dict + :type path: str + :raises HoneycombError: If ODL client fails to stop. + """ + + ssh = SSH() + ssh.connect(node) + + cmd = "{0}/*karaf*/bin/stop".format(path) + + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + logger.debug("ODL Client refused to shut down.") + cmd = "pkill -f 'karaf'" + (ret_code, _, _) = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + raise HoneycombError('Node {0} failed to stop ODL.'. + format(node['host'])) + + logger.info("ODL client service stopped.") + + @staticmethod + def stop_vpp_service(node): + """Stop VPP service on the specified node. + + :param node: VPP node. + :type node: dict + :raises RuntimeError: If VPP fails to stop. + """ + + ssh = SSH() + ssh.connect(node) + cmd = "service vpp stop" + ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=80) + if int(ret_code) != 0: + logger.debug("VPP service refused to shut down.") + +class HoneycombStartupConfig(object): + """Generator for Honeycomb startup configuration. + """ + def __init__(self): + """Initializer.""" + + self.template = """ + #!/bin/sh - + STATUS=100 + + while [ $STATUS -eq 100 ] + do + {java_call} -jar $(dirname $0)/{jar_filename} + STATUS=$? + echo "Honeycomb exited with status: $STATUS" + if [ $STATUS -eq 100 ] + then + echo "Restarting..." + fi + done + """ + + self.java_call = "{scheduler} {affinity} java {jit_mode} {params}" + + self.scheduler = "" + self.core_affinity = "" + self.jit_mode = "" + self.params = "" + self.numa = "" + + self.config = "" + self.ssh = SSH() + + def apply_config(self, node): + """Generate configuration file /opt/honeycomb/honeycomb on the specified + node. + + :param node: Honeycomb node. + :type node: dict + """ + + self.ssh.connect(node) + _, filename, _ = self.ssh.exec_command("ls /opt/honeycomb | grep .jar") + + java_call = self.java_call.format(scheduler=self.scheduler, + affinity=self.core_affinity, + jit_mode=self.jit_mode, + params=self.params) + self.config = self.template.format(java_call=java_call, + jar_filename=filename) + + self.ssh.connect(node) + cmd = "echo '{config}' > /tmp/honeycomb " \ + "&& chmod +x /tmp/honeycomb " \ + "&& sudo mv -f /tmp/honeycomb /opt/honeycomb".format( + config=self.config) + self.ssh.exec_command(cmd) + + def set_cpu_scheduler(self, scheduler="FIFO"): + """Use alternate CPU scheduler. + + Note: OTHER scheduler doesn't load-balance over isolcpus. + + :param scheduler: CPU scheduler to use. + :type scheduler: str + """ + + schedulers = {"FIFO": "-f 99", # First In, First Out + "RR": "-r 99", # Round Robin + "OTHER": "-o", # Ubuntu default + } + self.scheduler = "chrt {0}".format(schedulers[scheduler]) + + def set_cpu_core_affinity(self, low, high=None): + """Set core affinity for the honeycomb process and subprocesses. + + :param low: Lowest core ID number. + :param high: Highest core ID number. Leave empty to use a single core. + :type low: int + :type high: int + """ + + self.core_affinity = "taskset -c {low}-{high}".format( + low=low, high=high if high else low) + + def set_jit_compiler_mode(self, jit_mode): + """Set running mode for Java's JIT compiler. + + :param jit_mode: Desiret JIT mode. + :type jit_mode: str + """ + + modes = {"client": "-client", # Default + "server": "-server", # Higher performance but longer warmup + "classic": "-classic" # Disables JIT compiler + } + + self.jit_mode = modes[jit_mode] + + def set_memory_size(self, mem_min, mem_max=None): + """Set minimum and maximum memory use for the JVM. + + :param mem_min: Minimum amount of memory (MB). + :param mem_max: Maximum amount of memory (MB). Default is 4 times + minimum value. + :type mem_min: int + :type mem_max: int + """ + + self.params += " -Xms{min}m -Xmx{max}m".format( + min=mem_min, max=mem_max if mem_max else mem_min*4) + + def set_metaspace_size(self, mem_min, mem_max=None): + """Set minimum and maximum memory used for class metadata in the JVM. + + :param mem_min: Minimum metaspace size (MB). + :param mem_max: Maximum metaspace size (MB). Defailt is 4 times + minimum value. + :type mem_min: int + :type mem_max: int + """ + + self.params += " -XX:MetaspaceSize={min}m " \ + "-XX:MaxMetaspaceSize={max}m".format( + min=mem_min, max=mem_max if mem_max else mem_min*4) + + def set_numa_optimization(self): + """Use optimization of memory use and garbage collection for NUMA + architectures.""" + + self.params += " -XX:+UseNUMA -XX:+UseParallelGC"