X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2Fhoneycomb%2FHoneycombSetup.py;h=fba2bd42257d0f3cffe6b0430d32aab860da64f2;hp=551d44b5ae368d01fcb0e66c302ad9b6ddccbde4;hb=5a02dd13563a5c67e336f04eb526cbea206da29b;hpb=44cafa8a265cfb1144638430079ef4dbf2501d72 diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py index 551d44b5ae..fba2bd4225 100644 --- a/resources/libraries/python/honeycomb/HoneycombSetup.py +++ b/resources/libraries/python/honeycomb/HoneycombSetup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Cisco and/or its affiliates. +# Copyright (c) 2017 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -13,6 +13,9 @@ """Implementation of keywords for Honeycomb setup.""" +from json import loads +from time import time, sleep + from ipaddress import IPv6Address, AddressValueError from robot.api import logger @@ -58,7 +61,7 @@ class HoneycombSetup(object): HoneycombSetup.print_environment(nodes) - logger.console("\nStarting Honeycomb service ...") + logger.console("\n(re)Starting Honeycomb service ...") cmd = "sudo service honeycomb start" @@ -106,127 +109,115 @@ class HoneycombSetup(object): format(errors)) @staticmethod - def restart_honeycomb_and_vpp_on_duts(*nodes): - """Restart the Honeycomb service on specified DUT nodes. + def restart_honeycomb_on_dut(node): + """Restart Honeycomb on specified DUT nodes. - Use the keyword "Check Honeycomb Startup State" to check when Honeycomb - is fully restarted. - :param nodes: List of nodes to restart Honeycomb on. - :type nodes: list - :raises HoneycombError: If Honeycomb failed to restart. + This keyword restarts the Honeycomb service on specified DUTs. Use the + keyword "Check Honeycomb Startup State" to check if the Honeycomb is up + and running. + + :param node: Node to restart Honeycomb on. + :type node: dict + :raises HoneycombError: If Honeycomb fails to start. """ - logger.console("\nRestarting Honeycomb service ...") - cmd = "sudo service honeycomb restart && sudo service vpp restart" - errors = [] + logger.console("\n(re)Starting Honeycomb service ...") - for node in nodes: - if node['type'] == NodeType.DUT: - ssh = SSH() - ssh.connect(node) - (ret_code, _, _) = ssh.exec_command_sudo(cmd) - if int(ret_code) != 0: - errors.append(node['host']) - else: - logger.info("Restart of Honeycomb and VPP on node {0} is " - "in progress ...".format(node['host'])) - if errors: - raise HoneycombError('Node(s) {0} failed to restart Honeycomb' - ' and/or VPP.'. - format(errors)) + cmd = "sudo service honeycomb restart" - @staticmethod - def check_honeycomb_startup_state(*nodes): - """Check state of Honeycomb service during startup on specified nodes. + ssh = SSH() + ssh.connect(node) + (ret_code, _, _) = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + raise HoneycombError('Node {0} failed to restart Honeycomb.'. + format(node['host'])) + else: + logger.info( + "Honeycomb service restart is in progress on node {0}".format( + node['host'])) - Reads html path from template file oper_vpp_version.url. + @staticmethod + def check_honeycomb_startup_state(node, timeout=360, retries=20, + interval=15): + """Repeatedly check the status of Honeycomb startup until it is fully + started or until timeout or max retries is reached. - Honeycomb nodes reply with connection refused or the following status - codes depending on startup progress: codes 200, 401, 403, 404, 500, 503 + :param node: Honeycomb node. + :param timeout: Timeout value in seconds. + :param retries: Max number of retries. + :param interval: Interval between checks, in seconds. + :type node: dict + :type timeout: int + :type retries: int + :type interval: int + :raises HoneycombError: If the Honeycomb process IP cannot be found, + or if timeout or number of retries is exceeded.""" - :param nodes: List of DUT nodes starting Honeycomb. - :type nodes: list - :return: True if all GETs returned code 200(OK). - :rtype bool - """ - path = HcUtil.read_path_from_url_file("oper_vpp_version") - expected_status_codes = (HTTPCodes.UNAUTHORIZED, - HTTPCodes.FORBIDDEN, - HTTPCodes.NOT_FOUND, - HTTPCodes.SERVICE_UNAVAILABLE, - HTTPCodes.INTERNAL_SERVER_ERROR) + ssh = SSH() + ssh.connect(node) - for node in nodes: - if node['type'] == NodeType.DUT: - HoneycombSetup.print_ports(node) - status_code, _ = HTTPRequest.get(node, path, timeout=10, - enable_logging=False) - if status_code == HTTPCodes.OK: - logger.info("Honeycomb on node {0} is up and running". - format(node['host'])) - elif status_code in expected_status_codes: - if status_code == HTTPCodes.UNAUTHORIZED: - logger.info('Unauthorized. If this triggers keyword ' - 'timeout, verify Honeycomb username and ' - 'password.') - raise HoneycombError('Honeycomb on node {0} running but ' - 'not yet ready.'.format(node['host']), - enable_logging=False) - else: - raise HoneycombError('Unexpected return code: {0}.'. - format(status_code)) + count = 0 + start = time() + while time() - start < timeout and count < retries: + count += 1 - status_code, _ = HcUtil.get_honeycomb_data( + try: + status_code_version, _ = HcUtil.get_honeycomb_data( + node, "oper_vpp_version") + status_code_if_cfg, _ = HcUtil.get_honeycomb_data( node, "config_vpp_interfaces") - if status_code != HTTPCodes.OK: - raise HoneycombError('Honeycomb on node {0} running but ' - 'not yet ready.'.format(node['host']), - enable_logging=False) - return True + status_code_if_oper, _ = HcUtil.get_honeycomb_data( + node, "oper_vpp_interfaces") + except HTTPRequestError: + sleep(interval) + continue + if status_code_if_cfg == HTTPCodes.OK\ + and status_code_if_cfg == HTTPCodes.OK\ + and status_code_if_oper == HTTPCodes.OK: + logger.info("Check successful, Honeycomb is up and running.") + break + else: + logger.debug( + "Attempt ${count} failed on Restconf check. Status codes:\n" + "Version: {version}\n" + "Interface config: {if_cfg}\n" + "Interface operational: {if_oper}".format( + count=count, + version=status_code_version, + if_cfg=status_code_if_cfg, + if_oper=status_code_if_oper)) + sleep(interval) + continue + else: + _, vpp_status, _ = ssh.exec_command("sudo service vpp status") + raise HoneycombError( + "Timeout or max retries exceeded. Status of VPP:\n" + "{vpp_status}".format(vpp_status=vpp_status)) @staticmethod - def check_honeycomb_shutdown_state(*nodes): + def check_honeycomb_shutdown_state(node): """Check state of Honeycomb service during shutdown on specified nodes. Honeycomb nodes reply with connection refused or the following status codes depending on shutdown progress: codes 200, 404. - :param nodes: List of DUT nodes stopping Honeycomb. - :type nodes: list + :param node: List of DUT nodes stopping Honeycomb. + :type node: dict :return: True if all GETs fail to connect. :rtype bool """ - cmd = "ps -ef | grep -v grep | grep honeycomb" - for node in nodes: - if node['type'] == NodeType.DUT: - try: - status_code, _ = HTTPRequest.get(node, '/index.html', - timeout=5, - enable_logging=False) - if status_code == HTTPCodes.OK: - raise HoneycombError('Honeycomb on node {0} is still ' - 'running.'.format(node['host']), - enable_logging=False) - elif status_code == HTTPCodes.NOT_FOUND: - raise HoneycombError('Honeycomb on node {0} is shutting' - ' down.'.format(node['host']), - enable_logging=False) - else: - raise HoneycombError('Unexpected return code: {0}.'. - format(status_code)) - except HTTPRequestError: - logger.debug('Connection refused, checking the process ' - 'state ...') - ssh = SSH() - ssh.connect(node) - (ret_code, _, _) = ssh.exec_command_sudo(cmd) - if ret_code == 0: - raise HoneycombError('Honeycomb on node {0} is still ' - 'running.'.format(node['host']), - enable_logging=False) - else: - logger.info("Honeycomb on node {0} has stopped". - format(node['host'])) + cmd = "pgrep honeycomb" + + ssh = SSH() + ssh.connect(node) + (ret_code, _, _) = ssh.exec_command_sudo(cmd) + if ret_code == 0: + raise HoneycombError('Honeycomb on node {0} is still ' + 'running.'.format(node['host']), + enable_logging=False) + else: + logger.info("Honeycomb on node {0} has stopped". + format(node['host'])) return True @staticmethod @@ -248,7 +239,32 @@ class HoneycombSetup(object): replace = '\\"restconf-binding-address\\": \\"0.0.0.0\\",' argument = '"/{0}/c\\ {1}"'.format(find, replace) - path = "{0}/config/honeycomb.json".format(Const.REMOTE_HC_DIR) + path = "{0}/config/restconf.json".format(Const.REMOTE_HC_DIR) + command = "sed -i {0} {1}".format(argument, path) + + ssh = SSH() + ssh.connect(node) + (ret_code, _, stderr) = ssh.exec_command_sudo(command) + if ret_code != 0: + raise HoneycombError("Failed to modify configuration on " + "node {0}, {1}".format(node, stderr)) + + @staticmethod + def configure_jvpp_timeout(node, timeout=10): + """Configure timeout value for Java API commands Honeycomb sends to VPP. + + :param node: Information about a DUT node. + :param timeout: Timeout value in seconds. + :type node: dict + :type timeout: int + :raises HoneycombError: If the configuration could not be changed. + """ + + find = "jvpp-request-timeout" + replace = '\\"jvpp-request-timeout\\": {0}'.format(timeout) + + argument = '"/{0}/c\\ {1}"'.format(find, replace) + path = "{0}/config/jvpp.json".format(Const.REMOTE_HC_DIR) command = "sed -i {0} {1}".format(argument, path) ssh = SSH() @@ -277,7 +293,8 @@ class HoneycombSetup(object): "which java", "java -version", "dpkg --list | grep openjdk", - "ls -la /opt/honeycomb") + "ls -la /opt/honeycomb", + "cat /opt/honeycomb/modules/*module-config") for node in nodes: if node['type'] == NodeType.DUT: @@ -332,30 +349,39 @@ class HoneycombSetup(object): "node {0}, {1}".format(node, stderr)) @staticmethod - def enable_module_features(node, *features): - """Configure Honeycomb to use VPP modules that are disabled by default. + def manage_honeycomb_features(node, feature, disable=False): + """Configure Honeycomb to use features that are disabled by default, or + disable previously enabled features. ..Note:: If the module is not enabled in VPP, Honeycomb will be unable to establish VPP connection. :param node: Honeycomb node. - :param features: Features to enable. + :param feature: Feature to enable. + :param disable: Disable the specified feature instead of enabling it. :type node: dict - :type features: string + :type feature: string + :type disable: bool :raises HoneycombError: If the configuration could not be changed. """ disabled_features = { - "NSH": "io.fd.hc2vpp.vppnsh.impl.VppNshModule" + "NSH": ["io.fd.hc2vpp.vppnsh.impl.VppNshModule"], + "BGP": ["io.fd.hc2vpp.bgp.inet.BgpInetModule", + "io.fd.honeycomb.infra.bgp.BgpModule", + "io.fd.honeycomb.infra.bgp.BgpReadersModule", + "io.fd.honeycomb.infra.bgp.BgpWritersModule"] } ssh = SSH() ssh.connect(node) - for feature in features: - if feature in disabled_features.keys(): - # uncomment by replacing the entire line - find = replace = "{0}".format(disabled_features[feature]) + if feature in disabled_features.keys(): + # for every module, uncomment by replacing the entire line + for item in disabled_features[feature]: + find = replace = "{0}".format(item) + if disable: + replace = "// {0}".format(find) argument = '"/{0}/c\\ {1}"'.format(find, replace) path = "{0}/modules/*module-config"\ @@ -366,9 +392,9 @@ class HoneycombSetup(object): if ret_code != 0: raise HoneycombError("Failed to modify configuration on " "node {0}, {1}".format(node, stderr)) - else: - raise HoneycombError( - "Unrecognized feature {0}.".format(feature)) + else: + raise HoneycombError( + "Unrecognized feature {0}.".format(feature)) @staticmethod def copy_java_libraries(node): @@ -408,41 +434,52 @@ class HoneycombSetup(object): "node {0}, {1}".format(node, stderr)) @staticmethod - def find_odl_client(node): - """Check if there is a karaf directory in home. + def copy_odl_client(node, odl_name, src_path, dst_path): + """Copy ODL Client from source path to destination path. :param node: Honeycomb node. + :param odl_name: Name of ODL client version to use. + :param src_path: Source Path where to find ODl client. + :param dst_path: Destination path. :type node: dict - :returns: True if ODL client is present on node, else False. - :rtype: bool + :type odl_name: str + :type src_path: str + :type dst_path: str + :raises HoneycombError: If the operation fails. """ ssh = SSH() ssh.connect(node) - (ret_code, stdout, _) = ssh.exec_command_sudo( - "ls ~ | grep karaf") - logger.debug(stdout) - return not bool(ret_code) + cmd = "sudo rm -rf {dst}/*karaf_{odl_name} && " \ + "cp -r {src}/*karaf_{odl_name}* {dst}".format( + src=src_path, odl_name=odl_name, dst=dst_path) + + ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=180) + if int(ret_code) != 0: + raise HoneycombError( + "Failed to copy ODL client on node {0}".format(node["host"])) @staticmethod - def start_odl_client(node): + def setup_odl_client(node, path): """Start ODL client on the specified node. - karaf should be located in home directory, and VPP and Honeycomb should - already be running, otherwise the start will fail. - :param node: Nodes to start ODL client on. + Karaf should be located in the provided path, and VPP and Honeycomb + should already be running, otherwise the start will fail. + :param node: Node to start ODL client on. + :param path: Path to ODL client on node. :type node: dict + :type path: str :raises HoneycombError: If Honeycomb fails to start. """ logger.console("\nStarting ODL client ...") - - cmd = "~/*karaf*/bin/start" - ssh = SSH() ssh.connect(node) - (ret_code, _, _) = ssh.exec_command_sudo(cmd) + + cmd = "{path}/*karaf*/bin/start clean".format(path=path) + ret_code, _, _ = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: raise HoneycombError('Node {0} failed to start ODL.'. format(node['host'])) @@ -450,6 +487,33 @@ class HoneycombSetup(object): logger.info("Starting the ODL client on node {0} is " "in progress ...".format(node['host'])) + @staticmethod + def install_odl_features(node, path, *features): + """Install required features on a running ODL client. + + :param node: Honeycomb node. + :param path: Path to ODL client on node. + :param features: Optional, list of additional features to install. + :type node: dict + :type path: str + :type features: list + """ + + ssh = SSH() + ssh.connect(node) + + cmd = "{path}/*karaf*/bin/client -u karaf feature:install " \ + "odl-restconf-all " \ + "odl-netconf-connector-all " \ + "odl-netconf-topology".format(path=path) + for feature in features: + cmd += " {0}".format(feature) + + ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=250) + + if int(ret_code) != 0: + raise HoneycombError("Feature install did not succeed.") + @staticmethod def check_odl_startup_state(node): """Check the status of ODL client startup. @@ -486,6 +550,34 @@ class HoneycombSetup(object): format(status_code)) return True + @staticmethod + def check_odl_shutdown_state(node): + """Check the status of ODL client shutdown. + + :param node: Honeycomb node. + :type node: dict + :returns: True when ODL is stopped. + :rtype: bool + :raises HoneycombError: When the response is not code 200: OK. + """ + + cmd = "pgrep -f karaf" + path = HcUtil.read_path_from_url_file( + "odl_client/odl_netconf_connector") + + try: + HTTPRequest.get(node, path, timeout=10, enable_logging=False) + raise HoneycombError("ODL client is still running.") + except HTTPRequestError: + logger.debug("Connection refused, checking process state....") + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = ssh.exec_command(cmd) + if ret_code == 0: + raise HoneycombError("ODL client is still running.") + + return True + @staticmethod def mount_honeycomb_on_odl(node): """Tell ODL client to mount Honeycomb instance over netconf. @@ -499,18 +591,194 @@ class HoneycombSetup(object): "odl_client/odl_netconf_connector") url_file = "{0}/{1}".format(Const.RESOURCES_TPL_HC, - "odl_client/mount_honeycomb.xml") + "odl_client/mount_honeycomb.json") with open(url_file) as template: data = template.read() + data = loads(data) + status_code, _ = HTTPRequest.post( - node, path, headers={"Content-Type": "application/xml"}, - payload=data, timeout=10, enable_logging=False) + node, + path, + headers={"Content-Type": "application/json", + "Accept": "text/plain"}, + json=data, + timeout=10, + enable_logging=False) if status_code == HTTPCodes.OK: logger.info("ODL mount point configured successfully.") elif status_code == HTTPCodes.CONFLICT: - logger.warn("ODL mount point was already configured.") + logger.info("ODL mount point was already configured.") else: raise HoneycombError('Mount point configuration not successful') + + @staticmethod + def stop_odl_client(node, path): + """Stop ODL client service on the specified node. + + :param node: Node to start ODL client on. + :param path: Path to ODL client. + :type node: dict + :type path: str + :raises HoneycombError: If ODL client fails to stop. + """ + + ssh = SSH() + ssh.connect(node) + + cmd = "{0}/*karaf*/bin/stop".format(path) + + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + logger.debug("ODL Client refused to shut down.") + cmd = "pkill -f 'karaf'" + (ret_code, _, _) = ssh.exec_command_sudo(cmd) + if int(ret_code) != 0: + raise HoneycombError('Node {0} failed to stop ODL.'. + format(node['host'])) + + logger.info("ODL client service stopped.") + + + +class HoneycombStartupConfig(object): + """Generator for Honeycomb startup configuration. + """ + def __init__(self): + """Initializer.""" + + self.template = """#!/bin/sh - + STATUS=100 + + while [ $STATUS -eq 100 ] + do + {java_call} -jar $(dirname $0)/{jar_filename} + STATUS=$? + echo "Honeycomb exited with status: $STATUS" + if [ $STATUS -eq 100 ] + then + echo "Restarting..." + fi + done + """ + + self.java_call = "{scheduler} {affinity} java{jit_mode}{params}" + + self.scheduler = "" + self.core_affinity = "" + self.jit_mode = "" + self.params = "" + self.numa = "" + + self.config = "" + self.ssh = SSH() + + def apply_config(self, node): + """Generate configuration file /opt/honeycomb/honeycomb on the specified + node. + + :param node: Honeycomb node. + :type node: dict + """ + + self.ssh.connect(node) + _, filename, _ = self.ssh.exec_command("ls /opt/honeycomb | grep .jar") + + java_call = self.java_call.format(scheduler=self.scheduler, + affinity=self.core_affinity, + jit_mode=self.jit_mode, + params=self.params) + self.config = self.template.format(java_call=java_call, + jar_filename=filename) + + self.ssh.connect(node) + cmd = "echo '{config}' > /tmp/honeycomb " \ + "&& chmod +x /tmp/honeycomb " \ + "&& sudo mv -f /tmp/honeycomb /opt/honeycomb".\ + format(config=self.config) + self.ssh.exec_command(cmd) + + def set_cpu_scheduler(self, scheduler="FIFO"): + """Use alternate CPU scheduler. + + Note: OTHER scheduler doesn't load-balance over isolcpus. + + :param scheduler: CPU scheduler to use. + :type scheduler: str + """ + + schedulers = {"FIFO": "-f 99", # First In, First Out + "RR": "-r 99", # Round Robin + "OTHER": "-o", # Ubuntu default + } + self.scheduler = "chrt {0}".format(schedulers[scheduler]) + + def set_cpu_core_affinity(self, low, high=None): + """Set core affinity for the honeycomb process and subprocesses. + + :param low: Lowest core ID number. + :param high: Highest core ID number. Leave empty to use a single core. + :type low: int + :type high: int + """ + + self.core_affinity = "taskset -c {low}-{high}".format( + low=low, high=high if high else low) + + def set_jit_compiler_mode(self, jit_mode): + """Set running mode for Java's JIT compiler. + + :param jit_mode: Desiret JIT mode. + :type jit_mode: str + """ + + modes = {"client": " -client", # Default + "server": " -server", # Higher performance but longer warmup + "classic": " -classic" # Disables JIT compiler + } + + self.jit_mode = modes[jit_mode] + + def set_memory_size(self, mem_min, mem_max=None): + """Set minimum and maximum memory use for the JVM. + + :param mem_min: Minimum amount of memory (MB). + :param mem_max: Maximum amount of memory (MB). Default is 4 times + minimum value. + :type mem_min: int + :type mem_max: int + """ + + self.params += " -Xms{min}m -Xmx{max}m".format( + min=mem_min, max=mem_max if mem_max else mem_min*4) + + def set_metaspace_size(self, mem_min, mem_max=None): + """Set minimum and maximum memory used for class metadata in the JVM. + + :param mem_min: Minimum metaspace size (MB). + :param mem_max: Maximum metaspace size (MB). Defailt is 4 times + minimum value. + :type mem_min: int + :type mem_max: int + """ + + self.params += " -XX:MetaspaceSize={min}m " \ + "-XX:MaxMetaspaceSize={max}m".format( + min=mem_min, max=mem_max if mem_max else mem_min*4) + + def set_numa_optimization(self): + """Use optimization of memory use and garbage collection for NUMA + architectures.""" + + self.params += " -XX:+UseNUMA -XX:+UseParallelGC" + + def set_ssh_security_provider(self): + """Disables BouncyCastle for SSHD.""" + # Workaround for issue described in: + # https://wiki.fd.io/view/Honeycomb/Releases/1609/Honeycomb_and_ODL + + self.params += " -Dorg.apache.sshd.registerBouncyCastle=false"