X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2FPapiExecutor.py;h=60d35a854371f82bd6e4275eb082cdf7a8551127;hb=7b73d46872db5adfc8f4603a9ca783be7d3fa323;hp=0a009b37200f0f1c1c327dfaad6ed9817e98a484;hpb=f88a3d9178dfbd73d0479f9aa2f5224e0c89ca1f;p=csit.git diff --git a/resources/libraries/python/PapiExecutor.py b/resources/libraries/python/PapiExecutor.py index 0a009b3720..60d35a8543 100644 --- a/resources/libraries/python/PapiExecutor.py +++ b/resources/libraries/python/PapiExecutor.py @@ -15,9 +15,11 @@ """ import binascii +import copy import glob import json import shutil +import struct # vpp-papi can raise struct.error import subprocess import sys import tempfile @@ -33,6 +35,8 @@ from resources.libraries.python.PythonThree import raise_from from resources.libraries.python.PapiHistory import PapiHistory from resources.libraries.python.ssh import ( SSH, SSHTimeout, exec_cmd_no_error, scp_node) +from resources.libraries.python.topology import Topology, SocketType +from resources.libraries.python.VppApiCrc import VppApiCrcChecker __all__ = ["PapiExecutor", "PapiSocketExecutor"] @@ -67,6 +71,7 @@ def dictize(obj): ret.__getitem__ = new_get return ret + class PapiSocketExecutor(object): """Methods for executing VPP Python API commands on forwarded socket. @@ -91,8 +96,9 @@ class PapiSocketExecutor(object): Note: Use only with "with" statement, e.g.: + cmd = 'show_version' with PapiSocketExecutor(node) as papi_exec: - reply = papi_exec.add('show_version').get_reply(err_msg) + reply = papi_exec.add(cmd).get_reply(err_msg) This class processes two classes of VPP PAPI methods: 1. Simple request / reply: method='request'. @@ -106,8 +112,9 @@ class PapiSocketExecutor(object): a. One request with no arguments: + cmd = 'show_version' with PapiSocketExecutor(node) as papi_exec: - reply = papi_exec.add('show_version').get_reply(err_msg) + reply = papi_exec.add(cmd).get_reply(err_msg) b. Three requests with arguments, the second and the third ones are the same but with different arguments. @@ -125,9 +132,12 @@ class PapiSocketExecutor(object): """ # Class cache for reuse between instances. - cached_vpp_instance = None + vpp_instance = None + """Takes long time to create, stores all PAPI functions and types.""" + crc_checker = None + """Accesses .api.json files at creation, caching allows deleting them.""" - def __init__(self, node, remote_vpp_socket="/run/vpp-api.sock"): + def __init__(self, node, remote_vpp_socket=Constants.SOCKSVR_PATH): """Store the given arguments, declare managed variables. :param node: Node to connect to and forward unix domain socket from. @@ -143,30 +153,25 @@ class PapiSocketExecutor(object): self._temp_dir = None self._ssh_control_socket = None self._local_vpp_socket = None + self.initialize_vpp_instance() - @property - def vpp_instance(self): - """Return VPP instance with bindings to all API calls. - - The returned instance is initialized for unix domain socket access, - it has initialized all the bindings, but it is not connected - (to local socket) yet. + def initialize_vpp_instance(self): + """Create VPP instance with bindings to API calls, store as class field. - First invocation downloads .api.json files from self._node - into a temporary directory. + No-op if the instance had been stored already. - After first invocation, the result is cached, so other calls are quick. - Class variable is used as the cache, but this property is defined as - an instance method, so that _node (for api files) is known. + The instance is initialized for unix domain socket access, + it has initialized all the bindings, but it is not connected + (to a local socket) yet. - :returns: Initialized but not connected VPP instance. - :rtype: vpp_papi.VPPApiClient + This method downloads .api.json files from self._node + into a temporary directory, deletes them finally. """ - cls = self.__class__ - if cls.cached_vpp_instance is not None: - return cls.cached_vpp_instance + if self.vpp_instance: + return + cls = self.__class__ # Shorthand for setting class fields. + package_path = None tmp_dir = tempfile.mkdtemp(dir="/tmp") - package_path = "Not set yet." try: # Pack, copy and unpack Python part of VPP installation from _node. # TODO: Use rsync or recursive version of ssh.scp_node instead? @@ -183,24 +188,28 @@ class PapiSocketExecutor(object): exec_cmd_no_error(node, ["bash", "-c", "'" + inner_cmd + "'"]) scp_node(node, tmp_dir + "/papi.txz", "/tmp/papi.txz", get=True) run(["tar", "xf", tmp_dir + "/papi.txz", "-C", tmp_dir]) + api_json_directory = tmp_dir + "/usr/share/vpp/api" + # Perform initial checks before .api.json files are gone, + # by creating the checker instance. + cls.crc_checker = VppApiCrcChecker(api_json_directory) # When present locally, we finally can find the installation path. package_path = glob.glob(tmp_dir + installed_papi_glob)[0] # Package path has to be one level above the vpp_papi directory. package_path = package_path.rsplit('/', 1)[0] sys.path.append(package_path) + # pylint: disable=import-error from vpp_papi.vpp_papi import VPPApiClient as vpp_class - vpp_class.apidir = tmp_dir + "/usr/share/vpp/api" + vpp_class.apidir = api_json_directory # We need to create instance before removing from sys.path. - cls.cached_vpp_instance = vpp_class( + cls.vpp_instance = vpp_class( use_socket=True, server_address="TBD", async_thread=False, - read_timeout=6, logger=FilteredLogger(logger, "INFO")) + read_timeout=14, logger=FilteredLogger(logger, "INFO")) # Cannot use loglevel parameter, robot.api.logger lacks support. # TODO: Stop overriding read_timeout when VPP-1722 is fixed. finally: shutil.rmtree(tmp_dir) if sys.path[-1] == package_path: sys.path.pop() - return cls.cached_vpp_instance def __enter__(self): """Create a tunnel, connect VPP instance. @@ -287,7 +296,7 @@ class PapiSocketExecutor(object): for _ in xrange(2): try: vpp_instance.connect_sync("csit_socket") - except IOError as err: + except (IOError, struct.error) as err: logger.warn("Got initial connect error {err!r}".format(err=err)) vpp_instance.disconnect() else: @@ -306,14 +315,23 @@ class PapiSocketExecutor(object): run(["ssh", "-S", self._ssh_control_socket, "-O", "exit", "0.0.0.0"], check=False) shutil.rmtree(self._temp_dir) - return def add(self, csit_papi_command, history=True, **kwargs): """Add next command to internal command list; return self. + Unless disabled, new entry to papi history is also added at this point. The argument name 'csit_papi_command' must be unique enough as it cannot be repeated in kwargs. - Unless disabled, new entry to papi history is also added at this point. + The kwargs dict is deep-copied, so it is safe to use the original + with partial modifications for subsequent commands. + + Any pending conflicts from .api.json processing are raised. + Then the command name is checked for known CRCs. + Unsupported commands raise an exception, as CSIT change + should not start using messages without making sure which CRCs + are supported. + Each CRC issue is raised only once, so subsequent tests + can raise other issues. :param csit_papi_command: VPP API command. :param history: Enable/disable adding command to PAPI command history. @@ -323,12 +341,15 @@ class PapiSocketExecutor(object): :type kwargs: dict :returns: self, so that method chaining is possible. :rtype: PapiSocketExecutor + :raises RuntimeError: If unverified or conflicting CRC is encountered. """ + self.crc_checker.report_initial_conflicts() if history: PapiHistory.add_to_papi_history( self._node, csit_papi_command, **kwargs) + self.crc_checker.check_api_name(csit_papi_command) self._api_command_list.append( - dict(api_name=csit_papi_command, api_args=kwargs)) + dict(api_name=csit_papi_command, api_args=copy.deepcopy(kwargs))) return self def get_replies(self, err_msg="Failed to get replies."): @@ -380,7 +401,7 @@ class PapiSocketExecutor(object): :raises AssertionError: If retval is nonzero, parsing or ssh error. """ reply = self.get_reply(err_msg=err_msg) - logger.info("Getting index from {reply!r}".format(reply=reply)) + logger.trace("Getting index from {reply!r}".format(reply=reply)) return reply["sw_if_index"] def get_details(self, err_msg="Failed to get dump details."): @@ -401,30 +422,53 @@ class PapiSocketExecutor(object): return self._execute(err_msg) @staticmethod - def run_cli_cmd(node, cmd, log=True): + def run_cli_cmd(node, cli_cmd, log=True, + remote_vpp_socket=Constants.SOCKSVR_PATH): """Run a CLI command as cli_inband, return the "reply" field of reply. Optionally, log the field value. :param node: Node to run command on. - :param cmd: The CLI command to be run on the node. + :param cli_cmd: The CLI command to be run on the node. + :param remote_vpp_socket: Path to remote socket to tunnel to. :param log: If True, the response is logged. :type node: dict - :type cmd: str + :type remote_vpp_socket: str + :type cli_cmd: str :type log: bool :returns: CLI output. :rtype: str """ - cli = 'cli_inband' - args = dict(cmd=cmd) + cmd = 'cli_inband' + args = dict(cmd=cli_cmd) err_msg = "Failed to run 'cli_inband {cmd}' PAPI command on host " \ - "{host}".format(host=node['host'], cmd=cmd) - with PapiSocketExecutor(node) as papi_exec: - reply = papi_exec.add(cli, **args).get_reply(err_msg)["reply"] + "{host}".format(host=node['host'], cmd=cli_cmd) + with PapiSocketExecutor(node, remote_vpp_socket) as papi_exec: + reply = papi_exec.add(cmd, **args).get_reply(err_msg)["reply"] if log: - logger.info("{cmd}:\n{reply}".format(cmd=cmd, reply=reply)) + logger.info( + "{cmd} ({host} - {remote_vpp_socket}):\n{reply}". + format(cmd=cmd, reply=reply, + remote_vpp_socket=remote_vpp_socket, host=node['host'])) return reply + @staticmethod + def run_cli_cmd_on_all_sockets(node, cli_cmd, log=True): + """Run a CLI command as cli_inband, on all sockets in topology file. + + :param node: Node to run command on. + :param cli_cmd: The CLI command to be run on the node. + :param log: If True, the response is logged. + :type node: dict + :type cli_cmd: str + :type log: bool + """ + sockets = Topology.get_node_sockets(node, socket_type=SocketType.PAPI) + if sockets: + for socket in sockets.values(): + PapiSocketExecutor.run_cli_cmd( + node, cli_cmd, log=log, remote_vpp_socket=socket) + @staticmethod def dump_and_log(node, cmds): """Dump and log requested information, return None. @@ -468,20 +512,25 @@ class PapiSocketExecutor(object): api_name = command["api_name"] papi_fn = getattr(vpp_instance.api, api_name) try: - reply = papi_fn(**command["api_args"]) - except IOError as err: - # Ocassionally an error happens, try reconnect. - logger.warn("Reconnect after error: {err!r}".format(err=err)) - self.vpp_instance.disconnect() - # Testing showes immediate reconnect fails. - time.sleep(1) - self.vpp_instance.connect_sync("csit_socket") - logger.trace("Reconnected.") - reply = papi_fn(**command["api_args"]) + try: + reply = papi_fn(**command["api_args"]) + except (IOError, struct.error) as err: + # Ocassionally an error happens, try reconnect. + logger.warn("Reconnect after error: {err!r}".format( + err=err)) + self.vpp_instance.disconnect() + # Testing showes immediate reconnect fails. + time.sleep(1) + self.vpp_instance.connect_sync("csit_socket") + logger.trace("Reconnected.") + reply = papi_fn(**command["api_args"]) + except (AttributeError, IOError, struct.error) as err: + raise_from(AssertionError(err_msg), err, level="INFO") # *_dump commands return list of objects, convert, ordinary reply. if not isinstance(reply, list): reply = [reply] for item in reply: + self.crc_checker.check_api_name(item.__class__.__name__) dict_item = dictize(item) if "retval" in dict_item.keys(): # *_details messages do not contain retval. @@ -566,6 +615,8 @@ class PapiExecutor(object): The argument name 'csit_papi_command' must be unique enough as it cannot be repeated in kwargs. + The kwargs dict is deep-copied, so it is safe to use the original + with partial modifications for subsequent commands. :param csit_papi_command: VPP API command. :param history: Enable/disable adding command to PAPI command history. @@ -579,11 +630,12 @@ class PapiExecutor(object): if history: PapiHistory.add_to_papi_history( self._node, csit_papi_command, **kwargs) - self._api_command_list.append(dict(api_name=csit_papi_command, - api_args=kwargs)) + self._api_command_list.append(dict( + api_name=csit_papi_command, api_args=copy.deepcopy(kwargs))) return self - def get_stats(self, err_msg="Failed to get statistics.", timeout=120): + def get_stats(self, err_msg="Failed to get statistics.", timeout=120, + socket=Constants.SOCKSTAT_PATH): """Get VPP Stats from VPP Python API. :param err_msg: The message used if the PAPI command(s) execution fails. @@ -593,12 +645,12 @@ class PapiExecutor(object): :returns: Requested VPP statistics. :rtype: list of dict """ - paths = [cmd['api_args']['path'] for cmd in self._api_command_list] self._api_command_list = list() stdout = self._execute_papi( - paths, method='stats', err_msg=err_msg, timeout=timeout) + paths, method='stats', err_msg=err_msg, timeout=timeout, + socket=socket) return json.loads(stdout) @@ -643,7 +695,7 @@ class PapiExecutor(object): return api_data_processed def _execute_papi(self, api_data, method='request', err_msg="", - timeout=120): + timeout=120, socket=None): """Execute PAPI command(s) on remote node and store the result. :param api_data: List of APIs with their arguments. @@ -661,7 +713,6 @@ class PapiExecutor(object): :raises RuntimeError: If PAPI executor failed due to another reason. :raises AssertionError: If PAPI command(s) execution has failed. """ - if not api_data: raise RuntimeError("No API data provided.") @@ -669,10 +720,12 @@ class PapiExecutor(object): if method in ("stats", "stats_request") \ else json.dumps(self._process_api_data(api_data)) - cmd = "{fw_dir}/{papi_provider} --method {method} --data '{json}'".\ - format( - fw_dir=Constants.REMOTE_FW_DIR, method=method, json=json_data, - papi_provider=Constants.RESOURCES_PAPI_PROVIDER) + sock = " --socket {socket}".format(socket=socket) if socket else "" + cmd = ( + "{fw_dir}/{papi_provider} --method {method} --data '{json}'{socket}" + .format(fw_dir=Constants.REMOTE_FW_DIR, + papi_provider=Constants.RESOURCES_PAPI_PROVIDER, + method=method, json=json_data, socket=sock)) try: ret_code, stdout, _ = self._ssh.exec_command_sudo( cmd=cmd, timeout=timeout, log_stdout_err=False)