+ For (sometimes) unknown reasons, VPP client in async mode likes
+ to return None occasionally before time runs out.
+ This function retries in that case.
+
+ Most of the time, early None means VPP crashed (see VPP-2033),
+ but is is better to give VPP more chances to respond without failure.
+
+ TODO: Perhaps CSIT now never triggers VPP-2033,
+ so investigate and remove this layer if even more speed is needed.
+
+ :param vpp_instance: Client instance to read from.
+ :param tries: Maximum number of tries to attempt.
+ :type vpp_instance: vpp_papi.VPPApiClient
+ :type tries: int
+ :returns: Message read or None if nothing got read even with retries.
+ :rtype: Optional[namedtuple]
+ """
+ timeout = vpp_instance.read_timeout
+ for _ in range(tries):
+ time_stop = time.monotonic() + 0.9 * timeout
+ reply = PapiSocketExecutor._read_internal(vpp_instance)
+ if reply is None and time.monotonic() < time_stop:
+ logger.trace("Early None. Retry?")
+ continue
+ return reply
+ logger.trace(f"Got {tries} early Nones, probably a real None.")
+ return None
+
+ @staticmethod
+ def _drain(vpp_instance, err_msg, timeout=30.0):
+ """Keep reading with until None or timeout.
+
+ This is needed to mitigate the risk of a state with unread responses
+ (e.g. after non-zero retval in the middle of get_replies)
+ causing failures in everything subsequent (until disconnect).
+
+ The reads are done without any waiting.
+
+ It is possible some responses have not arrived yet,
+ but that is unlikely as Python is usually slower than VPP.
+
+ :param vpp_instance: Client instance to read from.
+ :param err_msg: Error message to use when overstepping timeout.
+ :param timeout: How long to try before giving up.
+ :type vpp_instance: vpp_papi.VPPApiClient
+ :type err_msg: str
+ :type timeout: float
+ :raises RuntimeError: If read keeps returning nonzero after timeout.
+ """
+ time_stop = time.monotonic() + timeout
+ while time.monotonic() < time_stop:
+ if PapiSocketExecutor._read_internal(vpp_instance, 0.0) is None:
+ return
+ raise RuntimeError(f"{err_msg}\nTimed out while draining.")
+
+ def _execute(self, err_msg, do_async, single_reply=True):
+ """Turn internal command list into data and execute; return replies.
+
+ This method also clears the internal command list.
+
+ :param err_msg: The message used if the PAPI command(s) execution fails.
+ :param do_async: If true, assume one reply per command and do not wait
+ for each reply before sending next request.
+ Dump commands (and calls causing VPP-2033) need False.
+ :param single_reply: For sync emulation mode (cannot be False
+ if do_async is True). When false use control ping.
+ When true, wait for a single reply.
+ :type err_msg: str
+ :type do_async: bool
+ :type single_reply: bool
+ :returns: Papi replies parsed into a dict-like object,
+ with fields due to API (possibly including retval).
+ :rtype: NoneType or list of dict
+ :raises RuntimeError: If the replies are not all correct.
+ """
+ local_list = self._api_command_list
+ # Clear first as execution may fail.
+ self._api_command_list = list()
+ if do_async:
+ if not single_reply:
+ raise RuntimeError("Async papi needs one reply per request.")
+ return self._execute_async(local_list, err_msg=err_msg)
+ return self._execute_sync(
+ local_list, err_msg=err_msg, single_reply=single_reply
+ )
+
+ def _execute_sync(self, local_list, err_msg, single_reply):
+ """Execute commands waiting for replies one by one; return replies.
+
+ This implementation either expects a single response per request,
+ or uses control ping to emulate sync PAPI calls.
+ Reliable, but slow. Required for dumps. Needed for calls
+ which trigger VPP-2033.
+
+ CRC checking is done for the replies (requests are checked in .add).
+
+ :param local_list: The list of PAPI commands to be executed on the node.
+ :param err_msg: The message used if the PAPI command(s) execution fails.
+ :param single_reply: When false use control ping.
+ When true, wait for a single reply.
+ :type local_list: list of dict
+ :type err_msg: str
+ :type single_reply: bool
+ :returns: Papi replies parsed into a dict-like object,
+ with fields due to API (possibly including retval).
+ :rtype: List[UserDict]
+ :raises RuntimeError: If the replies are not all correct.
+ """
+ vpp_instance = self.get_connected_client()
+ control_ping_fn = getattr(vpp_instance.api, "control_ping")
+ ret_list = list()
+ for command in local_list:
+ api_name = command["api_name"]
+ papi_fn = getattr(vpp_instance.api, api_name)
+ replies = list()
+ try:
+ # Send the command maybe followed by control ping.
+ main_context = papi_fn(**command["api_args"])
+ if single_reply:
+ replies.append(PapiSocketExecutor._read(vpp_instance))
+ else:
+ ping_context = control_ping_fn()
+ # Receive the replies.
+ while 1:
+ reply = PapiSocketExecutor._read(vpp_instance)
+ if reply is None:
+ raise RuntimeError(
+ f"{err_msg}\nSync PAPI timed out."
+ )
+ if reply.context == ping_context:
+ break
+ if reply.context != main_context:
+ raise RuntimeError(
+ f"{err_msg}\nUnexpected context: {reply!r}"
+ )
+ replies.append(reply)
+ except (AttributeError, IOError, struct.error) as err:
+ # TODO: Add retry if it is still needed.
+ raise AssertionError(f"{err_msg}") from err
+ finally:
+ # Discard any unprocessed replies to avoid secondary failures.
+ PapiSocketExecutor._drain(vpp_instance, err_msg)
+ # Process replies for this command.
+ for reply in replies:
+ self.crc_checker.check_api_name(reply.__class__.__name__)
+ dictized_reply = dictize_and_check_retval(reply, err_msg)
+ ret_list.append(dictized_reply)
+ return ret_list
+
+ def _execute_async(self, local_list, err_msg):
+ """Read, process and return replies.
+
+ The messages were already sent by .add() in this mode,
+ local_list is used just so we know how many replies to read.
+
+ Beware: It is not clear what to do when socket read fails
+ in the middle of async processing.
+
+ The implementation assumes each command results in exactly one reply,
+ there is no reordering in either commands nor replies,
+ and context numbers increase one by one (and are matching for replies).
+
+ To speed processing up, reply CRC values are not checked.
+
+ The current implementation does not limit the number of messages
+ in-flight, we rely on VPP PAPI background thread to move replies
+ from socket to queue fast enough.
+
+ :param local_list: The list of PAPI commands to get replies for.
+ :param err_msg: The message used if the PAPI command(s) execution fails.
+ :type local_list: list
+ :type err_msg: str
+ :returns: Papi replies parsed into a dict-like object, with fields
+ according to API (possibly including retval).
+ :rtype: List[UserDict]
+ :raises RuntimeError: If the replies are not all correct.
+ """
+ vpp_instance = self.get_connected_client()
+ ret_list = list()
+ try:
+ for index, _ in enumerate(local_list):
+ # Blocks up to timeout.
+ reply = PapiSocketExecutor._read(vpp_instance)
+ if reply is None:
+ time_msg = f"PAPI async timeout: idx {index}"
+ raise RuntimeError(f"{err_msg}\n{time_msg}")
+ ret_list.append(dictize_and_check_retval(reply, err_msg))
+ finally:
+ # Discard any unprocessed replies to avoid secondary failures.
+ PapiSocketExecutor._drain(vpp_instance, err_msg)
+ return ret_list
+
+
+class Disconnector:
+ """Class for holding a single keyword."""
+
+ @staticmethod
+ def disconnect_all_papi_connections():
+ """Disconnect all connected client instances, tear down the SSH tunnels.
+
+ Also remove the local sockets by deleting the temporary directory.
+ Put disconnected client instances to the reuse list.
+ The added attributes are not cleaned up,
+ as their values will get overwritten on next connect.
+
+ Call this method just before killing/restarting all VPP instances.
+
+ This could be a class method of PapiSocketExecutor.
+ But Robot calls methods on instances, and it would be weird
+ to give node argument for constructor in import.
+ Also, as we have a class of the same name as the module,
+ the keywords defined on module level are not accessible.
+ """
+ cls = PapiSocketExecutor
+ # Iterate over copy of entries so deletions do not mess with iterator.
+ for key in list(cls.conn_cache.keys()):
+ cls.disconnect_by_key(key)
+
+
+class PapiExecutor:
+ """Contains methods for executing VPP Python API commands on DUTs.
+
+ TODO: Remove .add step, make get_stats accept paths directly.
+
+ This class processes only one type of VPP PAPI methods: vpp-stats.
+
+ The recommended ways of use are (examples):
+
+ path = ['^/if', '/err/ip4-input', '/sys/node/ip4-input']