PAPI: Add verification of the response

Change-Id: I86afcaeae865f0af076b8dd974386a83de07bf44
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/libraries/python/PapiExecutor.py b/resources/libraries/python/PapiExecutor.py
index 1971e62..b0ddccf 100644 (file)
--- a/resources/libraries/python/PapiExecutor.py
+++ b/resources/libraries/python/PapiExecutor.py
@@ -11,7 +11,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

-"""Python API executor library."""
+"""Python API executor library.
+
+This version supports only simple request / reply VPP API methods.
+
+TODO:
+ - Implement:
+   - Dump functions
+   - vpp-stats
+
+"""

 
 import binascii
 import json
 
 import binascii
 import json


@@ -22,6 +31,7 @@ from resources.libraries.python.Constants import Constants
 from resources.libraries.python.ssh import SSH, SSHTimeout
 from resources.libraries.python.PapiHistory import PapiHistory
 
 from resources.libraries.python.ssh import SSH, SSHTimeout
 from resources.libraries.python.PapiHistory import PapiHistory
 

+

 __all__ = ["PapiExecutor", "PapiResponse"]
 
 
 __all__ = ["PapiExecutor", "PapiResponse"]
 
 


@@ -30,45 +40,60 @@ class PapiResponse(object):
     code.
     """
 
     code.
     """
 

-    def __init__(self, papi_reply=None, stdout="", stderr="", ret_code=None):
+    def __init__(self, papi_reply=None, stdout="", stderr="", ret_code=None,
+                 requests=None):

         """Construct the Papi response by setting the values needed.
 
         :param papi_reply: API reply from last executed PAPI command(s).
         :param stdout: stdout from last executed PAPI command(s).
         :param stderr: stderr from last executed PAPI command(s).
         :param ret_code: ret_code from last executed PAPI command(s).
         """Construct the Papi response by setting the values needed.
 
         :param papi_reply: API reply from last executed PAPI command(s).
         :param stdout: stdout from last executed PAPI command(s).
         :param stderr: stderr from last executed PAPI command(s).
         :param ret_code: ret_code from last executed PAPI command(s).

+        :param requests: List of used PAPI requests. It is used while verifying
+            replies. If None, expected replies must be provided for verify_reply
+            and verify_replies methods.

         :type papi_reply: list
         :type stdout: str
         :type stderr: str
         :type ret_code: int
         :type papi_reply: list
         :type stdout: str
         :type stderr: str
         :type ret_code: int

+        :type requests: list

         """
 
         """
 

-        # API reply from last executed PAPI command(s)
+        # API reply from last executed PAPI command(s).

         self.reply = papi_reply
 
         self.reply = papi_reply
 

-        # stdout from last executed PAPI command(s)
+        # stdout from last executed PAPI command(s).

         self.stdout = stdout
 
         # stderr from last executed PAPI command(s).
         self.stderr = stderr
 
         self.stdout = stdout
 
         # stderr from last executed PAPI command(s).
         self.stderr = stderr
 

-        # return code from last executed PAPI command(s)
+        # return code from last executed PAPI command(s).

         self.ret_code = ret_code
 
         self.ret_code = ret_code
 

+        # List of used PAPI requests.
+        self.requests = requests
+
+        # List of expected PAPI replies. It is used while verifying replies.
+        if self.requests:
+            self.expected_replies = \
+                ["{rqst}_reply".format(rqst=rqst) for rqst in self.requests]
+

     def __str__(self):
     def __str__(self):

-        """Return string with human readable description of the group.
+        """Return string with human readable description of the PapiResponse.

 
         :returns: Readable description.
         :rtype: str
         """
 
         :returns: Readable description.
         :rtype: str
         """

-        return ("papi_reply={papi_reply} "
-                "stdout={stdout} "
-                "stderr={stderr} "
-                "ret_code={ret_code}".
+        return ("papi_reply={papi_reply},"
+                "stdout={stdout},"
+                "stderr={stderr},"
+                "ret_code={ret_code},"
+                "requests={requests}".

                 format(papi_reply=self.reply,
                        stdout=self.stdout,
                        stderr=self.stderr,
                 format(papi_reply=self.reply,
                        stdout=self.stdout,
                        stderr=self.stderr,

-                       ret_code=self.ret_code))
+                       ret_code=self.ret_code,
+                       requests=self.requests))

 
     def __repr__(self):
         """Return string executable as Python constructor call.
 
     def __repr__(self):
         """Return string executable as Python constructor call.


@@ -76,14 +101,104 @@ class PapiResponse(object):
         :returns: Executable constructor call.
         :rtype: str
         """
         :returns: Executable constructor call.
         :rtype: str
         """

-        return ("PapiResponse(papi_reply={papi_reply} "
-                "stdout={stdout} "
-                "stderr={stderr} "
-                "ret_code={ret_code})".
-                format(papi_reply=self.reply,
-                       stdout=self.stdout,
-                       stderr=self.stderr,
-                       ret_code=self.ret_code))
+        return "PapiResponse({str})".format(str=str(self))
+
+    def verify_reply(self, cmd_reply=None, idx=0,
+                     err_msg="Failed to verify PAPI reply."):
+        """Verify and return data from the PAPI response.
+
+        Note: Use only with a simple request / reply command. In this case the
+        PAPI reply includes 'retval' which is checked in this method.
+
+        Use if PAPI response includes only one command reply.
+
+        Use it this way (preferred):
+
+        with PapiExecutor(node) as papi_exec:
+            data = papi_exec.add('show_version').execute_should_pass().\
+                verify_reply()
+
+        or if you must provide the expected reply (not recommended):
+
+        with PapiExecutor(node) as papi_exec:
+            data = papi_exec.add('show_version').execute_should_pass().\
+                verify_reply('show_version_reply')
+
+        :param cmd_reply: PAPI reply. If None, list of 'requests' should have
+            been provided to the __init__ method as pre-generated list of
+            replies is used in this method in this case.
+            The .execute* methods are providing the requests automatically.
+        :param idx: Index to PapiResponse.reply list.
+        :param err_msg: The message used if the verification fails.
+        :type cmd_reply: str
+        :type idx: int
+        :type err_msg: str or None
+        :returns: Verified data from PAPI response.
+        :rtype: dict
+        :raises AssertionError: If the PAPI return value is not 0, so the reply
+            is not valid.
+        :raises KeyError, IndexError: If the reply does not have expected
+            structure.
+        """
+        cmd_rpl = self.expected_replies[idx] if cmd_reply is None else cmd_reply
+
+        data = self.reply[idx]['api_reply'][cmd_rpl]
+        if data['retval'] != 0:
+            raise AssertionError("{msg}\nidx={idx}, cmd_reply={reply}".
+                                 format(msg=err_msg, idx=idx, reply=cmd_rpl))
+
+        return data
+
+    def verify_replies(self, cmd_replies=None,
+                       err_msg="Failed to verify PAPI reply."):
+        """Verify and return data from the PAPI response.
+
+        Note: Use only with request / reply commands. In this case each
+        PAPI reply includes 'retval' which is checked.
+
+        Use if PAPI response includes more than one command reply.
+
+        Use it this way:
+
+        with PapiExecutor(node) as papi_exec:
+            papi_exec.add(cmd1, **args1).add(cmd2, **args2).add(cmd2, **args3).\
+                execute_should_pass(err_msg).verify_replies()
+
+        or if you need the data from the PAPI response:
+
+        with PapiExecutor(node) as papi_exec:
+            data = papi_exec.add(cmd1, **args1).add(cmd2, **args2).\
+                add(cmd2, **args3).execute_should_pass(err_msg).verify_replies()
+
+        or if you must provide the list of expected replies (not recommended):
+
+        with PapiExecutor(node) as papi_exec:
+            data = papi_exec.add(cmd1, **args1).add(cmd2, **args2).\
+                add(cmd2, **args3).execute_should_pass(err_msg).\
+                verify_replies(cmd_replies=cmd_replies)
+
+        :param cmd_replies: List of PAPI command replies. If None, list of
+            'requests' should have been provided to the __init__ method as
+            pre-generated list of replies is used in this method in this case.
+            The .execute* methods are providing the requests automatically.
+        :param err_msg: The message used if the verification fails.
+        :type cmd_replies: list of str or None
+        :type err_msg: str
+        :returns: List of verified data from PAPI response.
+        :rtype list
+        :raises AssertionError: If the PAPI response does not include at least
+            one of specified command replies.
+        """
+        data = list()
+
+        cmd_rpls = self.expected_replies if cmd_replies is None else cmd_replies
+
+        if len(self.reply) != len(cmd_rpls):
+            raise AssertionError(err_msg)
+        for idx, cmd_reply in enumerate(cmd_rpls):
+            data.append(self.verify_reply(cmd_reply, idx, err_msg))
+
+        return data

 
 
 class PapiExecutor(object):
 
 
 class PapiExecutor(object):


@@ -108,9 +223,6 @@ class PapiExecutor(object):
         # The list of PAPI commands to be executed on the node.
         self._api_command_list = list()
 
         # The list of PAPI commands to be executed on the node.
         self._api_command_list = list()
 

-        # The response on the PAPI commands.
-        self.response = PapiResponse()
-

         self._ssh = SSH()
 
     def __enter__(self):
         self._ssh = SSH()
 
     def __enter__(self):


@@ -197,7 +309,8 @@ class PapiExecutor(object):
         return PapiResponse(papi_reply=papi_reply,
                             stdout=stdout,
                             stderr=stderr,
         return PapiResponse(papi_reply=papi_reply,
                             stdout=stdout,
                             stderr=stderr,

-                            ret_code=ret_code)
+                            ret_code=ret_code,
+                            requests=[rqst["api_name"] for rqst in local_list])

 
     def execute_should_pass(self, err_msg="Failed to execute PAPI command.",
                             process_reply=True, ignore_errors=False,
 
     def execute_should_pass(self, err_msg="Failed to execute PAPI command.",
                             process_reply=True, ignore_errors=False,


@@ -205,9 +318,6 @@ class PapiExecutor(object):
         """Execute the PAPI commands and check the return code.
         Raise exception if the PAPI command(s) failed.
 
         """Execute the PAPI commands and check the return code.
         Raise exception if the PAPI command(s) failed.
 

-        Note: There are two exceptions raised to distinguish two situations. If
-        not needed, re-implement using only RuntimeError.
-

         :param err_msg: The message used if the PAPI command(s) execution fails.
         :param process_reply: Indicate whether or not to process PAPI reply.
         :param ignore_errors: If true, the errors in the reply are ignored.
         :param err_msg: The message used if the PAPI command(s) execution fails.
         :param process_reply: Indicate whether or not to process PAPI reply.
         :param ignore_errors: If true, the errors in the reply are ignored.


@@ -219,8 +329,7 @@ class PapiExecutor(object):
         :returns: Papi response including: papi reply, stdout, stderr and
             return code.
         :rtype: PapiResponse
         :returns: Papi response including: papi reply, stdout, stderr and
             return code.
         :rtype: PapiResponse

-        :raises RuntimeError: If no PAPI command(s) executed.
-        :raises AssertionError: If PAPI command(s) execution passed.
+        :raises AssertionError: If PAPI command(s) execution failed.

         """
 
         response = self.execute(process_reply=process_reply,
         """
 
         response = self.execute(process_reply=process_reply,


@@ -240,9 +349,6 @@ class PapiExecutor(object):
 
         It does not return anything as we expect it fails.
 
 
         It does not return anything as we expect it fails.
 

-        Note: There are two exceptions raised to distinguish two situations. If
-        not needed, re-implement using only RuntimeError.
-

         :param err_msg: The message used if the PAPI command(s) execution fails.
         :param process_reply: Indicate whether or not to process PAPI reply.
         :param ignore_errors: If true, the errors in the reply are ignored.
         :param err_msg: The message used if the PAPI command(s) execution fails.
         :param process_reply: Indicate whether or not to process PAPI reply.
         :param ignore_errors: If true, the errors in the reply are ignored.


@@ -251,7 +357,6 @@ class PapiExecutor(object):
         :type process_reply: bool
         :type ignore_errors: bool
         :type timeout: int
         :type process_reply: bool
         :type ignore_errors: bool
         :type timeout: int

-        :raises RuntimeError: If no PAPI command(s) executed.

         :raises AssertionError: If PAPI command(s) execution passed.
         """
 
         :raises AssertionError: If PAPI command(s) execution passed.
         """
 

diff --git a/resources/libraries/python/VPPUtil.py b/resources/libraries/python/VPPUtil.py
index 1bc1b43..57a78fc 100644 (file)
--- a/resources/libraries/python/VPPUtil.py
+++ b/resources/libraries/python/VPPUtil.py
@@ -149,15 +149,15 @@ class VPPUtil(object):
         """
 
         with PapiExecutor(node) as papi_exec:
         """
 
         with PapiExecutor(node) as papi_exec:

-            papi_resp = papi_exec.add('show_version').execute_should_pass()
-        data = papi_resp.reply[0]['api_reply']['show_version_reply']
+            data = papi_exec.add('show_version').execute_should_pass().\
+                verify_reply()

         version = ('VPP version:      {ver}\n'.
                    format(ver=data['version'].rstrip('\0x00')))
         if verbose:
             version += ('Compile date:     {date}\n'
         version = ('VPP version:      {ver}\n'.
                    format(ver=data['version'].rstrip('\0x00')))
         if verbose:
             version += ('Compile date:     {date}\n'

-                        'Compile location: {loc}\n '.
+                        'Compile location: {cl}\n '.

                         format(date=data['build_date'].rstrip('\0x00'),
                         format(date=data['build_date'].rstrip('\0x00'),

-                               loc=data['build_directory'].rstrip('\0x00')))
+                               cl=data['build_directory'].rstrip('\0x00')))

         logger.info(version)
 
     @staticmethod
         logger.info(version)
 
     @staticmethod


@@ -303,7 +303,6 @@ class VPPUtil(object):
         :returns: VPP thread data.
         :rtype: list
         """
         :returns: VPP thread data.
         :rtype: list
         """

-

         with PapiExecutor(node) as papi_exec:
         with PapiExecutor(node) as papi_exec:

-            resp = papi_exec.add('show_threads').execute_should_pass()
-        return resp.reply[0]['api_reply']['show_threads_reply']['thread_data']
+            return papi_exec.add('show_threads').execute_should_pass().\
+                verify_reply()["thread_data"]