papi: support old style of providing apidir

[vpp.git] / src / vpp-api / python / vpp_papi / vpp_papi.py

diff --git a/src/vpp-api/python/vpp_papi/vpp_papi.py b/src/vpp-api/python/vpp_papi/vpp_papi.py
index 6c17fa8..5a9e0a7 100644 (file)
--- a/src/vpp-api/python/vpp_papi/vpp_papi.py
+++ b/src/vpp-api/python/vpp_papi/vpp_papi.py
@@ -21,6 +21,7 @@ import ipaddress
 import sys
 import multiprocessing as mp
 import os
 import sys
 import multiprocessing as mp
 import os

+import queue

 import logging
 import functools
 import json
 import logging
 import functools
 import json


@@ -29,19 +30,37 @@ import fnmatch
 import weakref
 import atexit
 import time
 import weakref
 import atexit
 import time

-from . vpp_format import verify_enum_hint
-from . vpp_serializer import VPPType, VPPEnumType, VPPUnionType
-from . vpp_serializer import VPPMessage, vpp_get_type, VPPTypeAlias
+from .vpp_format import verify_enum_hint
+from .vpp_serializer import VPPType, VPPEnumType, VPPEnumFlagType, VPPUnionType
+from .vpp_serializer import VPPMessage, vpp_get_type, VPPTypeAlias

 
 

-if sys.version[0] == '2':
-    import Queue as queue
-else:
-    import queue as queue
+try:
+    import VppTransport
+except ModuleNotFoundError:

 
 

-__all__ = ('FuncWrapper', 'VPP', 'VppApiDynamicMethodHolder',
-           'VppEnum', 'VppEnumType',
-           'VPPIOError', 'VPPRuntimeError', 'VPPValueError',
-           'VPPApiClient', )
+    class V:
+        """placeholder for VppTransport as the implementation is dependent on
+        VPPAPIClient's initialization values
+        """
+
+    VppTransport = V
+
+from .vpp_transport_socket import VppTransport
+
+logger = logging.getLogger("vpp_papi")
+logger.addHandler(logging.NullHandler())
+
+__all__ = (
+    "FuncWrapper",
+    "VppApiDynamicMethodHolder",
+    "VppEnum",
+    "VppEnumType",
+    "VppEnumFlag",
+    "VPPIOError",
+    "VPPRuntimeError",
+    "VPPValueError",
+    "VPPApiClient",
+)

 
 
 def metaclass(metaclass):
 
 
 def metaclass(metaclass):


@@ -59,7 +78,12 @@ class VppEnumType(type):
 
 
 @metaclass(VppEnumType)
 
 
 @metaclass(VppEnumType)

-class VppEnum(object):
+class VppEnum:
+    pass
+
+
+@metaclass(VppEnumType)
+class VppEnumFlag:

     pass
 
 
     pass
 
 


@@ -67,18 +91,10 @@ def vpp_atexit(vpp_weakref):
     """Clean up VPP connection on shutdown."""
     vpp_instance = vpp_weakref()
     if vpp_instance and vpp_instance.transport.connected:
     """Clean up VPP connection on shutdown."""
     vpp_instance = vpp_weakref()
     if vpp_instance and vpp_instance.transport.connected:

-        vpp_instance.logger.debug('Cleaning up VPP on exit')
+        logger.debug("Cleaning up VPP on exit")

         vpp_instance.disconnect()
 
 
         vpp_instance.disconnect()
 
 

-if sys.version[0] == '2':
-    def vpp_iterator(d):
-        return d.iteritems()
-else:
-    def vpp_iterator(d):
-        return d.items()
-
-

 def add_convenience_methods():
     # provide convenience methods to IP[46]Address.vapi_af
     def _vapi_af(self):
 def add_convenience_methods():
     # provide convenience methods to IP[46]Address.vapi_af
     def _vapi_af(self):


@@ -90,20 +106,20 @@ def add_convenience_methods():
 
     def _vapi_af_name(self):
         if 6 == self._version:
 
     def _vapi_af_name(self):
         if 6 == self._version:

-            return 'ip6'
+            return "ip6"

         if 4 == self._version:
         if 4 == self._version:

-            return 'ip4'
+            return "ip4"

         raise ValueError("Invalid _version.")
 
     ipaddress._IPAddressBase.vapi_af = property(_vapi_af)
     ipaddress._IPAddressBase.vapi_af_name = property(_vapi_af_name)
 
 
         raise ValueError("Invalid _version.")
 
     ipaddress._IPAddressBase.vapi_af = property(_vapi_af)
     ipaddress._IPAddressBase.vapi_af_name = property(_vapi_af_name)
 
 

-class VppApiDynamicMethodHolder(object):
+class VppApiDynamicMethodHolder:

     pass
 
 
     pass
 
 

-class FuncWrapper(object):
+class FuncWrapper:

     def __init__(self, func):
         self._func = func
         self.__name__ = func.__name__
     def __init__(self, func):
         self._func = func
         self.__name__ = func.__name__


@@ -113,7 +129,7 @@ class FuncWrapper(object):
         return self._func(**kwargs)
 
     def __repr__(self):
         return self._func(**kwargs)
 
     def __repr__(self):

-        return '<FuncWrapper(func=<%s(%s)>)>' % (self.__name__, self.__doc__)
+        return "<FuncWrapper(func=<%s(%s)>)>" % (self.__name__, self.__doc__)

 
 
 class VPPApiError(Exception):
 
 
 class VPPApiError(Exception):


@@ -136,9 +152,9 @@ class VPPValueError(ValueError):
     pass
 
 
     pass
 
 

-class VPPApiJSONFiles(object):
+class VPPApiJSONFiles:

     @classmethod
     @classmethod

-    def find_api_dir(cls, dirs):
+    def find_api_dir(cls, dirs=[]):

         """Attempt to find the best directory in which API definition
         files may reside. If the value VPP_API_DIR exists in the environment
         then it is first on the search list. If we're inside a recognized
         """Attempt to find the best directory in which API definition
         files may reside. If the value VPP_API_DIR exists in the environment
         then it is first on the search list. If we're inside a recognized


@@ -153,7 +169,11 @@ class VPPApiJSONFiles(object):
         # perhaps we're in the 'src/scripts' or 'src/vpp-api/python' dir;
         # in which case, plot a course to likely places in the src tree
         import __main__ as main
         # perhaps we're in the 'src/scripts' or 'src/vpp-api/python' dir;
         # in which case, plot a course to likely places in the src tree
         import __main__ as main

-        if hasattr(main, '__file__'):
+
+        if os.getenv("VPP_API_DIR"):
+            dirs.append(os.getenv("VPP_API_DIR"))
+
+        if hasattr(main, "__file__"):

             # get the path of the calling script
             localdir = os.path.dirname(os.path.realpath(main.__file__))
         else:
             # get the path of the calling script
             localdir = os.path.dirname(os.path.realpath(main.__file__))
         else:


@@ -163,7 +183,7 @@ class VPPApiJSONFiles(object):
 
         def dmatch(dir):
             """Match dir against right-hand components of the script dir"""
 
         def dmatch(dir):
             """Match dir against right-hand components of the script dir"""

-            d = dir.split('/')  # param 'dir' assumes a / separator
+            d = dir.split("/")  # param 'dir' assumes a / separator

             length = len(d)
             return len(localdir_s) > length and localdir_s[-length:] == d
 
             length = len(d)
             return len(localdir_s) > length and localdir_s[-length:] == d
 


@@ -172,43 +192,45 @@ class VPPApiJSONFiles(object):
             'variant'  (typically '' or '_debug')"""
             # Since 'core' and 'plugin' files are staged
             # in separate directories, we target the parent dir.
             'variant'  (typically '' or '_debug')"""
             # Since 'core' and 'plugin' files are staged
             # in separate directories, we target the parent dir.

-            return os.path.sep.join((
-                srcdir,
-                'build-root',
-                'install-vpp%s-native' % variant,
-                'vpp',
-                'share',
-                'vpp',
-                'api',
-            ))
+            return os.path.sep.join(
+                (
+                    srcdir,
+                    "build-root",
+                    "install-vpp%s-native" % variant,
+                    "vpp",
+                    "share",
+                    "vpp",
+                    "api",
+                )
+            )

 
         srcdir = None
 
         srcdir = None

-        if dmatch('src/scripts'):
+        if dmatch("src/scripts"):

             srcdir = os.path.sep.join(localdir_s[:-2])
             srcdir = os.path.sep.join(localdir_s[:-2])

-        elif dmatch('src/vpp-api/python'):
+        elif dmatch("src/vpp-api/python"):

             srcdir = os.path.sep.join(localdir_s[:-3])
             srcdir = os.path.sep.join(localdir_s[:-3])

-        elif dmatch('test'):
+        elif dmatch("test"):

             # we're apparently running tests
             srcdir = os.path.sep.join(localdir_s[:-1])
 
         if srcdir:
             # we're in the source tree, try both the debug and release
             # variants.
             # we're apparently running tests
             srcdir = os.path.sep.join(localdir_s[:-1])
 
         if srcdir:
             # we're in the source tree, try both the debug and release
             # variants.

-            dirs.append(sdir(srcdir, '_debug'))
-            dirs.append(sdir(srcdir, ''))
+            dirs.append(sdir(srcdir, "_debug"))
+            dirs.append(sdir(srcdir, ""))

 
         # Test for staged copies of the scripts
         # For these, since we explicitly know if we're running a debug versus
         # release variant, target only the relevant directory
 
         # Test for staged copies of the scripts
         # For these, since we explicitly know if we're running a debug versus
         # release variant, target only the relevant directory

-        if dmatch('build-root/install-vpp_debug-native/vpp/bin'):
+        if dmatch("build-root/install-vpp_debug-native/vpp/bin"):

             srcdir = os.path.sep.join(localdir_s[:-4])
             srcdir = os.path.sep.join(localdir_s[:-4])

-            dirs.append(sdir(srcdir, '_debug'))
-        if dmatch('build-root/install-vpp-native/vpp/bin'):
+            dirs.append(sdir(srcdir, "_debug"))
+        if dmatch("build-root/install-vpp-native/vpp/bin"):

             srcdir = os.path.sep.join(localdir_s[:-4])
             srcdir = os.path.sep.join(localdir_s[:-4])

-            dirs.append(sdir(srcdir, ''))
+            dirs.append(sdir(srcdir, ""))

 
         # finally, try the location system packages typically install into
 
         # finally, try the location system packages typically install into

-        dirs.append(os.path.sep.join(('', 'usr', 'share', 'vpp', 'api')))
+        dirs.append(os.path.sep.join(("", "usr", "share", "vpp", "api")))

 
         # check the directories for existence; first one wins
         for dir in dirs:
 
         # check the directories for existence; first one wins
         for dir in dirs:


@@ -218,7 +240,7 @@ class VPPApiJSONFiles(object):
         return None
 
     @classmethod
         return None
 
     @classmethod

-    def find_api_files(cls, api_dir=None, patterns='*'):
+    def find_api_files(cls, api_dir=None, patterns="*"):  # -> list

         """Find API definition files from the given directory tree with the
         given pattern. If no directory is given then find_api_dir() is used
         to locate one. If no pattern is given then all definition files found
         """Find API definition files from the given directory tree with the
         given pattern. If no directory is given then find_api_dir() is used
         to locate one. If no pattern is given then all definition files found


@@ -244,9 +266,9 @@ class VPPApiJSONFiles(object):
                 raise VPPApiError("api_dir cannot be located")
 
         if isinstance(patterns, list) or isinstance(patterns, tuple):
                 raise VPPApiError("api_dir cannot be located")
 
         if isinstance(patterns, list) or isinstance(patterns, tuple):

-            patterns = [p.strip() + '.api.json' for p in patterns]
+            patterns = [p.strip() + ".api.json" for p in patterns]

         else:
         else:

-            patterns = [p.strip() + '.api.json' for p in patterns.split(",")]
+            patterns = [p.strip() + ".api.json" for p in patterns.split(",")]

 
         api_files = []
         for root, dirnames, files in os.walk(api_dir):
 
         api_files = []
         for root, dirnames, files in os.walk(api_dir):


@@ -259,45 +281,83 @@ class VPPApiJSONFiles(object):
 
     @classmethod
     def process_json_file(self, apidef_file):
 
     @classmethod
     def process_json_file(self, apidef_file):

-        api = json.load(apidef_file)
+        return self._process_json(apidef_file.read())
+
+    @classmethod
+    def process_json_str(self, json_str):
+        return self._process_json(json_str)
+
+    @staticmethod
+    def _process_json(json_str):  # -> Tuple[Dict, Dict]
+        api = json.loads(json_str)

         types = {}
         services = {}
         messages = {}
         types = {}
         services = {}
         messages = {}

-        for t in api['enums']:
-            t[0] = 'vl_api_' + t[0] + '_t'
-            types[t[0]] = {'type': 'enum', 'data': t}
-        for t in api['unions']:
-            t[0] = 'vl_api_' + t[0] + '_t'
-            types[t[0]] = {'type': 'union', 'data': t}
-        for t in api['types']:
-            t[0] = 'vl_api_' + t[0] + '_t'
-            types[t[0]] = {'type': 'type', 'data': t}
-        for t, v in api['aliases'].items():
-            types['vl_api_' + t + '_t'] = {'type': 'alias', 'data': v}
-        services.update(api['services'])
+        try:
+            for t in api["enums"]:
+                t[0] = "vl_api_" + t[0] + "_t"
+                types[t[0]] = {"type": "enum", "data": t}
+        except KeyError:
+            pass
+        try:
+            for t in api["enumflags"]:
+                t[0] = "vl_api_" + t[0] + "_t"
+                types[t[0]] = {"type": "enum", "data": t}
+        except KeyError:
+            pass
+        try:
+            for t in api["unions"]:
+                t[0] = "vl_api_" + t[0] + "_t"
+                types[t[0]] = {"type": "union", "data": t}
+        except KeyError:
+            pass
+
+        try:
+            for t in api["types"]:
+                t[0] = "vl_api_" + t[0] + "_t"
+                types[t[0]] = {"type": "type", "data": t}
+        except KeyError:
+            pass
+
+        try:
+            for t, v in api["aliases"].items():
+                types["vl_api_" + t + "_t"] = {"type": "alias", "data": v}
+        except KeyError:
+            pass
+
+        try:
+            services.update(api["services"])
+        except KeyError:
+            pass

 
         i = 0
         while True:
             unresolved = {}
             for k, v in types.items():
 
         i = 0
         while True:
             unresolved = {}
             for k, v in types.items():

-                t = v['data']
+                t = v["data"]

                 if not vpp_get_type(k):
                 if not vpp_get_type(k):

-                    if v['type'] == 'enum':
+                    if v["type"] == "enum":

                         try:
                             VPPEnumType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v
                         try:
                             VPPEnumType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v

-                    elif v['type'] == 'union':
+                if not vpp_get_type(k):
+                    if v["type"] == "enumflag":
+                        try:
+                            VPPEnumFlagType(t[0], t[1:])
+                        except ValueError:
+                            unresolved[k] = v
+                    elif v["type"] == "union":

                         try:
                             VPPUnionType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v
                         try:
                             VPPUnionType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v

-                    elif v['type'] == 'type':
+                    elif v["type"] == "type":

                         try:
                             VPPType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v
                         try:
                             VPPType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v

-                    elif v['type'] == 'alias':
+                    elif v["type"] == "alias":

                         try:
                             VPPTypeAlias(k, t)
                         except ValueError:
                         try:
                             VPPTypeAlias(k, t)
                         except ValueError:


@@ -305,21 +365,46 @@ class VPPApiJSONFiles(object):
             if len(unresolved) == 0:
                 break
             if i > 3:
             if len(unresolved) == 0:
                 break
             if i > 3:

-                raise VPPValueError('Unresolved type definitions {}'
-                                    .format(unresolved))
+                raise VPPValueError("Unresolved type definitions {}".format(unresolved))

             types = unresolved
             i += 1
             types = unresolved
             i += 1

+        try:
+            for m in api["messages"]:
+                try:
+                    messages[m[0]] = VPPMessage(m[0], m[1:])
+                except VPPNotImplementedError:
+                    ### OLE FIXME
+                    logger.error("Not implemented error for {}".format(m[0]))
+        except KeyError:
+            pass
+        return messages, services

 
 

-        for m in api['messages']:
+    @staticmethod
+    def load_api(apifiles=None, apidir=None):
+        messages = {}
+        services = {}
+        if not apifiles:
+            # Pick up API definitions from default directory

             try:
             try:

-                messages[m[0]] = VPPMessage(m[0], m[1:])
-            except VPPNotImplementedError:
-                ### OLE FIXME
-                self.logger.error('Not implemented error for {}'.format(m[0]))
-        return messages, services
+                if isinstance(apidir, list):
+                    apifiles = []
+                    for d in apidir:
+                        apifiles += VPPApiJSONFiles.find_api_files(d)
+                else:
+                    apifiles = VPPApiJSONFiles.find_api_files(apidir)
+            except (RuntimeError, VPPApiError):
+                raise VPPRuntimeError
+
+        for file in apifiles:
+            with open(file) as apidef_file:
+                m, s = VPPApiJSONFiles.process_json_file(apidef_file)
+                messages.update(m)
+                services.update(s)

 
 

+        return apifiles, messages, services

 
 

-class VPPApiClient(object):
+
+class VPPApiClient:

     """VPP interface.
 
     This class provides the APIs to VPP.  The APIs are loaded
     """VPP interface.
 
     This class provides the APIs to VPP.  The APIs are loaded


@@ -331,18 +416,26 @@ class VPPApiClient(object):
     provides a means to register a callback function to receive
     these messages in a background thread.
     """
     provides a means to register a callback function to receive
     these messages in a background thread.
     """

-    apidir = None
+

     VPPApiError = VPPApiError
     VPPRuntimeError = VPPRuntimeError
     VPPValueError = VPPValueError
     VPPNotImplementedError = VPPNotImplementedError
     VPPIOError = VPPIOError
 
     VPPApiError = VPPApiError
     VPPRuntimeError = VPPRuntimeError
     VPPValueError = VPPValueError
     VPPNotImplementedError = VPPNotImplementedError
     VPPIOError = VPPIOError
 

-
-    def __init__(self, apifiles=None, testmode=False, async_thread=True,
-                 logger=None, loglevel=None,
-                 read_timeout=5, use_socket=False,
-                 server_address='/run/vpp/api.sock'):
+    def __init__(
+        self,
+        *,
+        apifiles=None,
+        apidir=None,
+        testmode=False,
+        async_thread=True,
+        logger=None,
+        loglevel=None,
+        read_timeout=5,
+        use_socket=True,
+        server_address="/run/vpp/api.sock",
+    ):

         """Create a VPP API object.
 
         apifiles is a list of files containing API
         """Create a VPP API object.
 
         apifiles is a list of files containing API


@@ -357,7 +450,8 @@ class VPPApiClient(object):
         """
         if logger is None:
             logger = logging.getLogger(
         """
         if logger is None:
             logger = logging.getLogger(

-                "{}.{}".format(__name__, self.__class__.__name__))
+                "{}.{}".format(__name__, self.__class__.__name__)
+            )

             if loglevel is not None:
                 logger.setLevel(loglevel)
         self.logger = logger
             if loglevel is not None:
                 logger.setLevel(loglevel)
         self.logger = logger


@@ -366,53 +460,41 @@ class VPPApiClient(object):
         self.services = {}
         self.id_names = []
         self.id_msgdef = []
         self.services = {}
         self.id_names = []
         self.id_msgdef = []

-        self.header = VPPType('header', [['u16', 'msgid'],
-                                         ['u32', 'client_index']])
+        self.header = VPPType("header", [["u16", "msgid"], ["u32", "client_index"]])

         self.apifiles = []
         self.apifiles = []

+        self.apidir = apidir

         self.event_callback = None
         self.message_queue = queue.Queue()
         self.read_timeout = read_timeout
         self.async_thread = async_thread
         self.event_thread = None
         self.testmode = testmode
         self.event_callback = None
         self.message_queue = queue.Queue()
         self.read_timeout = read_timeout
         self.async_thread = async_thread
         self.event_thread = None
         self.testmode = testmode

-        self.use_socket = use_socket

         self.server_address = server_address
         self._apifiles = apifiles
         self.stats = {}
 
         self.server_address = server_address
         self._apifiles = apifiles
         self.stats = {}
 

-        if use_socket:
-            from . vpp_transport_socket import VppTransport
-        else:
-            from . vpp_transport_shmem import VppTransport
-
-        if not apifiles:
-            # Pick up API definitions from default directory
-            try:
-                apifiles = VPPApiJSONFiles.find_api_files(self.apidir)
-            except RuntimeError:
-                # In test mode we don't care that we can't find the API files
-                if testmode:
-                    apifiles = []
-                else:
-                    raise VPPRuntimeError
-
-        for file in apifiles:
-            with open(file) as apidef_file:
-                m, s = VPPApiJSONFiles.process_json_file(apidef_file)
-                self.messages.update(m)
-                self.services.update(s)
-
-        self.apifiles = apifiles
+        if self.apidir is None and hasattr(self.__class__, "apidir"):
+            # Keep supporting the old style of providing apidir.
+            self.apidir = self.__class__.apidir
+        try:
+            self.apifiles, self.messages, self.services = VPPApiJSONFiles.load_api(
+                apifiles, self.apidir
+            )
+        except VPPRuntimeError as e:
+            if testmode:
+                self.apifiles = []
+            else:
+                raise e

 
         # Basic sanity check
         if len(self.messages) == 0 and not testmode:
 
         # Basic sanity check
         if len(self.messages) == 0 and not testmode:

-            raise VPPValueError(1, 'Missing JSON message definitions')
-        if not(verify_enum_hint(VppEnum.vl_api_address_family_t)):
-            raise VPPRuntimeError("Invalid address family hints. "
-                                  "Cannot continue.")
+            raise VPPValueError(1, "Missing JSON message definitions")
+        if not (verify_enum_hint(VppEnum.vl_api_address_family_t)):
+            raise VPPRuntimeError("Invalid address family hints. " "Cannot continue.")

 
 

-        self.transport = VppTransport(self, read_timeout=read_timeout,
-                                      server_address=server_address)
+        self.transport = VppTransport(
+            self, read_timeout=read_timeout, server_address=server_address
+        )

         # Make sure we allow VPP to clean up the message rings.
         atexit.register(vpp_atexit, weakref.ref(self))
 
         # Make sure we allow VPP to clean up the message rings.
         atexit.register(vpp_atexit, weakref.ref(self))
 


@@ -421,8 +503,9 @@ class VPPApiClient(object):
     def get_function(self, name):
         return getattr(self._api, name)
 
     def get_function(self, name):
         return getattr(self._api, name)
 

-    class ContextId(object):
+    class ContextId:

         """Multiprocessing-safe provider of unique context IDs."""
         """Multiprocessing-safe provider of unique context IDs."""

+

         def __init__(self):
             self.context = mp.Value(ctypes.c_uint, 0)
             self.lock = mp.Lock()
         def __init__(self):
             self.context = mp.Value(ctypes.c_uint, 0)
             self.lock = mp.Lock()


@@ -432,6 +515,7 @@ class VPPApiClient(object):
             with self.lock:
                 self.context.value += 1
                 return self.context.value
             with self.lock:
                 self.context.value += 1
                 return self.context.value

+

     get_context = ContextId()
 
     def get_type(self, name):
     get_context = ContextId()
 
     def get_type(self, name):


@@ -444,27 +528,37 @@ class VPPApiClient(object):
         return self._api
 
     def make_function(self, msg, i, multipart, do_async):
         return self._api
 
     def make_function(self, msg, i, multipart, do_async):

-        if (do_async):
+        if do_async:
+

             def f(**kwargs):
                 return self._call_vpp_async(i, msg, **kwargs)
             def f(**kwargs):
                 return self._call_vpp_async(i, msg, **kwargs)

+

         else:
         else:

+

             def f(**kwargs):
                 return self._call_vpp(i, msg, multipart, **kwargs)
 
         f.__name__ = str(msg.name)
             def f(**kwargs):
                 return self._call_vpp(i, msg, multipart, **kwargs)
 
         f.__name__ = str(msg.name)

-        f.__doc__ = ", ".join(["%s %s" %
-                               (msg.fieldtypes[j], k)
-                               for j, k in enumerate(msg.fields)])
+        f.__doc__ = ", ".join(
+            ["%s %s" % (msg.fieldtypes[j], k) for j, k in enumerate(msg.fields)]
+        )

         f.msg = msg
 
         return f
 
         f.msg = msg
 
         return f
 

+    def make_pack_function(self, msg, i, multipart):
+        def f(**kwargs):
+            return self._call_vpp_pack(i, msg, **kwargs)
+
+        f.msg = msg
+        return f
+

     def _register_functions(self, do_async=False):
         self.id_names = [None] * (self.vpp_dictionary_maxid + 1)
         self.id_msgdef = [None] * (self.vpp_dictionary_maxid + 1)
         self._api = VppApiDynamicMethodHolder()
     def _register_functions(self, do_async=False):
         self.id_names = [None] * (self.vpp_dictionary_maxid + 1)
         self.id_msgdef = [None] * (self.vpp_dictionary_maxid + 1)
         self._api = VppApiDynamicMethodHolder()

-        for name, msg in vpp_iterator(self.messages):
-            n = name + '_' + msg.crc[2:]
+        for name, msg in self.messages.items():
+            n = name + "_" + msg.crc[2:]

             i = self.transport.get_msg_index(n)
             if i > 0:
                 self.id_msgdef[i] = msg
             i = self.transport.get_msg_index(n)
             if i > 0:
                 self.id_msgdef[i] = msg


@@ -472,36 +566,30 @@ class VPPApiClient(object):
 
                 # Create function for client side messages.
                 if name in self.services:
 
                 # Create function for client side messages.
                 if name in self.services:

-                    if 'stream' in self.services[name] and \
-                       self.services[name]['stream']:
-                        multipart = True
-                    else:
-                        multipart = False
-                    f = self.make_function(msg, i, multipart, do_async)
+                    f = self.make_function(msg, i, self.services[name], do_async)
+                    f_pack = self.make_pack_function(msg, i, self.services[name])

                     setattr(self._api, name, FuncWrapper(f))
                     setattr(self._api, name, FuncWrapper(f))

+                    setattr(self._api, name + "_pack", FuncWrapper(f_pack))

             else:
             else:

-                self.logger.debug(
-                    'No such message type or failed CRC checksum: %s', n)
+                self.logger.debug("No such message type or failed CRC checksum: %s", n)

 
 

-    def connect_internal(self, name, msg_handler, chroot_prefix, rx_qlen,
-                         do_async):
-        pfx = chroot_prefix.encode('utf-8') if chroot_prefix else None
+    def connect_internal(self, name, msg_handler, chroot_prefix, rx_qlen, do_async):
+        pfx = chroot_prefix.encode("utf-8") if chroot_prefix else None

 
 

-        rv = self.transport.connect(name, pfx,
-                                    msg_handler, rx_qlen)
+        rv = self.transport.connect(name, pfx, msg_handler, rx_qlen, do_async)

         if rv != 0:
         if rv != 0:

-            raise VPPIOError(2, 'Connect failed')
+            raise VPPIOError(2, "Connect failed")

         self.vpp_dictionary_maxid = self.transport.msg_table_max_index()
         self._register_functions(do_async=do_async)
 
         # Initialise control ping
         self.vpp_dictionary_maxid = self.transport.msg_table_max_index()
         self._register_functions(do_async=do_async)
 
         # Initialise control ping

-        crc = self.messages['control_ping'].crc
+        crc = self.messages["control_ping"].crc

         self.control_ping_index = self.transport.get_msg_index(
         self.control_ping_index = self.transport.get_msg_index(

-            ('control_ping' + '_' + crc[2:]))
-        self.control_ping_msgdef = self.messages['control_ping']
+            ("control_ping" + "_" + crc[2:])
+        )
+        self.control_ping_msgdef = self.messages["control_ping"]

         if self.async_thread:
         if self.async_thread:

-            self.event_thread = threading.Thread(
-                target=self.thread_msg_handler)
+            self.event_thread = threading.Thread(target=self.thread_msg_handler)

             self.event_thread.daemon = True
             self.event_thread.start()
         else:
             self.event_thread.daemon = True
             self.event_thread.start()
         else:


@@ -518,8 +606,9 @@ class VPPApiClient(object):
         client and server.
         """
         msg_handler = self.transport.get_callback(do_async)
         client and server.
         """
         msg_handler = self.transport.get_callback(do_async)

-        return self.connect_internal(name, msg_handler, chroot_prefix, rx_qlen,
-                                     do_async)
+        return self.connect_internal(
+            name, msg_handler, chroot_prefix, rx_qlen, do_async
+        )

 
     def connect_sync(self, name, chroot_prefix=None, rx_qlen=32):
         """Attach to VPP in synchronous mode. Application must poll for events.
 
     def connect_sync(self, name, chroot_prefix=None, rx_qlen=32):
         """Attach to VPP in synchronous mode. Application must poll for events.


@@ -530,8 +619,7 @@ class VPPApiClient(object):
         client and server.
         """
 
         client and server.
         """
 

-        return self.connect_internal(name, None, chroot_prefix, rx_qlen,
-                                     do_async=False)
+        return self.connect_internal(name, None, chroot_prefix, rx_qlen, do_async=False)

 
     def disconnect(self):
         """Detach from VPP."""
 
     def disconnect(self):
         """Detach from VPP."""


@@ -552,42 +640,43 @@ class VPPApiClient(object):
         # If we have a context, then use the context to find any
         # request waiting for a reply
         context = 0
         # If we have a context, then use the context to find any
         # request waiting for a reply
         context = 0

-        if hasattr(r, 'context') and r.context > 0:
+        if hasattr(r, "context") and r.context > 0:

             context = r.context
 
         if context == 0:
             # No context -> async notification that we feed to the callback
             self.message_queue.put_nowait(r)
         else:
             context = r.context
 
         if context == 0:
             # No context -> async notification that we feed to the callback
             self.message_queue.put_nowait(r)
         else:

-            raise VPPIOError(2, 'RPC reply message received in event handler')
+            raise VPPIOError(2, "RPC reply message received in event handler")

 
     def has_context(self, msg):
         if len(msg) < 10:
             return False
 
 
     def has_context(self, msg):
         if len(msg) < 10:
             return False
 

-        header = VPPType('header_with_context', [['u16', 'msgid'],
-                                                 ['u32', 'client_index'],
-                                                 ['u32', 'context']])
+        header = VPPType(
+            "header_with_context",
+            [["u16", "msgid"], ["u32", "client_index"], ["u32", "context"]],
+        )

 
         (i, ci, context), size = header.unpack(msg, 0)
 
         (i, ci, context), size = header.unpack(msg, 0)

-        if self.id_names[i] == 'rx_thread_exit':
+        if self.id_names[i] == "rx_thread_exit":

             return
 
         #
         # Decode message and returns a tuple.
         #
         msgobj = self.id_msgdef[i]
             return
 
         #
         # Decode message and returns a tuple.
         #
         msgobj = self.id_msgdef[i]

-        if 'context' in msgobj.field_by_name and context >= 0:
+        if "context" in msgobj.field_by_name and context >= 0:

             return True
         return False
 
     def decode_incoming_msg(self, msg, no_type_conversion=False):
         if not msg:
             return True
         return False
 
     def decode_incoming_msg(self, msg, no_type_conversion=False):
         if not msg:

-            self.logger.warning('vpp_api.read failed')
+            logger.warning("vpp_api.read failed")

             return
 
         (i, ci), size = self.header.unpack(msg, 0)
             return
 
         (i, ci), size = self.header.unpack(msg, 0)

-        if self.id_names[i] == 'rx_thread_exit':
+        if self.id_names[i] == "rx_thread_exit":

             return
 
         #
             return
 
         #


@@ -595,7 +684,7 @@ class VPPApiClient(object):
         #
         msgobj = self.id_msgdef[i]
         if not msgobj:
         #
         msgobj = self.id_msgdef[i]
         if not msgobj:

-            raise VPPIOError(2, 'Reply message undefined')
+            raise VPPIOError(2, "Reply message undefined")

 
         r, size = msgobj.unpack(msg, ntc=no_type_conversion)
         return r
 
         r, size = msgobj.unpack(msg, ntc=no_type_conversion)
         return r


@@ -616,35 +705,42 @@ class VPPApiClient(object):
 
     def _control_ping(self, context):
         """Send a ping command."""
 
     def _control_ping(self, context):
         """Send a ping command."""

-        self._call_vpp_async(self.control_ping_index,
-                             self.control_ping_msgdef,
-                             context=context)
+        self._call_vpp_async(
+            self.control_ping_index, self.control_ping_msgdef, context=context
+        )

 
     def validate_args(self, msg, kwargs):
         d = set(kwargs.keys()) - set(msg.field_by_name.keys())
         if d:
 
     def validate_args(self, msg, kwargs):
         d = set(kwargs.keys()) - set(msg.field_by_name.keys())
         if d:

-            raise VPPValueError('Invalid argument {} to {}'
-                                .format(list(d), msg.name))
+            raise VPPValueError("Invalid argument {} to {}".format(list(d), msg.name))

 
     def _add_stat(self, name, ms):
         if not name in self.stats:
 
     def _add_stat(self, name, ms):
         if not name in self.stats:

-            self.stats[name] = {'max': ms, 'count': 1, 'avg': ms}
+            self.stats[name] = {"max": ms, "count": 1, "avg": ms}

         else:
         else:

-            if ms > self.stats[name]['max']:
-                self.stats[name]['max'] = ms
-            self.stats[name]['count'] += 1
-            n = self.stats[name]['count']
-            self.stats[name]['avg'] = self.stats[name]['avg'] * (n - 1) / n + ms / n
+            if ms > self.stats[name]["max"]:
+                self.stats[name]["max"] = ms
+            self.stats[name]["count"] += 1
+            n = self.stats[name]["count"]
+            self.stats[name]["avg"] = self.stats[name]["avg"] * (n - 1) / n + ms / n

 
     def get_stats(self):
 
     def get_stats(self):

-        s = '\n=== API PAPI STATISTICS ===\n'
-        s += '{:<30} {:>4} {:>6} {:>6}\n'.format('message', 'cnt', 'avg', 'max')
-        for n in sorted(self.stats.items(), key=lambda v: v[1]['avg'], reverse=True):
-            s += '{:<30} {:>4} {:>6.2f} {:>6.2f}\n'.format(n[0], n[1]['count'],
-                                                           n[1]['avg'], n[1]['max'])
+        s = "\n=== API PAPI STATISTICS ===\n"
+        s += "{:<30} {:>4} {:>6} {:>6}\n".format("message", "cnt", "avg", "max")
+        for n in sorted(self.stats.items(), key=lambda v: v[1]["avg"], reverse=True):
+            s += "{:<30} {:>4} {:>6.2f} {:>6.2f}\n".format(
+                n[0], n[1]["count"], n[1]["avg"], n[1]["max"]
+            )

         return s
 
         return s
 

-    def _call_vpp(self, i, msgdef, multipart, **kwargs):
+    def get_field_options(self, msg, fld_name):
+        # when there is an option, the msgdef has 3 elements.
+        # ['u32', 'ring_size', {'default': 1024}]
+        for _def in self.messages[msg].msgdef:
+            if isinstance(_def, list) and len(_def) == 3 and _def[1] == fld_name:
+                return _def[2]
+
+    def _call_vpp(self, i, msgdef, service, **kwargs):

         """Given a message, send the message and await a reply.
 
         msgdef - the message packing definition
         """Given a message, send the message and await a reply.
 
         msgdef - the message packing definition


@@ -660,25 +756,26 @@ class VPPApiClient(object):
         no response within the timeout window.
         """
         ts = time.time()
         no response within the timeout window.
         """
         ts = time.time()

-        if 'context' not in kwargs:
+        if "context" not in kwargs:

             context = self.get_context()
             context = self.get_context()

-            kwargs['context'] = context
+            kwargs["context"] = context

         else:
         else:

-            context = kwargs['context']
-        kwargs['_vl_msg_id'] = i
+            context = kwargs["context"]
+        kwargs["_vl_msg_id"] = i

 
 

-        no_type_conversion = kwargs.pop('_no_type_conversion', False)
-        timeout = kwargs.pop('_timeout', None)
+        no_type_conversion = kwargs.pop("_no_type_conversion", False)
+        timeout = kwargs.pop("_timeout", None)

 
         try:
             if self.transport.socket_index:
 
         try:
             if self.transport.socket_index:

-                kwargs['client_index'] = self.transport.socket_index
+                kwargs["client_index"] = self.transport.socket_index

         except AttributeError:
             pass
         self.validate_args(msgdef, kwargs)
 
         except AttributeError:
             pass
         self.validate_args(msgdef, kwargs)
 

-        s = 'Calling {}({})'.format(msgdef.name,
-            ','.join(['{!r}:{!r}'.format(k, v) for k, v in kwargs.items()]))
+        s = "Calling {}({})".format(
+            msgdef.name, ",".join(["{!r}:{!r}".format(k, v) for k, v in kwargs.items()])
+        )

         self.logger.debug(s)
 
         b = msgdef.pack(kwargs)
         self.logger.debug(s)
 
         b = msgdef.pack(kwargs)


@@ -686,34 +783,48 @@ class VPPApiClient(object):
 
         self.transport.write(b)
 
 
         self.transport.write(b)
 

-        if multipart:
-            # Send a ping after the request - we use its response
-            # to detect that we have seen all results.
-            self._control_ping(context)
+        msgreply = service["reply"]
+        stream = True if "stream" in service else False
+        if stream:
+            if "stream_msg" in service:
+                # New service['reply'] = _reply and service['stream_message'] = _details
+                stream_message = service["stream_msg"]
+                modern = True
+            else:
+                # Old  service['reply'] = _details
+                stream_message = msgreply
+                msgreply = "control_ping_reply"
+                modern = False
+                # Send a ping after the request - we use its response
+                # to detect that we have seen all results.
+                self._control_ping(context)

 
         # Block until we get a reply.
         rl = []
 
         # Block until we get a reply.
         rl = []

-        while (True):
+        while True:

             r = self.read_blocking(no_type_conversion, timeout)
             if r is None:
             r = self.read_blocking(no_type_conversion, timeout)
             if r is None:

-                raise VPPIOError(2, 'VPP API client: read failed')
+                raise VPPIOError(2, "VPP API client: read failed")

             msgname = type(r).__name__
             if context not in r or r.context == 0 or context != r.context:
                 # Message being queued
                 self.message_queue.put_nowait(r)
                 continue
             msgname = type(r).__name__
             if context not in r or r.context == 0 or context != r.context:
                 # Message being queued
                 self.message_queue.put_nowait(r)
                 continue

-
-            if not multipart:
+            if msgname != msgreply and (stream and (msgname != stream_message)):
+                print("REPLY MISMATCH", msgreply, msgname, stream_message, stream)
+            if not stream:

                 rl = r
                 break
                 rl = r
                 break

-            if msgname == 'control_ping_reply':
+            if msgname == msgreply:
+                if modern:  # Return both reply and list
+                    rl = r, rl

                 break
 
             rl.append(r)
 
         self.transport.resume()
 
                 break
 
             rl.append(r)
 
         self.transport.resume()
 

-        s = 'Return value: {!r}'.format(r)
+        s = "Return value: {!r}".format(r)

         if len(s) > 80:
             s = s[:80] + "..."
         self.logger.debug(s)
         if len(s) > 80:
             s = s[:80] + "..."
         self.logger.debug(s)


@@ -734,22 +845,29 @@ class VPPApiClient(object):
         The returned context will help with assigning which call
         the reply belongs to.
         """
         The returned context will help with assigning which call
         the reply belongs to.
         """

-        if 'context' not in kwargs:
+        if "context" not in kwargs:

             context = self.get_context()
             context = self.get_context()

-            kwargs['context'] = context
+            kwargs["context"] = context

         else:
         else:

-            context = kwargs['context']
+            context = kwargs["context"]

         try:
             if self.transport.socket_index:
         try:
             if self.transport.socket_index:

-                kwargs['client_index'] = self.transport.socket_index
+                kwargs["client_index"] = self.transport.socket_index

         except AttributeError:
         except AttributeError:

-            kwargs['client_index'] = 0
-        kwargs['_vl_msg_id'] = i
+            kwargs["client_index"] = 0
+        kwargs["_vl_msg_id"] = i

         b = msg.pack(kwargs)
 
         self.transport.write(b)
         return context
 
         b = msg.pack(kwargs)
 
         self.transport.write(b)
         return context
 

+    def _call_vpp_pack(self, i, msg, **kwargs):
+        """Given a message, return the binary representation."""
+        kwargs["_vl_msg_id"] = i
+        kwargs["client_index"] = 0
+        kwargs["context"] = 0
+        return msg.pack(kwargs)
+

     def read_blocking(self, no_type_conversion=False, timeout=None):
         """Get next received message from transport within timeout, decoded.
 
     def read_blocking(self, no_type_conversion=False, timeout=None):
         """Get next received message from transport within timeout, decoded.
 


@@ -830,25 +948,37 @@ class VPPApiClient(object):
         """Return VPPs API message table as name_crc dictionary,
         filtered by message name list."""
 
         """Return VPPs API message table as name_crc dictionary,
         filtered by message name list."""
 

-        replies = [self.services[n]['reply'] for n in msglist]
+        replies = [self.services[n]["reply"] for n in msglist]

         message_table_filtered = {}
         for name in msglist + replies:
         message_table_filtered = {}
         for name in msglist + replies:

-            for k,v in self.transport.message_table.items():
+            for k, v in self.transport.message_table.items():

                 if k.startswith(name):
                     message_table_filtered[k] = v
                     break
         return message_table_filtered
 
     def __repr__(self):
                 if k.startswith(name):
                     message_table_filtered[k] = v
                     break
         return message_table_filtered
 
     def __repr__(self):

-        return "<VPPApiClient apifiles=%s, testmode=%s, async_thread=%s, " \
-               "logger=%s, read_timeout=%s, use_socket=%s, " \
-               "server_address='%s'>" % (
-                   self._apifiles, self.testmode, self.async_thread,
-                   self.logger, self.read_timeout, self.use_socket,
-                   self.server_address)
-
-
-# Provide the old name for backward compatibility.
-VPP = VPPApiClient
-
-# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
+        return (
+            "<VPPApiClient apifiles=%s, testmode=%s, async_thread=%s, "
+            "logger=%s, read_timeout=%s, "
+            "server_address='%s'>"
+            % (
+                self._apifiles,
+                self.testmode,
+                self.async_thread,
+                self.logger,
+                self.read_timeout,
+                self.server_address,
+            )
+        )
+
+    def details_iter(self, f, **kwargs):
+        cursor = 0
+        while True:
+            kwargs["cursor"] = cursor
+            rv, details = f(**kwargs)
+            for d in details:
+                yield d
+            if rv.retval == 0 or rv.retval != -165:
+                break
+            cursor = rv.cursor