api: provide api definition over api
[vpp.git] / src / vpp-api / python / vpp_papi / vpp_papi.py
index 6ba7d64..30c00cd 100644 (file)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (c) 2016 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #
 # Copyright (c) 2016 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 from __future__ import print_function
 from __future__ import absolute_import
 import ctypes
 from __future__ import print_function
 from __future__ import absolute_import
 import ctypes
-import sys
+import ipaddress
 import multiprocessing as mp
 import os
 import multiprocessing as mp
 import os
+import queue
 import logging
 import functools
 import json
 import logging
 import functools
 import json
@@ -27,18 +28,39 @@ import threading
 import fnmatch
 import weakref
 import atexit
 import fnmatch
 import weakref
 import atexit
-from . vpp_serializer import VPPType, VPPEnumType, VPPUnionType
-from . vpp_serializer import VPPMessage, vpp_get_type, VPPTypeAlias
+import time
+import pkg_resources
+from .vpp_format import verify_enum_hint
+from .vpp_serializer import VPPType, VPPEnumType, VPPEnumFlagType, VPPUnionType
+from .vpp_serializer import VPPMessage, vpp_get_type, VPPTypeAlias
+
+try:
+    import VppTransport
+except ModuleNotFoundError:
+
+    class V:
+        """placeholder for VppTransport as the implementation is dependent on
+        VPPAPIClient's initialization values
+        """
+
+    VppTransport = V
 
 
-if sys.version[0] == '2':
-    import Queue as queue
-else:
-    import queue as queue
+from .vpp_transport_socket import VppTransport
 
 
-__all__ = ('FuncWrapper', 'VPP', 'VppApiDynamicMethodHolder',
-           'VppEnum', 'VppEnumType',
-           'VPPIOError', 'VPPRuntimeError', 'VPPValueError',
-           'VPPApiClient', )
+logger = logging.getLogger("vpp_papi")
+logger.addHandler(logging.NullHandler())
+
+__all__ = (
+    "FuncWrapper",
+    "VppApiDynamicMethodHolder",
+    "VppEnum",
+    "VppEnumType",
+    "VppEnumFlag",
+    "VPPIOError",
+    "VPPRuntimeError",
+    "VPPValueError",
+    "VPPApiClient",
+)
 
 
 def metaclass(metaclass):
 
 
 def metaclass(metaclass):
@@ -56,7 +78,12 @@ class VppEnumType(type):
 
 
 @metaclass(VppEnumType)
 
 
 @metaclass(VppEnumType)
-class VppEnum(object):
+class VppEnum:
+    pass
+
+
+@metaclass(VppEnumType)
+class VppEnumFlag:
     pass
 
 
     pass
 
 
@@ -64,23 +91,35 @@ def vpp_atexit(vpp_weakref):
     """Clean up VPP connection on shutdown."""
     vpp_instance = vpp_weakref()
     if vpp_instance and vpp_instance.transport.connected:
     """Clean up VPP connection on shutdown."""
     vpp_instance = vpp_weakref()
     if vpp_instance and vpp_instance.transport.connected:
-        vpp_instance.logger.debug('Cleaning up VPP on exit')
+        logger.debug("Cleaning up VPP on exit")
         vpp_instance.disconnect()
 
 
         vpp_instance.disconnect()
 
 
-if sys.version[0] == '2':
-    def vpp_iterator(d):
-        return d.iteritems()
-else:
-    def vpp_iterator(d):
-        return d.items()
+def add_convenience_methods():
+    # provide convenience methods to IP[46]Address.vapi_af
+    def _vapi_af(self):
+        if 6 == self._version:
+            return VppEnum.vl_api_address_family_t.ADDRESS_IP6.value
+        if 4 == self._version:
+            return VppEnum.vl_api_address_family_t.ADDRESS_IP4.value
+        raise ValueError("Invalid _version.")
 
 
+    def _vapi_af_name(self):
+        if 6 == self._version:
+            return "ip6"
+        if 4 == self._version:
+            return "ip4"
+        raise ValueError("Invalid _version.")
 
 
-class VppApiDynamicMethodHolder(object):
+    ipaddress._IPAddressBase.vapi_af = property(_vapi_af)
+    ipaddress._IPAddressBase.vapi_af_name = property(_vapi_af_name)
+
+
+class VppApiDynamicMethodHolder:
     pass
 
 
     pass
 
 
-class FuncWrapper(object):
+class FuncWrapper:
     def __init__(self, func):
         self._func = func
         self.__name__ = func.__name__
     def __init__(self, func):
         self._func = func
         self.__name__ = func.__name__
@@ -90,7 +129,7 @@ class FuncWrapper(object):
         return self._func(**kwargs)
 
     def __repr__(self):
         return self._func(**kwargs)
 
     def __repr__(self):
-        return '<FuncWrapper(func=<%s(%s)>)>' % (self.__name__, self.__doc__)
+        return "<FuncWrapper(func=<%s(%s)>)>" % (self.__name__, self.__doc__)
 
 
 class VPPApiError(Exception):
 
 
 class VPPApiError(Exception):
@@ -112,9 +151,10 @@ class VPPRuntimeError(RuntimeError):
 class VPPValueError(ValueError):
     pass
 
 class VPPValueError(ValueError):
     pass
 
-class VPPApiJSONFiles(object):
+
+class VPPApiJSONFiles:
     @classmethod
     @classmethod
-    def find_api_dir(cls, dirs):
+    def find_api_dir(cls, dirs=[]):
         """Attempt to find the best directory in which API definition
         files may reside. If the value VPP_API_DIR exists in the environment
         then it is first on the search list. If we're inside a recognized
         """Attempt to find the best directory in which API definition
         files may reside. If the value VPP_API_DIR exists in the environment
         then it is first on the search list. If we're inside a recognized
@@ -129,7 +169,11 @@ class VPPApiJSONFiles(object):
         # perhaps we're in the 'src/scripts' or 'src/vpp-api/python' dir;
         # in which case, plot a course to likely places in the src tree
         import __main__ as main
         # perhaps we're in the 'src/scripts' or 'src/vpp-api/python' dir;
         # in which case, plot a course to likely places in the src tree
         import __main__ as main
-        if hasattr(main, '__file__'):
+
+        if os.getenv("VPP_API_DIR"):
+            dirs.append(os.getenv("VPP_API_DIR"))
+
+        if hasattr(main, "__file__"):
             # get the path of the calling script
             localdir = os.path.dirname(os.path.realpath(main.__file__))
         else:
             # get the path of the calling script
             localdir = os.path.dirname(os.path.realpath(main.__file__))
         else:
@@ -139,7 +183,7 @@ class VPPApiJSONFiles(object):
 
         def dmatch(dir):
             """Match dir against right-hand components of the script dir"""
 
         def dmatch(dir):
             """Match dir against right-hand components of the script dir"""
-            d = dir.split('/')  # param 'dir' assumes a / separator
+            d = dir.split("/")  # param 'dir' assumes a / separator
             length = len(d)
             return len(localdir_s) > length and localdir_s[-length:] == d
 
             length = len(d)
             return len(localdir_s) > length and localdir_s[-length:] == d
 
@@ -148,43 +192,45 @@ class VPPApiJSONFiles(object):
             'variant'  (typically '' or '_debug')"""
             # Since 'core' and 'plugin' files are staged
             # in separate directories, we target the parent dir.
             'variant'  (typically '' or '_debug')"""
             # Since 'core' and 'plugin' files are staged
             # in separate directories, we target the parent dir.
-            return os.path.sep.join((
-                srcdir,
-                'build-root',
-                'install-vpp%s-native' % variant,
-                'vpp',
-                'share',
-                'vpp',
-                'api',
-            ))
+            return os.path.sep.join(
+                (
+                    srcdir,
+                    "build-root",
+                    "install-vpp%s-native" % variant,
+                    "vpp",
+                    "share",
+                    "vpp",
+                    "api",
+                )
+            )
 
         srcdir = None
 
         srcdir = None
-        if dmatch('src/scripts'):
+        if dmatch("src/scripts"):
             srcdir = os.path.sep.join(localdir_s[:-2])
             srcdir = os.path.sep.join(localdir_s[:-2])
-        elif dmatch('src/vpp-api/python'):
+        elif dmatch("src/vpp-api/python"):
             srcdir = os.path.sep.join(localdir_s[:-3])
             srcdir = os.path.sep.join(localdir_s[:-3])
-        elif dmatch('test'):
+        elif dmatch("test"):
             # we're apparently running tests
             srcdir = os.path.sep.join(localdir_s[:-1])
 
         if srcdir:
             # we're in the source tree, try both the debug and release
             # variants.
             # we're apparently running tests
             srcdir = os.path.sep.join(localdir_s[:-1])
 
         if srcdir:
             # we're in the source tree, try both the debug and release
             # variants.
-            dirs.append(sdir(srcdir, '_debug'))
-            dirs.append(sdir(srcdir, ''))
+            dirs.append(sdir(srcdir, "_debug"))
+            dirs.append(sdir(srcdir, ""))
 
         # Test for staged copies of the scripts
         # For these, since we explicitly know if we're running a debug versus
         # release variant, target only the relevant directory
 
         # Test for staged copies of the scripts
         # For these, since we explicitly know if we're running a debug versus
         # release variant, target only the relevant directory
-        if dmatch('build-root/install-vpp_debug-native/vpp/bin'):
+        if dmatch("build-root/install-vpp_debug-native/vpp/bin"):
             srcdir = os.path.sep.join(localdir_s[:-4])
             srcdir = os.path.sep.join(localdir_s[:-4])
-            dirs.append(sdir(srcdir, '_debug'))
-        if dmatch('build-root/install-vpp-native/vpp/bin'):
+            dirs.append(sdir(srcdir, "_debug"))
+        if dmatch("build-root/install-vpp-native/vpp/bin"):
             srcdir = os.path.sep.join(localdir_s[:-4])
             srcdir = os.path.sep.join(localdir_s[:-4])
-            dirs.append(sdir(srcdir, ''))
+            dirs.append(sdir(srcdir, ""))
 
         # finally, try the location system packages typically install into
 
         # finally, try the location system packages typically install into
-        dirs.append(os.path.sep.join(('', 'usr', 'share', 'vpp', 'api')))
+        dirs.append(os.path.sep.join(("", "usr", "share", "vpp", "api")))
 
         # check the directories for existence; first one wins
         for dir in dirs:
 
         # check the directories for existence; first one wins
         for dir in dirs:
@@ -194,7 +240,7 @@ class VPPApiJSONFiles(object):
         return None
 
     @classmethod
         return None
 
     @classmethod
-    def find_api_files(cls, api_dir=None, patterns='*'):
+    def find_api_files(cls, api_dir=None, patterns="*"):  # -> list
         """Find API definition files from the given directory tree with the
         given pattern. If no directory is given then find_api_dir() is used
         to locate one. If no pattern is given then all definition files found
         """Find API definition files from the given directory tree with the
         given pattern. If no directory is given then find_api_dir() is used
         to locate one. If no pattern is given then all definition files found
@@ -220,9 +266,9 @@ class VPPApiJSONFiles(object):
                 raise VPPApiError("api_dir cannot be located")
 
         if isinstance(patterns, list) or isinstance(patterns, tuple):
                 raise VPPApiError("api_dir cannot be located")
 
         if isinstance(patterns, list) or isinstance(patterns, tuple):
-            patterns = [p.strip() + '.api.json' for p in patterns]
+            patterns = [p.strip() + ".api.json" for p in patterns]
         else:
         else:
-            patterns = [p.strip() + '.api.json' for p in patterns.split(",")]
+            patterns = [p.strip() + ".api.json" for p in patterns.split(",")]
 
         api_files = []
         for root, dirnames, files in os.walk(api_dir):
 
         api_files = []
         for root, dirnames, files in os.walk(api_dir):
@@ -236,44 +282,95 @@ class VPPApiJSONFiles(object):
     @classmethod
     def process_json_file(self, apidef_file):
         api = json.load(apidef_file)
     @classmethod
     def process_json_file(self, apidef_file):
         api = json.load(apidef_file)
+        return self._process_json(api)
+
+    @classmethod
+    def process_json_str(self, json_str):
+        api = json.loads(json_str)
+        return self._process_json(api)
+
+    @classmethod
+    def process_json_array_str(self, json_str):
+        services = {}
+        messages = {}
+
+        apis = json.loads(json_str)
+        for a in apis:
+            m, s = self._process_json(a)
+            messages.update(m)
+            services.update(s)
+        return messages, services
+
+    @staticmethod
+    def _process_json(api):  # -> Tuple[Dict, Dict]
         types = {}
         services = {}
         messages = {}
         types = {}
         services = {}
         messages = {}
-        for t in api['enums']:
-            t[0] = 'vl_api_' + t[0] + '_t'
-            types[t[0]] = {'type': 'enum', 'data': t}
-        for t in api['unions']:
-            t[0] = 'vl_api_' + t[0] + '_t'
-            types[t[0]] = {'type': 'union', 'data': t}
-        for t in api['types']:
-            t[0] = 'vl_api_' + t[0] + '_t'
-            types[t[0]] = {'type': 'type', 'data': t}
-        for t, v in api['aliases'].items():
-            types['vl_api_' + t + '_t'] = {'type': 'alias', 'data': v}
-        services.update(api['services'])
+        try:
+            for t in api["enums"]:
+                t[0] = "vl_api_" + t[0] + "_t"
+                types[t[0]] = {"type": "enum", "data": t}
+        except KeyError:
+            pass
+        try:
+            for t in api["enumflags"]:
+                t[0] = "vl_api_" + t[0] + "_t"
+                types[t[0]] = {"type": "enum", "data": t}
+        except KeyError:
+            pass
+        try:
+            for t in api["unions"]:
+                t[0] = "vl_api_" + t[0] + "_t"
+                types[t[0]] = {"type": "union", "data": t}
+        except KeyError:
+            pass
+
+        try:
+            for t in api["types"]:
+                t[0] = "vl_api_" + t[0] + "_t"
+                types[t[0]] = {"type": "type", "data": t}
+        except KeyError:
+            pass
+
+        try:
+            for t, v in api["aliases"].items():
+                types["vl_api_" + t + "_t"] = {"type": "alias", "data": v}
+        except KeyError:
+            pass
+
+        try:
+            services.update(api["services"])
+        except KeyError:
+            pass
 
         i = 0
         while True:
             unresolved = {}
             for k, v in types.items():
 
         i = 0
         while True:
             unresolved = {}
             for k, v in types.items():
-                t = v['data']
+                t = v["data"]
                 if not vpp_get_type(k):
                 if not vpp_get_type(k):
-                    if v['type'] == 'enum':
+                    if v["type"] == "enum":
                         try:
                             VPPEnumType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v
                         try:
                             VPPEnumType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v
-                    elif v['type'] == 'union':
+                if not vpp_get_type(k):
+                    if v["type"] == "enumflag":
+                        try:
+                            VPPEnumFlagType(t[0], t[1:])
+                        except ValueError:
+                            unresolved[k] = v
+                    elif v["type"] == "union":
                         try:
                             VPPUnionType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v
                         try:
                             VPPUnionType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v
-                    elif v['type'] == 'type':
+                    elif v["type"] == "type":
                         try:
                             VPPType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v
                         try:
                             VPPType(t[0], t[1:])
                         except ValueError:
                             unresolved[k] = v
-                    elif v['type'] == 'alias':
+                    elif v["type"] == "alias":
                         try:
                             VPPTypeAlias(k, t)
                         except ValueError:
                         try:
                             VPPTypeAlias(k, t)
                         except ValueError:
@@ -281,20 +378,45 @@ class VPPApiJSONFiles(object):
             if len(unresolved) == 0:
                 break
             if i > 3:
             if len(unresolved) == 0:
                 break
             if i > 3:
-                raise VPPValueError('Unresolved type definitions {}'
-                                    .format(unresolved))
+                raise VPPValueError("Unresolved type definitions {}".format(unresolved))
             types = unresolved
             i += 1
             types = unresolved
             i += 1
+        try:
+            for m in api["messages"]:
+                try:
+                    messages[m[0]] = VPPMessage(m[0], m[1:])
+                except VPPNotImplementedError:
+                    logger.error("Not implemented error for {}".format(m[0]))
+        except KeyError:
+            pass
+        return messages, services
 
 
-        for m in api['messages']:
+    @staticmethod
+    def load_api(apifiles=None, apidir=None):
+        messages = {}
+        services = {}
+        if not apifiles:
+            # Pick up API definitions from default directory
             try:
             try:
-                messages[m[0]] = VPPMessage(m[0], m[1:])
-            except VPPNotImplementedError:
-                ### OLE FIXME
-                self.logger.error('Not implemented error for {}'.format(m[0]))
-        return messages, services
+                if isinstance(apidir, list):
+                    apifiles = []
+                    for d in apidir:
+                        apifiles += VPPApiJSONFiles.find_api_files(d)
+                else:
+                    apifiles = VPPApiJSONFiles.find_api_files(apidir)
+            except (RuntimeError, VPPApiError):
+                raise VPPRuntimeError
+
+        for file in apifiles:
+            with open(file) as apidef_file:
+                m, s = VPPApiJSONFiles.process_json_file(apidef_file)
+                messages.update(m)
+                services.update(s)
+
+        return apifiles, messages, services
 
 
-class VPPApiClient(object):
+
+class VPPApiClient:
     """VPP interface.
 
     This class provides the APIs to VPP.  The APIs are loaded
     """VPP interface.
 
     This class provides the APIs to VPP.  The APIs are loaded
@@ -306,18 +428,27 @@ class VPPApiClient(object):
     provides a means to register a callback function to receive
     these messages in a background thread.
     """
     provides a means to register a callback function to receive
     these messages in a background thread.
     """
-    apidir = None
+
     VPPApiError = VPPApiError
     VPPRuntimeError = VPPRuntimeError
     VPPValueError = VPPValueError
     VPPNotImplementedError = VPPNotImplementedError
     VPPIOError = VPPIOError
 
     VPPApiError = VPPApiError
     VPPRuntimeError = VPPRuntimeError
     VPPValueError = VPPValueError
     VPPNotImplementedError = VPPNotImplementedError
     VPPIOError = VPPIOError
 
-
-    def __init__(self, apifiles=None, testmode=False, async_thread=True,
-                 logger=None, loglevel=None,
-                 read_timeout=5, use_socket=False,
-                 server_address='/run/vpp/api.sock'):
+    def __init__(
+        self,
+        *,
+        apifiles=None,
+        apidir=None,
+        testmode=False,
+        async_thread=True,
+        logger=None,
+        loglevel=None,
+        read_timeout=5,
+        use_socket=True,
+        server_address="/run/vpp/api.sock",
+        bootstrapapi=False,
+    ):
         """Create a VPP API object.
 
         apifiles is a list of files containing API
         """Create a VPP API object.
 
         apifiles is a list of files containing API
@@ -332,7 +463,8 @@ class VPPApiClient(object):
         """
         if logger is None:
             logger = logging.getLogger(
         """
         if logger is None:
             logger = logging.getLogger(
-                "{}.{}".format(__name__, self.__class__.__name__))
+                "{}.{}".format(__name__, self.__class__.__name__)
+            )
             if loglevel is not None:
                 logger.setLevel(loglevel)
         self.logger = logger
             if loglevel is not None:
                 logger.setLevel(loglevel)
         self.logger = logger
@@ -341,58 +473,64 @@ class VPPApiClient(object):
         self.services = {}
         self.id_names = []
         self.id_msgdef = []
         self.services = {}
         self.id_names = []
         self.id_msgdef = []
-        self.header = VPPType('header', [['u16', 'msgid'],
-                                         ['u32', 'client_index']])
+        self.header = VPPType("header", [["u16", "msgid"], ["u32", "client_index"]])
         self.apifiles = []
         self.apifiles = []
+        self.apidir = apidir
         self.event_callback = None
         self.message_queue = queue.Queue()
         self.read_timeout = read_timeout
         self.async_thread = async_thread
         self.event_thread = None
         self.testmode = testmode
         self.event_callback = None
         self.message_queue = queue.Queue()
         self.read_timeout = read_timeout
         self.async_thread = async_thread
         self.event_thread = None
         self.testmode = testmode
-        self.use_socket = use_socket
         self.server_address = server_address
         self._apifiles = apifiles
         self.server_address = server_address
         self._apifiles = apifiles
+        self.stats = {}
+        self.bootstrapapi = bootstrapapi
 
 
-        if use_socket:
-            from . vpp_transport_socket import VppTransport
-        else:
-            from . vpp_transport_shmem import VppTransport
-
-        if not apifiles:
-            # Pick up API definitions from default directory
+        if not bootstrapapi:
+            if self.apidir is None and hasattr(self.__class__, "apidir"):
+                # Keep supporting the old style of providing apidir.
+                self.apidir = self.__class__.apidir
             try:
             try:
-                apifiles = VPPApiJSONFiles.find_api_files(self.apidir)
-            except RuntimeError:
-                # In test mode we don't care that we can't find the API files
+                self.apifiles, self.messages, self.services = VPPApiJSONFiles.load_api(
+                    apifiles, self.apidir
+                )
+            except VPPRuntimeError as e:
                 if testmode:
                 if testmode:
-                    apifiles = []
+                    self.apifiles = []
                 else:
                 else:
-                    raise VPPRuntimeError
-
-        for file in apifiles:
-            with open(file) as apidef_file:
-                m, s = VPPApiJSONFiles.process_json_file(apidef_file)
-                self.messages.update(m)
-                self.services.update(s)
-
-        self.apifiles = apifiles
+                    raise e
+        else:
+            # Bootstrap the API (memclnt.api bundled with VPP PAPI)
+            resource_path = "/".join(("data", "memclnt.api.json"))
+            file_content = pkg_resources.resource_string(__name__, resource_path)
+            self.messages, self.services = VPPApiJSONFiles.process_json_str(
+                file_content
+            )
 
         # Basic sanity check
         if len(self.messages) == 0 and not testmode:
 
         # Basic sanity check
         if len(self.messages) == 0 and not testmode:
-            raise VPPValueError(1, 'Missing JSON message definitions')
-
-        self.transport = VppTransport(self, read_timeout=read_timeout,
-                                      server_address=server_address)
+            raise VPPValueError(1, "Missing JSON message definitions")
+        if not bootstrapapi:
+            if not (verify_enum_hint(VppEnum.vl_api_address_family_t)):
+                raise VPPRuntimeError(
+                    "Invalid address family hints. " "Cannot continue."
+                )
+
+        self.transport = VppTransport(
+            self, read_timeout=read_timeout, server_address=server_address
+        )
         # Make sure we allow VPP to clean up the message rings.
         atexit.register(vpp_atexit, weakref.ref(self))
 
         # Make sure we allow VPP to clean up the message rings.
         atexit.register(vpp_atexit, weakref.ref(self))
 
+        add_convenience_methods()
+
     def get_function(self, name):
         return getattr(self._api, name)
 
     def get_function(self, name):
         return getattr(self._api, name)
 
-
-    class ContextId(object):
+    class ContextId:
         """Multiprocessing-safe provider of unique context IDs."""
         """Multiprocessing-safe provider of unique context IDs."""
+
         def __init__(self):
             self.context = mp.Value(ctypes.c_uint, 0)
             self.lock = mp.Lock()
         def __init__(self):
             self.context = mp.Value(ctypes.c_uint, 0)
             self.lock = mp.Lock()
@@ -402,6 +540,7 @@ class VPPApiClient(object):
             with self.lock:
                 self.context.value += 1
                 return self.context.value
             with self.lock:
                 self.context.value += 1
                 return self.context.value
+
     get_context = ContextId()
 
     def get_type(self, name):
     get_context = ContextId()
 
     def get_type(self, name):
@@ -414,27 +553,37 @@ class VPPApiClient(object):
         return self._api
 
     def make_function(self, msg, i, multipart, do_async):
         return self._api
 
     def make_function(self, msg, i, multipart, do_async):
-        if (do_async):
+        if do_async:
+
             def f(**kwargs):
                 return self._call_vpp_async(i, msg, **kwargs)
             def f(**kwargs):
                 return self._call_vpp_async(i, msg, **kwargs)
+
         else:
         else:
+
             def f(**kwargs):
                 return self._call_vpp(i, msg, multipart, **kwargs)
 
         f.__name__ = str(msg.name)
             def f(**kwargs):
                 return self._call_vpp(i, msg, multipart, **kwargs)
 
         f.__name__ = str(msg.name)
-        f.__doc__ = ", ".join(["%s %s" %
-                               (msg.fieldtypes[j], k)
-                               for j, k in enumerate(msg.fields)])
+        f.__doc__ = ", ".join(
+            ["%s %s" % (msg.fieldtypes[j], k) for j, k in enumerate(msg.fields)]
+        )
         f.msg = msg
 
         return f
 
         f.msg = msg
 
         return f
 
+    def make_pack_function(self, msg, i, multipart):
+        def f(**kwargs):
+            return self._call_vpp_pack(i, msg, **kwargs)
+
+        f.msg = msg
+        return f
+
     def _register_functions(self, do_async=False):
         self.id_names = [None] * (self.vpp_dictionary_maxid + 1)
         self.id_msgdef = [None] * (self.vpp_dictionary_maxid + 1)
         self._api = VppApiDynamicMethodHolder()
     def _register_functions(self, do_async=False):
         self.id_names = [None] * (self.vpp_dictionary_maxid + 1)
         self.id_msgdef = [None] * (self.vpp_dictionary_maxid + 1)
         self._api = VppApiDynamicMethodHolder()
-        for name, msg in vpp_iterator(self.messages):
-            n = name + '_' + msg.crc[2:]
+        for name, msg in self.messages.items():
+            n = name + "_" + msg.crc[2:]
             i = self.transport.get_msg_index(n)
             if i > 0:
                 self.id_msgdef[i] = msg
             i = self.transport.get_msg_index(n)
             if i > 0:
                 self.id_msgdef[i] = msg
@@ -442,36 +591,50 @@ class VPPApiClient(object):
 
                 # Create function for client side messages.
                 if name in self.services:
 
                 # Create function for client side messages.
                 if name in self.services:
-                    if 'stream' in self.services[name] and \
-                       self.services[name]['stream']:
-                        multipart = True
-                    else:
-                        multipart = False
-                    f = self.make_function(msg, i, multipart, do_async)
+                    f = self.make_function(msg, i, self.services[name], do_async)
+                    f_pack = self.make_pack_function(msg, i, self.services[name])
                     setattr(self._api, name, FuncWrapper(f))
                     setattr(self._api, name, FuncWrapper(f))
+                    setattr(self._api, name + "_pack", FuncWrapper(f_pack))
             else:
             else:
-                self.logger.debug(
-                    'No such message type or failed CRC checksum: %s', n)
+                self.logger.debug("No such message type or failed CRC checksum: %s", n)
+
+    def get_api_definitions(self):
+        """get_api_definition. Bootstrap from the embedded memclnt.api.json file."""
+
+        # Bootstrap so we can call the get_api_json function
+        self._register_functions(do_async=False)
 
 
-    def connect_internal(self, name, msg_handler, chroot_prefix, rx_qlen,
-                         do_async):
-        pfx = chroot_prefix.encode('utf-8') if chroot_prefix else None
+        r = self.api.get_api_json()
+        if r.retval != 0:
+            raise VPPApiError("Failed to load API definitions from VPP")
 
 
-        rv = self.transport.connect(name, pfx,
-                                    msg_handler, rx_qlen)
+        # Process JSON
+        m, s = VPPApiJSONFiles.process_json_array_str(r.json)
+        self.messages.update(m)
+        self.services.update(s)
+
+    def connect_internal(self, name, msg_handler, chroot_prefix, rx_qlen, do_async):
+        pfx = chroot_prefix.encode("utf-8") if chroot_prefix else None
+
+        rv = self.transport.connect(name, pfx, msg_handler, rx_qlen, do_async)
         if rv != 0:
         if rv != 0:
-            raise VPPIOError(2, 'Connect failed')
+            raise VPPIOError(2, "Connect failed")
         self.vpp_dictionary_maxid = self.transport.msg_table_max_index()
         self.vpp_dictionary_maxid = self.transport.msg_table_max_index()
+
+        # Register functions
+        if self.bootstrapapi:
+            self.get_api_definitions()
         self._register_functions(do_async=do_async)
 
         # Initialise control ping
         self._register_functions(do_async=do_async)
 
         # Initialise control ping
-        crc = self.messages['control_ping'].crc
+        crc = self.messages["control_ping"].crc
         self.control_ping_index = self.transport.get_msg_index(
         self.control_ping_index = self.transport.get_msg_index(
-            ('control_ping' + '_' + crc[2:]))
-        self.control_ping_msgdef = self.messages['control_ping']
+            ("control_ping" + "_" + crc[2:])
+        )
+        self.control_ping_msgdef = self.messages["control_ping"]
+
         if self.async_thread:
         if self.async_thread:
-            self.event_thread = threading.Thread(
-                target=self.thread_msg_handler)
+            self.event_thread = threading.Thread(target=self.thread_msg_handler)
             self.event_thread.daemon = True
             self.event_thread.start()
         else:
             self.event_thread.daemon = True
             self.event_thread.start()
         else:
@@ -488,8 +651,9 @@ class VPPApiClient(object):
         client and server.
         """
         msg_handler = self.transport.get_callback(do_async)
         client and server.
         """
         msg_handler = self.transport.get_callback(do_async)
-        return self.connect_internal(name, msg_handler, chroot_prefix, rx_qlen,
-                                     do_async)
+        return self.connect_internal(
+            name, msg_handler, chroot_prefix, rx_qlen, do_async
+        )
 
     def connect_sync(self, name, chroot_prefix=None, rx_qlen=32):
         """Attach to VPP in synchronous mode. Application must poll for events.
 
     def connect_sync(self, name, chroot_prefix=None, rx_qlen=32):
         """Attach to VPP in synchronous mode. Application must poll for events.
@@ -500,8 +664,7 @@ class VPPApiClient(object):
         client and server.
         """
 
         client and server.
         """
 
-        return self.connect_internal(name, None, chroot_prefix, rx_qlen,
-                                     do_async=False)
+        return self.connect_internal(name, None, chroot_prefix, rx_qlen, do_async=False)
 
     def disconnect(self):
         """Detach from VPP."""
 
     def disconnect(self):
         """Detach from VPP."""
@@ -522,42 +685,44 @@ class VPPApiClient(object):
         # If we have a context, then use the context to find any
         # request waiting for a reply
         context = 0
         # If we have a context, then use the context to find any
         # request waiting for a reply
         context = 0
-        if hasattr(r, 'context') and r.context > 0:
+        if hasattr(r, "context") and r.context > 0:
             context = r.context
 
         if context == 0:
             # No context -> async notification that we feed to the callback
             self.message_queue.put_nowait(r)
         else:
             context = r.context
 
         if context == 0:
             # No context -> async notification that we feed to the callback
             self.message_queue.put_nowait(r)
         else:
-            raise VPPIOError(2, 'RPC reply message received in event handler')
+            raise VPPIOError(2, "RPC reply message received in event handler")
 
     def has_context(self, msg):
         if len(msg) < 10:
             return False
 
 
     def has_context(self, msg):
         if len(msg) < 10:
             return False
 
-        header = VPPType('header_with_context', [['u16', 'msgid'],
-                                                 ['u32', 'client_index'],
-                                                 ['u32', 'context']])
+        header = VPPType(
+            "header_with_context",
+            [["u16", "msgid"], ["u32", "client_index"], ["u32", "context"]],
+        )
 
         (i, ci, context), size = header.unpack(msg, 0)
 
         (i, ci, context), size = header.unpack(msg, 0)
-        if self.id_names[i] == 'rx_thread_exit':
+
+        if self.id_names[i] == "rx_thread_exit":
             return
 
         #
         # Decode message and returns a tuple.
         #
         msgobj = self.id_msgdef[i]
             return
 
         #
         # Decode message and returns a tuple.
         #
         msgobj = self.id_msgdef[i]
-        if 'context' in msgobj.field_by_name and context >= 0:
+        if "context" in msgobj.field_by_name and context >= 0:
             return True
         return False
 
     def decode_incoming_msg(self, msg, no_type_conversion=False):
         if not msg:
             return True
         return False
 
     def decode_incoming_msg(self, msg, no_type_conversion=False):
         if not msg:
-            self.logger.warning('vpp_api.read failed')
+            logger.warning("vpp_api.read failed")
             return
 
         (i, ci), size = self.header.unpack(msg, 0)
             return
 
         (i, ci), size = self.header.unpack(msg, 0)
-        if self.id_names[i] == 'rx_thread_exit':
+        if self.id_names[i] == "rx_thread_exit":
             return
 
         #
             return
 
         #
@@ -565,7 +730,7 @@ class VPPApiClient(object):
         #
         msgobj = self.id_msgdef[i]
         if not msgobj:
         #
         msgobj = self.id_msgdef[i]
         if not msgobj:
-            raise VPPIOError(2, 'Reply message undefined')
+            raise VPPIOError(2, "Reply message undefined")
 
         r, size = msgobj.unpack(msg, ntc=no_type_conversion)
         return r
 
         r, size = msgobj.unpack(msg, ntc=no_type_conversion)
         return r
@@ -586,17 +751,42 @@ class VPPApiClient(object):
 
     def _control_ping(self, context):
         """Send a ping command."""
 
     def _control_ping(self, context):
         """Send a ping command."""
-        self._call_vpp_async(self.control_ping_index,
-                             self.control_ping_msgdef,
-                             context=context)
+        self._call_vpp_async(
+            self.control_ping_index, self.control_ping_msgdef, context=context
+        )
 
     def validate_args(self, msg, kwargs):
         d = set(kwargs.keys()) - set(msg.field_by_name.keys())
         if d:
 
     def validate_args(self, msg, kwargs):
         d = set(kwargs.keys()) - set(msg.field_by_name.keys())
         if d:
-            raise VPPValueError('Invalid argument {} to {}'
-                                .format(list(d), msg.name))
+            raise VPPValueError("Invalid argument {} to {}".format(list(d), msg.name))
 
 
-    def _call_vpp(self, i, msgdef, multipart, **kwargs):
+    def _add_stat(self, name, ms):
+        if not name in self.stats:
+            self.stats[name] = {"max": ms, "count": 1, "avg": ms}
+        else:
+            if ms > self.stats[name]["max"]:
+                self.stats[name]["max"] = ms
+            self.stats[name]["count"] += 1
+            n = self.stats[name]["count"]
+            self.stats[name]["avg"] = self.stats[name]["avg"] * (n - 1) / n + ms / n
+
+    def get_stats(self):
+        s = "\n=== API PAPI STATISTICS ===\n"
+        s += "{:<30} {:>4} {:>6} {:>6}\n".format("message", "cnt", "avg", "max")
+        for n in sorted(self.stats.items(), key=lambda v: v[1]["avg"], reverse=True):
+            s += "{:<30} {:>4} {:>6.2f} {:>6.2f}\n".format(
+                n[0], n[1]["count"], n[1]["avg"], n[1]["max"]
+            )
+        return s
+
+    def get_field_options(self, msg, fld_name):
+        # when there is an option, the msgdef has 3 elements.
+        # ['u32', 'ring_size', {'default': 1024}]
+        for _def in self.messages[msg].msgdef:
+            if isinstance(_def, list) and len(_def) == 3 and _def[1] == fld_name:
+                return _def[2]
+
+    def _call_vpp(self, i, msgdef, service, **kwargs):
         """Given a message, send the message and await a reply.
 
         msgdef - the message packing definition
         """Given a message, send the message and await a reply.
 
         msgdef - the message packing definition
@@ -611,25 +801,27 @@ class VPPApiClient(object):
         the response.  It will raise an IOError exception if there was
         no response within the timeout window.
         """
         the response.  It will raise an IOError exception if there was
         no response within the timeout window.
         """
-
-        if 'context' not in kwargs:
+        ts = time.time()
+        if "context" not in kwargs:
             context = self.get_context()
             context = self.get_context()
-            kwargs['context'] = context
+            kwargs["context"] = context
         else:
         else:
-            context = kwargs['context']
-        kwargs['_vl_msg_id'] = i
+            context = kwargs["context"]
+        kwargs["_vl_msg_id"] = i
 
 
-        no_type_conversion = kwargs.pop('_no_type_conversion', False)
+        no_type_conversion = kwargs.pop("_no_type_conversion", False)
+        timeout = kwargs.pop("_timeout", None)
 
         try:
             if self.transport.socket_index:
 
         try:
             if self.transport.socket_index:
-                kwargs['client_index'] = self.transport.socket_index
+                kwargs["client_index"] = self.transport.socket_index
         except AttributeError:
             pass
         self.validate_args(msgdef, kwargs)
 
         except AttributeError:
             pass
         self.validate_args(msgdef, kwargs)
 
-        s = 'Calling {}({})'.format(msgdef.name,
-            ','.join(['{!r}:{!r}'.format(k, v) for k, v in kwargs.items()]))
+        s = "Calling {}({})".format(
+            msgdef.name, ",".join(["{!r}:{!r}".format(k, v) for k, v in kwargs.items()])
+        )
         self.logger.debug(s)
 
         b = msgdef.pack(kwargs)
         self.logger.debug(s)
 
         b = msgdef.pack(kwargs)
@@ -637,38 +829,53 @@ class VPPApiClient(object):
 
         self.transport.write(b)
 
 
         self.transport.write(b)
 
-        if multipart:
-            # Send a ping after the request - we use its response
-            # to detect that we have seen all results.
-            self._control_ping(context)
+        msgreply = service["reply"]
+        stream = True if "stream" in service else False
+        if stream:
+            if "stream_msg" in service:
+                # New service['reply'] = _reply and service['stream_message'] = _details
+                stream_message = service["stream_msg"]
+                modern = True
+            else:
+                # Old  service['reply'] = _details
+                stream_message = msgreply
+                msgreply = "control_ping_reply"
+                modern = False
+                # Send a ping after the request - we use its response
+                # to detect that we have seen all results.
+                self._control_ping(context)
 
         # Block until we get a reply.
         rl = []
 
         # Block until we get a reply.
         rl = []
-        while (True):
-            msg = self.transport.read()
-            if not msg:
-                raise VPPIOError(2, 'VPP API client: read failed')
-            r = self.decode_incoming_msg(msg, no_type_conversion)
+        while True:
+            r = self.read_blocking(no_type_conversion, timeout)
+            if r is None:
+                raise VPPIOError(2, "VPP API client: read failed")
             msgname = type(r).__name__
             if context not in r or r.context == 0 or context != r.context:
                 # Message being queued
                 self.message_queue.put_nowait(r)
                 continue
             msgname = type(r).__name__
             if context not in r or r.context == 0 or context != r.context:
                 # Message being queued
                 self.message_queue.put_nowait(r)
                 continue
-
-            if not multipart:
+            if msgname != msgreply and (stream and (msgname != stream_message)):
+                print("REPLY MISMATCH", msgreply, msgname, stream_message, stream)
+            if not stream:
                 rl = r
                 break
                 rl = r
                 break
-            if msgname == 'control_ping_reply':
+            if msgname == msgreply:
+                if modern:  # Return both reply and list
+                    rl = r, rl
                 break
 
             rl.append(r)
 
         self.transport.resume()
 
                 break
 
             rl.append(r)
 
         self.transport.resume()
 
-        s = 'Return value: {!r}'.format(r)
+        s = "Return value: {!r}".format(r)
         if len(s) > 80:
             s = s[:80] + "..."
         self.logger.debug(s)
         if len(s) > 80:
             s = s[:80] + "..."
         self.logger.debug(s)
+        te = time.time()
+        self._add_stat(msgdef.name, (te - ts) * 1000)
         return rl
 
     def _call_vpp_async(self, i, msg, **kwargs):
         return rl
 
     def _call_vpp_async(self, i, msg, **kwargs):
@@ -684,22 +891,57 @@ class VPPApiClient(object):
         The returned context will help with assigning which call
         the reply belongs to.
         """
         The returned context will help with assigning which call
         the reply belongs to.
         """
-        if 'context' not in kwargs:
+        if "context" not in kwargs:
             context = self.get_context()
             context = self.get_context()
-            kwargs['context'] = context
+            kwargs["context"] = context
         else:
         else:
-            context = kwargs['context']
+            context = kwargs["context"]
         try:
             if self.transport.socket_index:
         try:
             if self.transport.socket_index:
-                kwargs['client_index'] = self.transport.socket_index
+                kwargs["client_index"] = self.transport.socket_index
         except AttributeError:
         except AttributeError:
-            kwargs['client_index'] = 0
-        kwargs['_vl_msg_id'] = i
+            kwargs["client_index"] = 0
+        kwargs["_vl_msg_id"] = i
         b = msg.pack(kwargs)
 
         self.transport.write(b)
         return context
 
         b = msg.pack(kwargs)
 
         self.transport.write(b)
         return context
 
+    def _call_vpp_pack(self, i, msg, **kwargs):
+        """Given a message, return the binary representation."""
+        kwargs["_vl_msg_id"] = i
+        kwargs["client_index"] = 0
+        kwargs["context"] = 0
+        return msg.pack(kwargs)
+
+    def read_blocking(self, no_type_conversion=False, timeout=None):
+        """Get next received message from transport within timeout, decoded.
+
+        Note that notifications have context zero
+        and are not put into receive queue (at least for socket transport),
+        use async_thread with registered callback for processing them.
+
+        If no message appears in the queue within timeout, return None.
+
+        Optionally, type conversion can be skipped,
+        as some of conversions are into less precise types.
+
+        When r is the return value of this, the caller can get message name as:
+            msgname = type(r).__name__
+        and context number (type long) as:
+            context = r.context
+
+        :param no_type_conversion: If false, type conversions are applied.
+        :type no_type_conversion: bool
+        :returns: Decoded message, or None if no message (within timeout).
+        :rtype: Whatever VPPType.unpack returns, depends on no_type_conversion.
+        :raises VppTransportShmemIOError if timed out.
+        """
+        msg = self.transport.read(timeout=timeout)
+        if not msg:
+            return None
+        return self.decode_incoming_msg(msg, no_type_conversion)
+
     def register_event_callback(self, callback):
         """Register a callback for async messages.
 
     def register_event_callback(self, callback):
         """Register a callback for async messages.
 
@@ -733,16 +975,56 @@ class VPPApiClient(object):
             if self.event_callback:
                 self.event_callback(msgname, r)
 
             if self.event_callback:
                 self.event_callback(msgname, r)
 
-    def __repr__(self):
-        return "<VPPApiClient apifiles=%s, testmode=%s, async_thread=%s, " \
-               "logger=%s, read_timeout=%s, use_socket=%s, " \
-               "server_address='%s'>" % (
-                   self._apifiles, self.testmode, self.async_thread,
-                   self.logger, self.read_timeout, self.use_socket,
-                   self.server_address)
-
-
-# Provide the old name for backward compatibility.
-VPP = VPPApiClient
+    def validate_message_table(self, namecrctable):
+        """Take a dictionary of name_crc message names
+        and returns an array of missing messages"""
+
+        missing_table = []
+        for name_crc in namecrctable:
+            i = self.transport.get_msg_index(name_crc)
+            if i <= 0:
+                missing_table.append(name_crc)
+        return missing_table
+
+    def dump_message_table(self):
+        """Return VPPs API message table as name_crc dictionary"""
+        return self.transport.message_table
+
+    def dump_message_table_filtered(self, msglist):
+        """Return VPPs API message table as name_crc dictionary,
+        filtered by message name list."""
+
+        replies = [self.services[n]["reply"] for n in msglist]
+        message_table_filtered = {}
+        for name in msglist + replies:
+            for k, v in self.transport.message_table.items():
+                if k.startswith(name):
+                    message_table_filtered[k] = v
+                    break
+        return message_table_filtered
 
 
-# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
+    def __repr__(self):
+        return (
+            "<VPPApiClient apifiles=%s, testmode=%s, async_thread=%s, "
+            "logger=%s, read_timeout=%s, "
+            "server_address='%s'>"
+            % (
+                self._apifiles,
+                self.testmode,
+                self.async_thread,
+                self.logger,
+                self.read_timeout,
+                self.server_address,
+            )
+        )
+
+    def details_iter(self, f, **kwargs):
+        cursor = 0
+        while True:
+            kwargs["cursor"] = cursor
+            rv, details = f(**kwargs)
+            for d in details:
+                yield d
+            if rv.retval == 0 or rv.retval != -165:
+                break
+            cursor = rv.cursor