X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvpp-api%2Fpython%2Fvpp_papi.py;h=7305ef387639a0adbbb8710a2f04b51d132b6afb;hb=68e2ffb3ef7e1db908fc874e733d0f4db18e0cb5;hp=c1fa9e8bff937b489c7eacfee6150b9c204dd049;hpb=4df97165159b3b115b31eb1cad55782ac97e3c7e;p=vpp.git diff --git a/src/vpp-api/python/vpp_papi.py b/src/vpp-api/python/vpp_papi.py index c1fa9e8bff9..7305ef38763 100644 --- a/src/vpp-api/python/vpp_papi.py +++ b/src/vpp-api/python/vpp_papi.py @@ -22,9 +22,11 @@ import collections import struct import json import threading -import glob +import fnmatch +import weakref import atexit from cffi import FFI +import cffi if sys.version[0] == '2': import Queue as queue @@ -55,11 +57,13 @@ void vac_set_error_handler(vac_error_callback_t); vpp_api = ffi.dlopen('libvppapiclient.so') -def vpp_atexit(self): +def vpp_atexit(vpp_weakref): """Clean up VPP connection on shutdown.""" - if self.connected: - self.logger.debug('Cleaning up VPP on exit') - self.disconnect() + vpp_instance = vpp_weakref() + if vpp_instance.connected: + vpp_instance.logger.debug('Cleaning up VPP on exit') + vpp_instance.disconnect() + vpp_object = None @@ -112,7 +116,8 @@ class VPP(): these messages in a background thread. """ def __init__(self, apifiles=None, testmode=False, async_thread=True, - logger=logging.getLogger('vpp_papi'), loglevel='debug'): + logger=logging.getLogger('vpp_papi'), loglevel='debug', + read_timeout=0): """Create a VPP API object. apifiles is a list of files containing API @@ -129,23 +134,25 @@ class VPP(): self.messages = {} self.id_names = [] self.id_msgdef = [] - self.buffersize = 10000 self.connected = False self.header = struct.Struct('>HI') self.apifiles = [] self.event_callback = None self.message_queue = queue.Queue() - self.read_timeout = 0 + self.read_timeout = read_timeout self.vpp_api = vpp_api - if async_thread: - self.event_thread = threading.Thread( - target=self.thread_msg_handler) - self.event_thread.daemon = True - self.event_thread.start() + self.async_thread = async_thread if not apifiles: # Pick up API definitions from default directory - apifiles = glob.glob('/usr/share/vpp/api/*.api.json') + try: + apifiles = self.find_api_files() + except RuntimeError: + # In test mode we don't care that we can't find the API files + if testmode: + apifiles = [] + else: + raise for file in apifiles: with open(file) as apidef_file: @@ -162,11 +169,20 @@ class VPP(): raise ValueError(1, 'Missing JSON message definitions') # Make sure we allow VPP to clean up the message rings. - atexit.register(vpp_atexit, self) + atexit.register(vpp_atexit, weakref.ref(self)) # Register error handler vpp_api.vac_set_error_handler(vac_error_handler) + # Support legacy CFFI + # from_buffer supported from 1.8.0 + (major, minor, patch) = [int(s) for s in + cffi.__version__.split('.', 3)] + if major >= 1 and minor >= 8: + self._write = self._write_new_cffi + else: + self._write = self._write_legacy_cffi + class ContextId(object): """Thread-safe provider of unique context IDs.""" def __init__(self): @@ -180,6 +196,130 @@ class VPP(): return self.context get_context = ContextId() + @classmethod + def find_api_dir(cls): + """Attempt to find the best directory in which API definition + files may reside. If the value VPP_API_DIR exists in the environment + then it is first on the search list. If we're inside a recognized + location in a VPP source tree (src/scripts and src/vpp-api/python) + then entries from there to the likely locations in build-root are + added. Finally the location used by system packages is added. + + :returns: A single directory name, or None if no such directory + could be found. + """ + dirs = [] + + if 'VPP_API_DIR' in os.environ: + dirs.append(os.environ['VPP_API_DIR']) + + # perhaps we're in the 'src/scripts' or 'src/vpp-api/python' dir; + # in which case, plot a course to likely places in the src tree + import __main__ as main + if hasattr(main, '__file__'): + # get the path of the calling script + localdir = os.path.dirname(os.path.realpath(main.__file__)) + else: + # use cwd if there is no calling script + localdir = os.getcwd() + localdir_s = localdir.split(os.path.sep) + + def dmatch(dir): + """Match dir against right-hand components of the script dir""" + d = dir.split('/') # param 'dir' assumes a / separator + length = len(d) + return len(localdir_s) > length and localdir_s[-length:] == d + + def sdir(srcdir, variant): + """Build a path from srcdir to the staged API files of + 'variant' (typically '' or '_debug')""" + # Since 'core' and 'plugin' files are staged + # in separate directories, we target the parent dir. + return os.path.sep.join(( + srcdir, + 'build-root', + 'install-vpp%s-native' % variant, + 'vpp', + 'share', + 'vpp', + 'api', + )) + + srcdir = None + if dmatch('src/scripts'): + srcdir = os.path.sep.join(localdir_s[:-2]) + elif dmatch('src/vpp-api/python'): + srcdir = os.path.sep.join(localdir_s[:-3]) + elif dmatch('test'): + # we're apparently running tests + srcdir = os.path.sep.join(localdir_s[:-1]) + + if srcdir: + # we're in the source tree, try both the debug and release + # variants. + dirs.append(sdir(srcdir, '_debug')) + dirs.append(sdir(srcdir, '')) + + # Test for staged copies of the scripts + # For these, since we explicitly know if we're running a debug versus + # release variant, target only the relevant directory + if dmatch('build-root/install-vpp_debug-native/vpp/bin'): + srcdir = os.path.sep.join(localdir_s[:-4]) + dirs.append(sdir(srcdir, '_debug')) + if dmatch('build-root/install-vpp-native/vpp/bin'): + srcdir = os.path.sep.join(localdir_s[:-4]) + dirs.append(sdir(srcdir, '')) + + # finally, try the location system packages typically install into + dirs.append(os.path.sep.join(('', 'usr', 'share', 'vpp', 'api'))) + + # check the directories for existance; first one wins + for dir in dirs: + if os.path.isdir(dir): + return dir + + return None + + @classmethod + def find_api_files(cls, api_dir=None, patterns='*'): + """Find API definition files from the given directory tree with the + given pattern. If no directory is given then find_api_dir() is used + to locate one. If no pattern is given then all definition files found + in the directory tree are used. + + :param api_dir: A directory tree in which to locate API definition + files; subdirectories are descended into. + If this is None then find_api_dir() is called to discover it. + :param patterns: A list of patterns to use in each visited directory + when looking for files. + This can be a list/tuple object or a comma-separated string of + patterns. Each value in the list will have leading/trialing + whitespace stripped. + The pattern specifies the first part of the filename, '.api.json' + is appended. + The results are de-duplicated, thus overlapping patterns are fine. + If this is None it defaults to '*' meaning "all API files". + :returns: A list of file paths for the API files found. + """ + if api_dir is None: + api_dir = cls.find_api_dir() + if api_dir is None: + raise RuntimeError("api_dir cannot be located") + + if isinstance(patterns, list) or isinstance(patterns, tuple): + patterns = [p.strip() + '.api.json' for p in patterns] + else: + patterns = [p.strip() + '.api.json' for p in patterns.split(",")] + + api_files = [] + for root, dirnames, files in os.walk(api_dir): + # iterate all given patterns and de-dup the result + files = set(sum([fnmatch.filter(files, p) for p in patterns], [])) + for filename in files: + api_files.append(os.path.join(root, filename)) + + return api_files + def status(self): """Debug function: report current VPP API status to stdout.""" print('Connected') if self.connected else print('Not Connected') @@ -193,41 +333,50 @@ class VPP(): 'i32': 'i', 'u64': 'Q', 'f64': 'd', } - pack = None + if t in base_types: - pack = base_types[t] if not vl: if e > 0 and t == 'u8': # Fixed byte array - return struct.Struct('>' + str(e) + 's') + s = struct.Struct('>' + str(e) + 's') + return s.size, s if e > 0: # Fixed array of base type - return [e, struct.Struct('>' + base_types[t])] + s = struct.Struct('>' + base_types[t]) + return s.size, [e, s] elif e == 0: # Old style variable array - return [-1, struct.Struct('>' + base_types[t])] + s = struct.Struct('>' + base_types[t]) + return s.size, [-1, s] else: # Variable length array - return [vl, struct.Struct('>s')] if t == 'u8' else \ - [vl, struct.Struct('>' + base_types[t])] + if t == 'u8': + s = struct.Struct('>s') + return s.size, [vl, s] + else: + s = struct.Struct('>' + base_types[t]) + return s.size, [vl, s] - return struct.Struct('>' + base_types[t]) + s = struct.Struct('>' + base_types[t]) + return s.size, s if t in self.messages: + size = self.messages[t]['sizes'][0] + # Return a list in case of array if e > 0 and not vl: - return [e, lambda self, encode, buf, offset, args: ( + return size, [e, lambda self, encode, buf, offset, args: ( self.__struct_type(encode, self.messages[t], buf, offset, args))] if vl: - return [vl, lambda self, encode, buf, offset, args: ( + return size, [vl, lambda self, encode, buf, offset, args: ( self.__struct_type(encode, self.messages[t], buf, offset, args))] elif e == 0: # Old style VLA raise NotImplementedError(1, 'No support for compound types ' + t) - return lambda self, encode, buf, offset, args: ( + return size, lambda self, encode, buf, offset, args: ( self.__struct_type(encode, self.messages[t], buf, offset, args) ) @@ -246,7 +395,9 @@ class VPP(): for k in kwargs: if k not in msgdef['args']: - raise ValueError(1, 'Invalid field-name in message call ' + k) + raise ValueError(1, 'Non existing argument [' + k + ']' + + ' used in call to: ' + + self.id_names[kwargs['_vl_msg_id']] + '()') for k, v in vpp_iterator(msgdef['args']): off += size @@ -254,18 +405,29 @@ class VPP(): if type(v) is list: if callable(v[1]): e = kwargs[v[0]] if v[0] in kwargs else v[0] + if e != len(kwargs[k]): + raise (ValueError(1, + 'Input list length mismatch: ' + '%s (%s != %s)' % + (k, e, len(kwargs[k])))) size = 0 for i in range(e): size += v[1](self, True, buf, off + size, kwargs[k][i]) else: if v[0] in kwargs: - l = kwargs[v[0]] + kwargslen = kwargs[v[0]] + if kwargslen != len(kwargs[k]): + raise ValueError(1, + 'Input list length mismatch:' + ' %s (%s != %s)' % + (k, kwargslen, + len(kwargs[k]))) else: - l = len(kwargs[k]) + kwargslen = len(kwargs[k]) if v[1].size == 1: - buf[off:off + l] = bytearray(kwargs[k]) - size = l + buf[off:off + kwargslen] = bytearray(kwargs[k]) + size = kwargslen else: size = 0 for i in kwargs[k]: @@ -275,6 +437,11 @@ class VPP(): if callable(v): size = v(self, True, buf, off, kwargs[k]) else: + if type(kwargs[k]) is str and v.size < len(kwargs[k]): + raise ValueError(1, + 'Input list length mismatch: ' + '%s (%s < %s)' % + (k, v.size, len(kwargs[k]))) v.pack_into(buf, off, kwargs[k]) size = v.size else: @@ -287,9 +454,17 @@ class VPP(): return self.messages[name] return None + def get_size(self, sizes, kwargs): + total_size = sizes[0] + for e in sizes[1]: + if e in kwargs and type(kwargs[e]) is list: + total_size += len(kwargs[e]) * sizes[1][e] + return total_size + def encode(self, msgdef, kwargs): # Make suitably large buffer - buf = bytearray(self.buffersize) + size = self.get_size(msgdef['sizes'], kwargs) + buf = bytearray(size) offset = 0 size = self.__struct_type(True, msgdef, buf, offset, kwargs) return buf[:offset + size] @@ -335,6 +510,7 @@ class VPP(): size += v[1].size else: if callable(v): + size = 0 (s, l) = v(self, False, buf, off, None) res.append(l) size += s @@ -349,14 +525,31 @@ class VPP(): return self.messages[name]['return_tuple'] return None + def duplicate_check_ok(self, name, msgdef): + crc = None + for c in msgdef: + if type(c) is dict and 'crc' in c: + crc = c['crc'] + break + if crc: + # We can get duplicates because of imports + if crc == self.messages[name]['crc']: + return True + return False + def add_message(self, name, msgdef, typeonly=False): if name in self.messages: + if typeonly: + if self.duplicate_check_ok(name, msgdef): + return raise ValueError('Duplicate message name: ' + name) args = collections.OrderedDict() argtypes = collections.OrderedDict() fields = [] msg = {} + total_size = 0 + sizes = {} for i, f in enumerate(msgdef): if type(f) is dict and 'crc' in f: msg['crc'] = f['crc'] @@ -365,7 +558,19 @@ class VPP(): field_name = f[1] if len(f) == 3 and f[2] == 0 and i != len(msgdef) - 2: raise ValueError('Variable Length Array must be last: ' + name) - args[field_name] = self.__struct(*f) + size, s = self.__struct(*f) + args[field_name] = s + if type(s) == list and type(s[0]) == int and \ + type(s[1]) == struct.Struct: + if s[0] < 0: + sizes[field_name] = size + else: + sizes[field_name] = size + total_size += s[0] * size + else: + sizes[field_name] = size + total_size += size + argtypes[field_name] = field_type if len(f) == 4: # Find offset to # elements field idx = list(args.keys()).index(f[3]) - i @@ -377,6 +582,7 @@ class VPP(): self.messages[name]['args'] = args self.messages[name]['argtypes'] = argtypes self.messages[name]['typeonly'] = typeonly + self.messages[name]['sizes'] = [total_size, sizes] return self.messages[name] def add_type(self, name, typedef): @@ -385,10 +591,11 @@ class VPP(): def make_function(self, name, i, msgdef, multipart, async): if (async): - f = lambda **kwargs: (self._call_vpp_async(i, msgdef, **kwargs)) + def f(**kwargs): + return self._call_vpp_async(i, msgdef, **kwargs) else: - f = lambda **kwargs: (self._call_vpp(i, msgdef, multipart, - **kwargs)) + def f(**kwargs): + return self._call_vpp(i, msgdef, multipart, **kwargs) args = self.messages[name]['args'] argtypes = self.messages[name]['argtypes'] f.__name__ = str(name) @@ -418,23 +625,22 @@ class VPP(): multipart = True if name.find('_dump') > 0 else False f = self.make_function(name, i, msgdef, multipart, async) setattr(self._api, name, FuncWrapper(f)) - - # old API stuff starts here - will be removed in 17.07 - if hasattr(self, name): - raise NameError( - 3, "Conflicting name in JSON definition: `%s'" % name) - setattr(self, name, f) - # old API stuff ends here else: self.logger.debug( 'No such message type or failed CRC checksum: %s', n) - def _write(self, buf): + def _write_new_cffi(self, buf): """Send a binary-packed message to VPP.""" if not self.connected: raise IOError(1, 'Not connected') return vpp_api.vac_write(ffi.from_buffer(buf), len(buf)) + def _write_legacy_cffi(self, buf): + """Send a binary-packed message to VPP.""" + if not self.connected: + raise IOError(1, 'Not connected') + return vpp_api.vac_write(bytes(buf), len(buf)) + def _read(self): if not self.connected: raise IOError(1, 'Not connected') @@ -442,15 +648,15 @@ class VPP(): size = ffi.new("int *") rv = vpp_api.vac_read(mem, size, self.read_timeout) if rv: - raise IOError(rv, 'vac_read filed') + raise IOError(rv, 'vac_read failed') msg = bytes(ffi.buffer(mem[0], size[0])) vpp_api.vac_free(mem[0]) return msg def connect_internal(self, name, msg_handler, chroot_prefix, rx_qlen, async): - rv = vpp_api.vac_connect(name.encode(), chroot_prefix.encode(), - msg_handler, rx_qlen) + pfx = chroot_prefix.encode() if chroot_prefix else ffi.NULL + rv = vpp_api.vac_connect(name.encode(), pfx, msg_handler, rx_qlen) if rv != 0: raise IOError(2, 'Connect failed') self.connected = True @@ -463,9 +669,14 @@ class VPP(): self.control_ping_index = vpp_api.vac_get_msg_index( ('control_ping' + '_' + crc[2:]).encode()) self.control_ping_msgdef = self.messages['control_ping'] + if self.async_thread: + self.event_thread = threading.Thread( + target=self.thread_msg_handler) + self.event_thread.daemon = True + self.event_thread.start() return rv - def connect(self, name, chroot_prefix=ffi.NULL, async=False, rx_qlen=32): + def connect(self, name, chroot_prefix=None, async=False, rx_qlen=32): """Attach to VPP. name - the name of the client. @@ -478,7 +689,7 @@ class VPP(): return self.connect_internal(name, msg_handler, chroot_prefix, rx_qlen, async) - def connect_sync(self, name, chroot_prefix=ffi.NULL, rx_qlen=32): + def connect_sync(self, name, chroot_prefix=None, rx_qlen=32): """Attach to VPP in synchronous mode. Application must poll for events. name - the name of the client. @@ -494,6 +705,7 @@ class VPP(): """Detach from VPP.""" rv = vpp_api.vac_disconnect() self.connected = False + self.message_queue.put("terminate event thread") return rv def msg_handler_sync(self, msg): @@ -511,8 +723,6 @@ class VPP(): if hasattr(r, 'context') and r.context > 0: context = r.context - msgname = type(r).__name__ - if context == 0: # No context -> async notification that we feed to the callback self.message_queue.put_nowait(r) @@ -662,6 +872,11 @@ class VPP(): """ while True: r = self.message_queue.get() + if r == "terminate event thread": + break msgname = type(r).__name__ if self.event_callback: self.event_callback(msgname, r) + + +# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4