X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvpp-api%2Fpython%2Fvpp_papi%2Fvpp_papi.py;h=05688cec7319ff86f2992a33402a789bfa102182;hb=ead1e536d66d83b546528c32e2112085a97c8e13;hp=8dcd63a0c55a86c3e4f2edc5c73fba162396e7b6;hpb=19542299d3f4095acda802b73b8a71a2f208cdf2;p=vpp.git diff --git a/src/vpp-api/python/vpp_papi/vpp_papi.py b/src/vpp-api/python/vpp_papi/vpp_papi.py index 8dcd63a0c55..05688cec731 100644 --- a/src/vpp-api/python/vpp_papi/vpp_papi.py +++ b/src/vpp-api/python/vpp_papi/vpp_papi.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (c) 2016 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,28 +16,30 @@ from __future__ import print_function from __future__ import absolute_import +import ctypes import sys +import multiprocessing as mp import os import logging -import collections -import struct import functools import json import threading import fnmatch import weakref import atexit -from . vpp_serializer import VPPType, VPPEnumType, VPPUnionType, BaseTypes +from . vpp_serializer import VPPType, VPPEnumType, VPPUnionType from . vpp_serializer import VPPMessage, vpp_get_type, VPPTypeAlias -from . macaddress import MACAddress, mac_pton, mac_ntop - -logger = logging.getLogger(__name__) if sys.version[0] == '2': import Queue as queue else: import queue as queue +__all__ = ('FuncWrapper', 'VPP', 'VppApiDynamicMethodHolder', + 'VppEnum', 'VppEnumType', + 'VPPIOError', 'VPPRuntimeError', 'VPPValueError', + 'VPPApiClient', ) + def metaclass(metaclass): @functools.wraps(metaclass) @@ -74,19 +76,6 @@ else: return d.items() -def call_logger(msgdef, kwargs): - s = 'Calling {}('.format(msgdef.name) - for k, v in kwargs.items(): - s += '{}:{} '.format(k, v) - s += ')' - return s - - -def return_logger(r): - s = 'Return from {}'.format(r) - return s - - class VppApiDynamicMethodHolder(object): pass @@ -100,6 +89,9 @@ class FuncWrapper(object): def __call__(self, **kwargs): return self._func(**kwargs) + def __repr__(self): + return ')>' % (self.__name__, self.__doc__) + class VPPApiError(Exception): pass @@ -120,29 +112,133 @@ class VPPRuntimeError(RuntimeError): class VPPValueError(ValueError): pass +class VPPApiJSONFiles(object): + @classmethod + def find_api_dir(cls, dirs): + """Attempt to find the best directory in which API definition + files may reside. If the value VPP_API_DIR exists in the environment + then it is first on the search list. If we're inside a recognized + location in a VPP source tree (src/scripts and src/vpp-api/python) + then entries from there to the likely locations in build-root are + added. Finally the location used by system packages is added. -class VPPApiClient(object): - """VPP interface. + :returns: A single directory name, or None if no such directory + could be found. + """ - This class provides the APIs to VPP. The APIs are loaded - from provided .api.json files and makes functions accordingly. - These functions are documented in the VPP .api files, as they - are dynamically created. + # perhaps we're in the 'src/scripts' or 'src/vpp-api/python' dir; + # in which case, plot a course to likely places in the src tree + import __main__ as main + if hasattr(main, '__file__'): + # get the path of the calling script + localdir = os.path.dirname(os.path.realpath(main.__file__)) + else: + # use cwd if there is no calling script + localdir = os.getcwd() + localdir_s = localdir.split(os.path.sep) - Additionally, VPP can send callback messages; this class - provides a means to register a callback function to receive - these messages in a background thread. - """ - apidir = None - VPPApiError = VPPApiError - VPPRuntimeError = VPPRuntimeError - VPPValueError = VPPValueError - VPPNotImplementedError = VPPNotImplementedError - VPPIOError = VPPIOError + def dmatch(dir): + """Match dir against right-hand components of the script dir""" + d = dir.split('/') # param 'dir' assumes a / separator + length = len(d) + return len(localdir_s) > length and localdir_s[-length:] == d + def sdir(srcdir, variant): + """Build a path from srcdir to the staged API files of + 'variant' (typically '' or '_debug')""" + # Since 'core' and 'plugin' files are staged + # in separate directories, we target the parent dir. + return os.path.sep.join(( + srcdir, + 'build-root', + 'install-vpp%s-native' % variant, + 'vpp', + 'share', + 'vpp', + 'api', + )) + + srcdir = None + if dmatch('src/scripts'): + srcdir = os.path.sep.join(localdir_s[:-2]) + elif dmatch('src/vpp-api/python'): + srcdir = os.path.sep.join(localdir_s[:-3]) + elif dmatch('test'): + # we're apparently running tests + srcdir = os.path.sep.join(localdir_s[:-1]) + + if srcdir: + # we're in the source tree, try both the debug and release + # variants. + dirs.append(sdir(srcdir, '_debug')) + dirs.append(sdir(srcdir, '')) + + # Test for staged copies of the scripts + # For these, since we explicitly know if we're running a debug versus + # release variant, target only the relevant directory + if dmatch('build-root/install-vpp_debug-native/vpp/bin'): + srcdir = os.path.sep.join(localdir_s[:-4]) + dirs.append(sdir(srcdir, '_debug')) + if dmatch('build-root/install-vpp-native/vpp/bin'): + srcdir = os.path.sep.join(localdir_s[:-4]) + dirs.append(sdir(srcdir, '')) + + # finally, try the location system packages typically install into + dirs.append(os.path.sep.join(('', 'usr', 'share', 'vpp', 'api'))) + + # check the directories for existence; first one wins + for dir in dirs: + if os.path.isdir(dir): + return dir + + return None + + @classmethod + def find_api_files(cls, api_dir=None, patterns='*'): + """Find API definition files from the given directory tree with the + given pattern. If no directory is given then find_api_dir() is used + to locate one. If no pattern is given then all definition files found + in the directory tree are used. + + :param api_dir: A directory tree in which to locate API definition + files; subdirectories are descended into. + If this is None then find_api_dir() is called to discover it. + :param patterns: A list of patterns to use in each visited directory + when looking for files. + This can be a list/tuple object or a comma-separated string of + patterns. Each value in the list will have leading/trialing + whitespace stripped. + The pattern specifies the first part of the filename, '.api.json' + is appended. + The results are de-duplicated, thus overlapping patterns are fine. + If this is None it defaults to '*' meaning "all API files". + :returns: A list of file paths for the API files found. + """ + if api_dir is None: + api_dir = cls.find_api_dir([]) + if api_dir is None: + raise VPPApiError("api_dir cannot be located") + + if isinstance(patterns, list) or isinstance(patterns, tuple): + patterns = [p.strip() + '.api.json' for p in patterns] + else: + patterns = [p.strip() + '.api.json' for p in patterns.split(",")] + + api_files = [] + for root, dirnames, files in os.walk(api_dir): + # iterate all given patterns and de-dup the result + files = set(sum([fnmatch.filter(files, p) for p in patterns], [])) + for filename in files: + api_files.append(os.path.join(root, filename)) + + return api_files + + @classmethod def process_json_file(self, apidef_file): api = json.load(apidef_file) types = {} + services = {} + messages = {} for t in api['enums']: t[0] = 'vl_api_' + t[0] + '_t' types[t[0]] = {'type': 'enum', 'data': t} @@ -154,7 +250,7 @@ class VPPApiClient(object): types[t[0]] = {'type': 'type', 'data': t} for t, v in api['aliases'].items(): types['vl_api_' + t + '_t'] = {'type': 'alias', 'data': v} - self.services.update(api['services']) + services.update(api['services']) i = 0 while True: @@ -192,14 +288,36 @@ class VPPApiClient(object): for m in api['messages']: try: - self.messages[m[0]] = VPPMessage(m[0], m[1:]) + messages[m[0]] = VPPMessage(m[0], m[1:]) except VPPNotImplementedError: + ### OLE FIXME self.logger.error('Not implemented error for {}'.format(m[0])) + return messages, services + +class VPPApiClient(object): + """VPP interface. + + This class provides the APIs to VPP. The APIs are loaded + from provided .api.json files and makes functions accordingly. + These functions are documented in the VPP .api files, as they + are dynamically created. + + Additionally, VPP can send callback messages; this class + provides a means to register a callback function to receive + these messages in a background thread. + """ + apidir = None + VPPApiError = VPPApiError + VPPRuntimeError = VPPRuntimeError + VPPValueError = VPPValueError + VPPNotImplementedError = VPPNotImplementedError + VPPIOError = VPPIOError + def __init__(self, apifiles=None, testmode=False, async_thread=True, logger=None, loglevel=None, read_timeout=5, use_socket=False, - server_address='/run/vpp-api.sock'): + server_address='/run/vpp/api.sock'): """Create a VPP API object. apifiles is a list of files containing API @@ -212,11 +330,9 @@ class VPPApiClient(object): loglevel, if supplied, is the log level this logger is set to report at (from the loglevels in the logging module). """ - if apifiles is None and self.__class__.apidir is None: - raise ValueError("Either apifiles or apidir must be specified.") - if logger is None: - logger = logging.getLogger(__name__) + logger = logging.getLogger( + "{}.{}".format(__name__, self.__class__.__name__)) if loglevel is not None: logger.setLevel(loglevel) self.logger = logger @@ -232,6 +348,11 @@ class VPPApiClient(object): self.message_queue = queue.Queue() self.read_timeout = read_timeout self.async_thread = async_thread + self.event_thread = None + self.testmode = testmode + self.use_socket = use_socket + self.server_address = server_address + self._apifiles = apifiles if use_socket: from . vpp_transport_socket import VppTransport @@ -241,7 +362,7 @@ class VPPApiClient(object): if not apifiles: # Pick up API definitions from default directory try: - apifiles = self.find_api_files() + apifiles = VPPApiJSONFiles.find_api_files(self.apidir) except RuntimeError: # In test mode we don't care that we can't find the API files if testmode: @@ -251,7 +372,9 @@ class VPPApiClient(object): for file in apifiles: with open(file) as apidef_file: - self.process_json_file(apidef_file) + m, s = VPPApiJSONFiles.process_json_file(apidef_file) + self.messages.update(m) + self.services.update(s) self.apifiles = apifiles @@ -264,143 +387,26 @@ class VPPApiClient(object): # Make sure we allow VPP to clean up the message rings. atexit.register(vpp_atexit, weakref.ref(self)) + def get_function(self, name): + return getattr(self._api, name) + + class ContextId(object): - """Thread-safe provider of unique context IDs.""" + """Multiprocessing-safe provider of unique context IDs.""" def __init__(self): - self.context = 0 - self.lock = threading.Lock() + self.context = mp.Value(ctypes.c_uint, 0) + self.lock = mp.Lock() def __call__(self): """Get a new unique (or, at least, not recently used) context.""" with self.lock: - self.context += 1 - return self.context + self.context.value += 1 + return self.context.value get_context = ContextId() def get_type(self, name): return vpp_get_type(name) - @classmethod - def find_api_dir(cls): - """Attempt to find the best directory in which API definition - files may reside. If the value VPP_API_DIR exists in the environment - then it is first on the search list. If we're inside a recognized - location in a VPP source tree (src/scripts and src/vpp-api/python) - then entries from there to the likely locations in build-root are - added. Finally the location used by system packages is added. - - :returns: A single directory name, or None if no such directory - could be found. - """ - dirs = [cls.apidir] - - # perhaps we're in the 'src/scripts' or 'src/vpp-api/python' dir; - # in which case, plot a course to likely places in the src tree - import __main__ as main - if hasattr(main, '__file__'): - # get the path of the calling script - localdir = os.path.dirname(os.path.realpath(main.__file__)) - else: - # use cwd if there is no calling script - localdir = os.getcwd() - localdir_s = localdir.split(os.path.sep) - - def dmatch(dir): - """Match dir against right-hand components of the script dir""" - d = dir.split('/') # param 'dir' assumes a / separator - length = len(d) - return len(localdir_s) > length and localdir_s[-length:] == d - - def sdir(srcdir, variant): - """Build a path from srcdir to the staged API files of - 'variant' (typically '' or '_debug')""" - # Since 'core' and 'plugin' files are staged - # in separate directories, we target the parent dir. - return os.path.sep.join(( - srcdir, - 'build-root', - 'install-vpp%s-native' % variant, - 'vpp', - 'share', - 'vpp', - 'api', - )) - - srcdir = None - if dmatch('src/scripts'): - srcdir = os.path.sep.join(localdir_s[:-2]) - elif dmatch('src/vpp-api/python'): - srcdir = os.path.sep.join(localdir_s[:-3]) - elif dmatch('test'): - # we're apparently running tests - srcdir = os.path.sep.join(localdir_s[:-1]) - - if srcdir: - # we're in the source tree, try both the debug and release - # variants. - dirs.append(sdir(srcdir, '_debug')) - dirs.append(sdir(srcdir, '')) - - # Test for staged copies of the scripts - # For these, since we explicitly know if we're running a debug versus - # release variant, target only the relevant directory - if dmatch('build-root/install-vpp_debug-native/vpp/bin'): - srcdir = os.path.sep.join(localdir_s[:-4]) - dirs.append(sdir(srcdir, '_debug')) - if dmatch('build-root/install-vpp-native/vpp/bin'): - srcdir = os.path.sep.join(localdir_s[:-4]) - dirs.append(sdir(srcdir, '')) - - # finally, try the location system packages typically install into - dirs.append(os.path.sep.join(('', 'usr', 'share', 'vpp', 'api'))) - - # check the directories for existence; first one wins - for dir in dirs: - if os.path.isdir(dir): - return dir - - return None - - @classmethod - def find_api_files(cls, api_dir=None, patterns='*'): - """Find API definition files from the given directory tree with the - given pattern. If no directory is given then find_api_dir() is used - to locate one. If no pattern is given then all definition files found - in the directory tree are used. - - :param api_dir: A directory tree in which to locate API definition - files; subdirectories are descended into. - If this is None then find_api_dir() is called to discover it. - :param patterns: A list of patterns to use in each visited directory - when looking for files. - This can be a list/tuple object or a comma-separated string of - patterns. Each value in the list will have leading/trialing - whitespace stripped. - The pattern specifies the first part of the filename, '.api.json' - is appended. - The results are de-duplicated, thus overlapping patterns are fine. - If this is None it defaults to '*' meaning "all API files". - :returns: A list of file paths for the API files found. - """ - if api_dir is None: - api_dir = cls.find_api_dir() - if api_dir is None: - raise VPPApiError("api_dir cannot be located") - - if isinstance(patterns, list) or isinstance(patterns, tuple): - patterns = [p.strip() + '.api.json' for p in patterns] - else: - patterns = [p.strip() + '.api.json' for p in patterns.split(",")] - - api_files = [] - for root, dirnames, files in os.walk(api_dir): - # iterate all given patterns and de-dup the result - files = set(sum([fnmatch.filter(files, p) for p in patterns], [])) - for filename in files: - api_files.append(os.path.join(root, filename)) - - return api_files - @property def api(self): if not hasattr(self, "_api"): @@ -429,7 +435,7 @@ class VPPApiClient(object): self._api = VppApiDynamicMethodHolder() for name, msg in vpp_iterator(self.messages): n = name + '_' + msg.crc[2:] - i = self.transport.get_msg_index(n.encode('utf-8')) + i = self.transport.get_msg_index(n) if i > 0: self.id_msgdef[i] = msg self.id_names[i] = name @@ -451,7 +457,7 @@ class VPPApiClient(object): do_async): pfx = chroot_prefix.encode('utf-8') if chroot_prefix else None - rv = self.transport.connect(name.encode('utf-8'), pfx, + rv = self.transport.connect(name, pfx, msg_handler, rx_qlen) if rv != 0: raise VPPIOError(2, 'Connect failed') @@ -461,13 +467,15 @@ class VPPApiClient(object): # Initialise control ping crc = self.messages['control_ping'].crc self.control_ping_index = self.transport.get_msg_index( - ('control_ping' + '_' + crc[2:]).encode('utf-8')) + ('control_ping' + '_' + crc[2:])) self.control_ping_msgdef = self.messages['control_ping'] if self.async_thread: self.event_thread = threading.Thread( target=self.thread_msg_handler) self.event_thread.daemon = True self.event_thread.start() + else: + self.event_thread = None return rv def connect(self, name, chroot_prefix=None, do_async=False, rx_qlen=32): @@ -498,7 +506,8 @@ class VPPApiClient(object): def disconnect(self): """Detach from VPP.""" rv = self.transport.disconnect() - self.message_queue.put("terminate event thread") + if self.event_thread is not None: + self.message_queue.put("terminate event thread") return rv def msg_handler_sync(self, msg): @@ -619,7 +628,9 @@ class VPPApiClient(object): pass self.validate_args(msgdef, kwargs) - logging.debug(call_logger(msgdef, kwargs)) + s = 'Calling {}({})'.format(msgdef.name, + ','.join(['{!r}:{!r}'.format(k, v) for k, v in kwargs.items()])) + self.logger.debug(s) b = msgdef.pack(kwargs) self.transport.suspend() @@ -634,10 +645,9 @@ class VPPApiClient(object): # Block until we get a reply. rl = [] while (True): - msg = self.transport.read() - if not msg: + r = self.read_blocking(no_type_conversion) + if r is None: raise VPPIOError(2, 'VPP API client: read failed') - r = self.decode_incoming_msg(msg, no_type_conversion) msgname = type(r).__name__ if context not in r or r.context == 0 or context != r.context: # Message being queued @@ -654,17 +664,24 @@ class VPPApiClient(object): self.transport.resume() - logger.debug(return_logger(rl)) + s = 'Return value: {!r}'.format(r) + if len(s) > 80: + s = s[:80] + "..." + self.logger.debug(s) return rl def _call_vpp_async(self, i, msg, **kwargs): - """Given a message, send the message and await a reply. + """Given a message, send the message and return the context. msgdef - the message packing definition i - the message type index context - context number - chosen at random if not supplied. The remainder of the kwargs are the arguments to the API call. + + The reply message(s) will be delivered later to the registered callback. + The returned context will help with assigning which call + the reply belongs to. """ if 'context' not in kwargs: context = self.get_context() @@ -680,6 +697,34 @@ class VPPApiClient(object): b = msg.pack(kwargs) self.transport.write(b) + return context + + def read_blocking(self, no_type_conversion=False): + """Get next received message from transport within timeout, decoded. + + Note that noticifations have context zero + and are not put into receive queue (at least for socket transport), + use async_thread with registered callback for processing them. + + If no message appears in the queue within timeout, return None. + + Optionally, type conversion can be skipped, + as some of conversions are into less precise types. + + When r is the return value of this, the caller can get message name as: + msgname = type(r).__name__ + and context number (type long) as: + context = r.context + + :param no_type_conversion: If false, type conversions are applied. + :type no_type_conversion: bool + :returns: Decoded message, or None if no message (within timeout). + :rtype: Whatever VPPType.unpack returns, depends on no_type_conversion. + """ + msg = self.transport.read() + if not msg: + return None + return self.decode_incoming_msg(msg, no_type_conversion) def register_event_callback(self, callback): """Register a callback for async messages. @@ -714,6 +759,15 @@ class VPPApiClient(object): if self.event_callback: self.event_callback(msgname, r) + def __repr__(self): + return "" % ( + self._apifiles, self.testmode, self.async_thread, + self.logger, self.read_timeout, self.use_socket, + self.server_address) + + # Provide the old name for backward compatibility. VPP = VPPApiClient