From 5f9dcff39d5e25c6bef30d569e405635633f3c69 Mon Sep 17 00:00:00 2001 From: Ole Troan Date: Mon, 1 Aug 2016 04:59:13 +0200 Subject: [PATCH] VPP Python language binding - plugin support - Moved Python generator tool to tools directory - Added build-vpp-api Makefile target - Generator now only creates a Python representation of the .api the rest of the framework is in the vpp_papi script - Each plugin has its own namespace. - Plugin Python files are installed in vpp_papi_plugins for easy use inside the build tree. Change-Id: I272c83bb7e5d5e416bdbd8a790a3cc35c5a04e38 Signed-off-by: Ole Troan --- .gitignore | 4 +- Makefile | 6 +- build-data/platforms.mk | 18 ++- plugins/ioam-plugin/Makefile.am | 11 +- plugins/snat-plugin/Makefile.am | 11 +- vlib-api/Makefile.am | 4 + vpp-api/python/Makefile.am | 45 ++++-- vpp-api/python/pneum/pneum.c | 20 +-- vpp-api/python/pneum/pneum.h | 5 +- vpp-api/python/setup.cfg | 7 + vpp-api/python/setup.py | 19 +-- vpp-api/python/tests/test_base.py | 7 + vpp-api/python/tests/test_modules.py | 17 ++ vpp-api/python/tests/test_papi.py | 158 +++++++++---------- vpp-api/python/vpp_papi/__init__.py | 4 +- vpp-api/python/vpp_papi/pneum_wrap.c | 29 +++- vpp-api/python/vpp_papi/vpp_api_base.py | 97 ++++++++++++ vpp-api/python/vpp_papi/vpp_papi.py | 155 ++++++++++++++++++ vpp/vpp-api/api.c | 12 +- vppapigen/Makefile.am | 1 + vppapigen/lex.c | 8 +- vppapigen/node.c | 5 + vppapigen/pyvppapigen.py | 271 ++++++++++++++++++++++++++++++++ 23 files changed, 757 insertions(+), 157 deletions(-) create mode 100644 vpp-api/python/setup.cfg create mode 100644 vpp-api/python/tests/test_base.py create mode 100755 vpp-api/python/tests/test_modules.py create mode 100644 vpp-api/python/vpp_papi/vpp_api_base.py create mode 100644 vpp-api/python/vpp_papi/vpp_papi.py create mode 100755 vppapigen/pyvppapigen.py diff --git a/.gitignore b/.gitignore index 425261836aa..8cbf1e61e0e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ /build-root/*.rpm /build-root/*.changes /build-config.mk -/vpp-api/python/vpp_papi/vpp_papi.py /dpdk/*.tar.gz /dpdk/*.tar.xz /path_setup @@ -51,7 +50,7 @@ test-driver .settings # stop autotools ignore -# OSX and some IDE +# OSX and some IDE .DS_Store .idea/ .project @@ -63,6 +62,7 @@ test-driver GPATH GRTAGS GTAGS +TAGS # Generated documentation /build-root/docs diff --git a/Makefile b/Makefile index 4e3d65bcd02..39930651289 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ endif .PHONY: help bootstrap wipe wipe-release build build-release rebuild rebuild-release .PHONY: run run-release debug debug-release build-vat run-vat pkg-deb pkg-rpm -.PHONY: ctags cscope doxygen wipe-doxygen plugins plugins-release +.PHONY: ctags cscope doxygen wipe-doxygen plugins plugins-release build-vpp-api help: @echo "Make Targets:" @@ -73,6 +73,7 @@ help: @echo " debug - run debug binary with debugger" @echo " debug-release - run release binary with debugger" @echo " build-vat - build vpp-api-test tool" + @echo " build-vpp-api - build vpp-api" @echo " run-vat - run vpp-api-test tool" @echo " pkg-deb - build DEB packages" @echo " pkg-rpm - build RPM packages" @@ -172,6 +173,9 @@ plugins: $(BR)/.bootstrap.ok plugins-release: $(BR)/.bootstrap.ok $(call make,$(PLATFORM),plugins-install) +build-vpp-api: $(BR)/.bootstrap.ok + $(call make,$(PLATFORM)_debug,vpp-api-install) + STARTUP_DIR ?= $(PWD) ifeq ("$(wildcard $(STARTUP_CONF))","") define run diff --git a/build-data/platforms.mk b/build-data/platforms.mk index cd65f67c057..36cfc878b37 100644 --- a/build-data/platforms.mk +++ b/build-data/platforms.mk @@ -56,14 +56,16 @@ install-deb: $(patsubst %,%-find-source,$(ROOT_PACKAGES)) >> deb/debian/vpp.install ; \ \ : dev package needs a couple of additions ; \ - echo ../build-tool-native/vppapigen/vppapigen /usr/bin \ - >> deb/debian/vpp-dev.install ; \ - echo ../../vpp-api/java/jvpp/gen/jvpp_gen.py /usr/bin \ - >> deb/debian/vpp-dev.install ; \ - for i in $$(ls ../vpp-api/java/jvpp/gen/jvppgen/*.py); do \ - echo ../$${i} /usr/lib/python2.7/dist-packages/jvppgen \ - >> deb/debian/vpp-dev.install; \ - done; \ + echo ../build-tool-native/vppapigen/vppapigen /usr/bin \ + >> deb/debian/vpp-dev.install ; \ + echo ../../vppapigen/pyvppapigen.py /usr/bin \ + >> deb/debian/vpp-dev.install ; \ + echo ../../vpp-api/java/jvpp/gen/jvpp_gen.py /usr/bin \ + >> deb/debian/vpp-dev.install ; \ + for i in $$(ls ../vpp-api/java/jvpp/gen/jvppgen/*.py); do \ + echo ../$${i} /usr/lib/python2.7/dist-packages/jvppgen \ + >> deb/debian/vpp-dev.install; \ + done; \ \ : generate changelog; \ ./scripts/generate-deb-changelog \ diff --git a/plugins/ioam-plugin/Makefile.am b/plugins/ioam-plugin/Makefile.am index 2ea29e006d9..68e792482a9 100644 --- a/plugins/ioam-plugin/Makefile.am +++ b/plugins/ioam-plugin/Makefile.am @@ -28,7 +28,7 @@ ioam_pot_plugin_la_SOURCES = \ ioam/lib-pot/pot_api.c BUILT_SOURCES = \ - ioam/lib-pot/pot.api.h + ioam/lib-pot/pot.api.h ioam/lib-pot/pot.py SUFFIXES = .api.h .api @@ -37,6 +37,15 @@ SUFFIXES = .api.h .api $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ | vppapigen --input - --output $@ --show-name $@ +%.py: %.api + $(info Creating Python binding for $@) + $(CC) $(CPPFLAGS) -E -P -C -x c $< \ + | vppapigen --input - --python - \ + | pyvppapigen.py --input - > $@ + +pyapidir = ${prefix}/vpp_papi_plugins +pyapi_DATA = ioam/lib-pot/pot.py + noinst_HEADERS = \ ioam/lib-pot/pot_all_api_h.h \ ioam/lib-pot/pot_msg_enum.h \ diff --git a/plugins/snat-plugin/Makefile.am b/plugins/snat-plugin/Makefile.am index 0fe694cf7bd..91fec414363 100644 --- a/plugins/snat-plugin/Makefile.am +++ b/plugins/snat-plugin/Makefile.am @@ -28,7 +28,7 @@ snat_plugin_la_SOURCES = snat/snat.c \ snat/out2in.c \ snat/snat_plugin.api.h -BUILT_SOURCES = snat/snat.api.h +BUILT_SOURCES = snat/snat.api.h snat/snat.py SUFFIXES = .api.h .api @@ -37,6 +37,15 @@ SUFFIXES = .api.h .api $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ | vppapigen --input - --output $@ --show-name $@ +%.py: %.api + $(info Creating Python binding for $@) + $(CC) $(CPPFLAGS) -E -P -C -x c $< \ + | vppapigen --input - --python - \ + | pyvppapigen.py --input - > $@ + +pyapidir = ${prefix}/vpp_papi_plugins +pyapi_DATA = snat/snat.py + noinst_HEADERS = \ snat/snat_all_api_h.h \ snat/snat_msg_enum.h \ diff --git a/vlib-api/Makefile.am b/vlib-api/Makefile.am index 4b0129a3a5f..5bc00e7479b 100644 --- a/vlib-api/Makefile.am +++ b/vlib-api/Makefile.am @@ -75,3 +75,7 @@ SUFFIXES = .api.h .api mkdir -p `dirname $@` ; \ $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ | vppapigen --input - --output $@ --show-name $@ + +# install the API definition, so we can produce java bindings, etc. +apidir = $(prefix)/vlibmemory +api_DATA = vlibmemory/memclnt.api diff --git a/vpp-api/python/Makefile.am b/vpp-api/python/Makefile.am index 4d2d221d231..eb589335b05 100644 --- a/vpp-api/python/Makefile.am +++ b/vpp-api/python/Makefile.am @@ -13,33 +13,50 @@ AUTOMAKE_OPTIONS = foreign subdir-objects ACLOCAL_AMFLAGS = -I m4 -AM_CFLAGS = -Wall +AM_CFLAGS = -Wall BUILT_SOURCES = -bin_PROGRAMS = -CLEANFILES = -lib_LTLIBRARIES = +bin_PROGRAMS = +CLEANFILES = +lib_LTLIBRARIES = noinst_PROGRAMS = test_pneum nobase_include_HEADERS = pneum/pneum.h # -# Python binding +# Python / C extension # +lib_LTLIBRARIES += vpp_api.la +vpp_api_la_SOURCES = pneum/pneum.c vpp_papi/pneum_wrap.c +vpp_api_la_LIBADD = -lvlibmemoryclient -lvlibapi -lsvm -lvppinfra -lpthread -lm -lrt +vpp_api_la_LDFLAGS = -module $(shell python-config --ldflags) +vpp_api_la_CPPFLAGS = $(shell python-config --includes) + +# Kept around for setuptools based install. lib_LTLIBRARIES += libpneum.la -libpneum_la_SOURCES = pneum/pneum.c +libpneum_la_SOURCES = pneum/pneum.c setup.py libpneum_la_LIBADD = -lvlibmemoryclient -lvlibapi -lsvm -lvppinfra -lpthread -lm -lrt libpneum_la_LDFLAGS = -module libpneum_la_CPPFLAGS = -BUILT_SOURCES += vpp_papi.py +# +# Core VPP API +# +BUILT_SOURCES += \ + $(prefix)/../vpp/vpp-api/vpe.py \ + $(prefix)/../vlib-api/vlibmemory/memclnt.py -vpp_papi.py: $(prefix)/../vpp/vpp-api/vpe.api pneum/api-gen.py - @echo " PYTHON API"; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $< \ - | vppapigen --input - --python defs_$@; \ - echo "#include " \ - | $(CC) $(CPPFLAGS) -E -P -x c - | grep VL_API \ - | @srcdir@/pneum/api-gen.py -i defs_$@ > @srcdir@/vpp_papi/$@ +%.py: %.api + $(info Creating Python binding for $@) + $(CC) $(CPPFLAGS) -E -P -C -x c $< \ + | vppapigen --input - --python - \ + | pyvppapigen.py --input - > $@ + +# +# TODO: Support both Python 2 and 3. +install-exec-local: + cd $(srcdir); \ + mkdir -p $(prefix)/lib/python2.7/site-packages; \ + PYTHONUSERBASE=$(prefix) python setup.py install --user # # Test client diff --git a/vpp-api/python/pneum/pneum.c b/vpp-api/python/pneum/pneum.c index ac518493163..2637d43f5d9 100644 --- a/vpp-api/python/pneum/pneum.c +++ b/vpp-api/python/pneum/pneum.c @@ -10,7 +10,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include #include @@ -36,11 +36,11 @@ #include "pneum.h" #define vl_typedefs /* define message structures */ -#include +#include #undef vl_typedefs #define vl_endianfun /* define message structures */ -#include +#include #undef vl_endianfun typedef struct { @@ -54,12 +54,12 @@ pneum_main_t pneum_main; extern int wrap_pneum_callback(char *data, int len); -/* +/* * Satisfy external references when -lvlib is not available. */ void vlib_cli_output (struct vlib_main_t * vm, char * fmt, ...) { - clib_warning ("vlib_cli_output callled..."); + clib_warning ("vlib_cli_output called..."); } #define vl_api_version(n,v) static u32 vpe_api_version = v; @@ -89,7 +89,7 @@ pneum_api_handler (void *msg) int l = ntohl(msgbuf->data_len); if (l == 0) clib_warning("Message ID %d has wrong length: %d\n", id, l); - + /* Call Python callback */ (void)wrap_pneum_callback(msg, l); vl_msg_api_free(msg); @@ -121,12 +121,6 @@ pneum_connect (char *name) int rv = 0; pneum_main_t *pm = &pneum_main; - /* - * Bail out now if we're not running as root - */ - if (geteuid() != 0) - return (-1); - if ((rv = vl_client_api_map("/vpe-api"))) { clib_warning ("vl_client_api map rv %d", rv); return rv; @@ -199,7 +193,7 @@ pneum_read (char **p, int *l) *p = (char *)msg; } else { printf("Read failed with %d\n", rv); - } + } return (rv); } diff --git a/vpp-api/python/pneum/pneum.h b/vpp-api/python/pneum/pneum.h index b99cbd4e3ec..75fccf84bcf 100644 --- a/vpp-api/python/pneum/pneum.h +++ b/vpp-api/python/pneum/pneum.h @@ -12,10 +12,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __included_pneum_h__ -#define __included_pneum_h__ +#ifndef included_pneum_h +#define included_pneum_h -unsigned int vpe_client_index(void); int pneum_connect(char *name); int pneum_disconnect(void); int pneum_read(char **data, int *l); diff --git a/vpp-api/python/setup.cfg b/vpp-api/python/setup.cfg new file mode 100644 index 00000000000..5e19e8c5e74 --- /dev/null +++ b/vpp-api/python/setup.cfg @@ -0,0 +1,7 @@ +[bdist_wheel] +# This flag says that the code is written to work on both Python 2 and Python +# 3. If at all possible, it is good practice to do this. If you cannot, you +# will need to generate wheels for each Python version that you support. +universal=0 + + diff --git a/vpp-api/python/setup.py b/vpp-api/python/setup.py index d890ba709dc..e369a0cb479 100644 --- a/vpp-api/python/setup.py +++ b/vpp-api/python/setup.py @@ -1,21 +1,16 @@ -from distutils.core import setup, Extension - -module1 = Extension('vpp_api', - define_macros = [('MAJOR_VERSION', '1'), - ('MINOR_VERSION', '0')], - include_dirs = ['pneum'], - libraries = ['pneum'], - library_dirs = ['../../build-root/install-vpp_debug-native/vpp-api/lib64'], - sources = ['vpp_papi/pneum_wrap.c']) +try: + from setuptools import setup +except ImportError: + from distutils.core import setup setup (name = 'vpp_papi', - version = '1.0', + version = '1.1', description = 'VPP Python binding', author = 'Ole Troan', author_email = 'ot@cisco.com', #url = 'https://docs.python.org/extending/building', + test_suite = 'tests', packages=['vpp_papi'], long_description = ''' VPP Python language binding. -''', - ext_modules = [module1]) +''',) diff --git a/vpp-api/python/tests/test_base.py b/vpp-api/python/tests/test_base.py new file mode 100644 index 00000000000..8ff5dd4782c --- /dev/null +++ b/vpp-api/python/tests/test_base.py @@ -0,0 +1,7 @@ +# Manipulate sys.path to allow tests be run inside the build environment. +import os, sys, glob +scriptdir = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.dirname(glob.glob(scriptdir+'/../../../build-root/install*/vpp-api/lib64/vpp_api.so')[0])) +sys.path.append(os.path.dirname(glob.glob(scriptdir+'/../../../build-root/install*/vlib-api/vlibmemory/memclnt.py')[0])) +sys.path.append(os.path.dirname(glob.glob(scriptdir+'/../../../build-root/install*/vpp/vpp-api/vpe.py')[0])) +sys.path.append(glob.glob(scriptdir+'/../../../build-root/install*/plugins/vpp_papi_plugins')[0]) diff --git a/vpp-api/python/tests/test_modules.py b/vpp-api/python/tests/test_modules.py new file mode 100755 index 00000000000..f3066b29395 --- /dev/null +++ b/vpp-api/python/tests/test_modules.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +from __future__ import print_function +import unittest +import test_base +import vpp_papi +import pot, snat +print('Plugins:') +vpp_papi.plugin_show() +r = vpp_papi.connect('ole') + +r = vpp_papi.show_version() +print('R:', r) + +r = snat.snat_interface_add_del_feature(1, 1, 1) +print('R:', r) + +vpp_papi.disconnect() diff --git a/vpp-api/python/tests/test_papi.py b/vpp-api/python/tests/test_papi.py index bede7171519..ab90eeaa45c 100755 --- a/vpp-api/python/tests/test_papi.py +++ b/vpp-api/python/tests/test_papi.py @@ -1,102 +1,104 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python -import vpp_papi +from __future__ import print_function import unittest, sys, time, threading, struct, logging +import test_base +import vpp_papi from ipaddress import * papi_event = threading.Event() +print(vpp_papi.VL_API_SW_INTERFACE_SET_FLAGS) def papi_event_handler(result): - if result.vl_msg_id == vpp_papi.VL_API_SW_INTERFACE_SET_FLAGS: - papi_event.set() + if result.vl_msg_id == vpp_papi.vpe.VL_API_SW_INTERFACE_SET_FLAGS: return - if result.vl_msg_id == vpp_papi.VL_API_VNET_INTERFACE_COUNTERS: - format = '>' + str(int(len(result.data) / 8)) + 'Q' - counters = struct.unpack(format, result.data) - print('Counters:', counters) + if result.vl_msg_id == vpp_papi.vpe.VL_API_VNET_INTERFACE_COUNTERS: + print('Interface counters', result) return - if result.vl_msg_id == vpp_papi.VL_API_VNET_IP6_FIB_COUNTERS: - print('IP6 FIB Counters:', result.count, len(result.c), len(result)) - i = 0 - # FIB counters allocate a large (1000 bytes) block so message length does not match reality - for c in struct.iter_unpack('>16sBQQ', result.c): - # In Python 3.5 we can use a tuple for prefix, length - print(str(IPv6Address(c[0])) + '/' + str(c[1]), str(c[2]), str(c[3])) - i += 1 - if i >= result.count: - break + if result.vl_msg_id == vpp_papi.vpe.VL_API_VNET_IP6_FIB_COUNTERS: + print('IPv6 FIB counters', result) + papi_event.set() return print('Unknown message id:', result.vl_msg_id) +import glob, subprocess class TestPAPI(unittest.TestCase): + @classmethod + def setUpClass(cls): + # + # Start main VPP process + cls.vpp_bin = glob.glob(test_base.scriptdir+'/../../../build-root/install-vpp*-native/vpp/bin/vpp')[0] + print("VPP BIN:", cls.vpp_bin) + cls.vpp = subprocess.Popen([cls.vpp_bin, "unix", "nodaemon"], stderr=subprocess.PIPE) + print('Started VPP') + # For some reason unless we let VPP start up the API cannot connect. + time.sleep(0.3) + @classmethod + def tearDownClass(cls): + cls.vpp.terminate() def setUp(self): + print("Connecting API") r = vpp_papi.connect("test_papi") self.assertEqual(r, 0) def tearDown(self): r = vpp_papi.disconnect() self.assertEqual(r, 0) - + + # + # The tests themselves + # + + # + # Basic request / reply + # def test_show_version(self): t = vpp_papi.show_version() + print('T', t); program = t.program.decode().rstrip('\x00') self.assertEqual('vpe', program) # - # Add a few MAP domains, then dump them later + # Details / Dump # - def test_map(self): - t = vpp_papi.map_summary_stats() - print(t) - ip6 = IPv6Address(u'2001:db8::1').packed - ip4 = IPv4Address(u'10.0.0.0').packed - ip6_src = IPv6Address(u'2001:db9::1').packed - t = vpp_papi.map_add_domain(ip6, ip4, ip6_src, 32, 24, 128, 0, 0, 6, 0, 0) - print(t) - self.assertEqual(t.retval, 0) + def test_details_dump(self): + t = vpp_papi.sw_interface_dump(0, b'') + print('Dump/details T', t) - ip4 = IPv4Address(u'10.0.1.0').packed - t = vpp_papi.map_add_domain(ip6, ip4, ip6_src, 32, 24, 128, 0, 0, 6, 0, 0) + # + # Arrays + # + def test_arrays(self): + t = vpp_papi.vnet_get_summary_stats() + print('Summary stats', t) + print('Packets:', t.total_pkts[0]) + print('Packets:', t.total_pkts[1]) + # + # Variable sized arrays and counters + # + #@unittest.skip("stats") + def test_want_stats(self): + pid = 123 + vpp_papi.register_event_callback(papi_event_handler) + papi_event.clear() + + # Need to configure IPv6 to get som IPv6 FIB stats + t = vpp_papi.create_loopback('') print(t) self.assertEqual(t.retval, 0) - t = vpp_papi.map_summary_stats() + ifindex = t.sw_if_index + addr = str(IPv6Address('1::1').packed) + t = vpp_papi.sw_interface_add_del_address(ifindex, 1, 1, 0, 16, addr) print(t) - self.assertEqual(t.total_bindings, 2) - - t = vpp_papi.map_domain_dump() - print (t) - self.assertEqual(len(t), 2) - - def test_sw_interface_dump(self): - # - # Dump interfaces - # - t = vpp_papi.sw_interface_dump(0, b'ignored') - for interface in t: - if interface.vl_msg_id == vpp_papi.VL_API_SW_INTERFACE_DETAILS: - print(interface.interface_name.decode()) + self.assertEqual(t.retval, 0) - def test_want_interface_events(self): - pid = 123 - vpp_papi.register_event_callback(papi_event_handler) - papi_event.clear() - t = vpp_papi.want_interface_events(True, pid) - print (t) - print('Setting interface up') - t = vpp_papi.sw_interface_set_flags(0, 1, 1, 0) - print (t) - self.assertEqual(papi_event.wait(5), True) - t = vpp_papi.sw_interface_set_flags(0, 0, 0, 0) - print (t) - self.assertEqual(papi_event.wait(5), True) + # Check if interface is up + # XXX: Add new API to query interface state based on ifindex, instead of dump all. + t = vpp_papi.sw_interface_set_flags(ifindex, 1, 1, 0) + self.assertEqual(t.retval, 0) - @unittest.skip("not quite ready yet") - def test_want_stats(self): - pid = 123 - vpp_papi.register_event_callback(papi_event_handler) - papi_event.clear() t = vpp_papi.want_stats(True, pid) print (t) @@ -104,33 +106,17 @@ class TestPAPI(unittest.TestCase): # # Wait for some stats # - self.assertEqual(papi_event.wait(30), True) + self.assertEqual(papi_event.wait(15), True) t = vpp_papi.want_stats(False, pid) print (t) - def test_tap(self): - pid = 123 - vpp_papi.register_event_callback(papi_event_handler) - papi_event.clear() - t = vpp_papi.want_stats(True, pid) - - print (t) - - t = vpp_papi.tap_connect(1, b'tap', b'foo', 1, 0) - print (t) - self.assertEqual(t.retval, 0) - swifindex = t.sw_if_index - - t = vpp_papi.sw_interface_set_flags(swifindex, 1, 1, 0) - print (t) - self.assertEqual(t.retval, 0) - - ip6 = IPv6Address(u'2001:db8::1').packed - t = vpp_papi.sw_interface_add_del_address(swifindex, 1, 1, 0, 16, ip6) - print (t) - time.sleep(40) + # + # Plugins? + # if __name__ == '__main__': #logging.basicConfig(level=logging.DEBUG) unittest.main() +def test_papi(): + print('test') diff --git a/vpp-api/python/vpp_papi/__init__.py b/vpp-api/python/vpp_papi/__init__.py index 8be644d7cb7..19d78a3a05d 100644 --- a/vpp-api/python/vpp_papi/__init__.py +++ b/vpp-api/python/vpp_papi/__init__.py @@ -1,2 +1,4 @@ __import__('pkg_resources').declare_namespace(__name__) -from .vpp_papi import * +from . vpp_papi import * + + diff --git a/vpp-api/python/vpp_papi/pneum_wrap.c b/vpp-api/python/vpp_papi/pneum_wrap.c index d1795aa13a7..7a5119746be 100644 --- a/vpp-api/python/vpp_papi/pneum_wrap.c +++ b/vpp-api/python/vpp_papi/pneum_wrap.c @@ -1,5 +1,20 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include -#include "pneum.h" +#include "../pneum/pneum.h" static PyObject *pneum_callback = NULL; @@ -35,7 +50,7 @@ wrap_connect (PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "sO:set_callback", &name, &temp)) return (NULL); - + if (!PyCallable_Check(temp)) { PyErr_SetString(PyExc_TypeError, "parameter must be callable"); return NULL; @@ -66,8 +81,8 @@ wrap_write (PyObject *self, PyObject *args) char *data; int len, rv; - if (!PyArg_ParseTuple(args, "s#", &data, &len)) - return NULL; + if (!PyArg_ParseTuple(args, "s#", &data, &len)) + return NULL; Py_BEGIN_ALLOW_THREADS rv = pneum_write(data, len); Py_END_ALLOW_THREADS @@ -117,16 +132,16 @@ initvpp_api (void) { #if PY_VERSION_HEX >= 0x03000000 static struct PyModuleDef vpp_api_module = { -# if PY_VERSION_HEX >= 0x03020000 +#if PY_VERSION_HEX >= 0x03020000 PyModuleDef_HEAD_INIT, -# else +#else { PyObject_HEAD_INIT(NULL) NULL, /* m_init */ 0, /* m_index */ NULL, /* m_copy */ }, -# endif +#endif (char *) "vpp_api", NULL, -1, diff --git a/vpp-api/python/vpp_papi/vpp_api_base.py b/vpp-api/python/vpp_papi/vpp_api_base.py new file mode 100644 index 00000000000..a1ef87a3f0a --- /dev/null +++ b/vpp-api/python/vpp_papi/vpp_api_base.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Module storing all global variables, shared between main module and plugins +# +import threading + +# +# Global variables +# +results = {} +waiting_for_reply = False +plugins = {} + +class ContextId(object): + def __init__(self): + self.context = 0 + def __call__(self, id): + self.context += 1 + return self.context +get_context = ContextId() + +def waiting_for_reply_clear(): + global waiting_for_reply + waiting_for_reply = False + +def waiting_for_reply_set(): + global waiting_for_reply + waiting_for_reply = True + +def is_waiting_for_reply(): + return waiting_for_reply + +def event_callback_set(callback): + global event_callback + event_callback = callback + +def event_callback_call(r): + global event_callback + event_callback(r) + +def results_event_set(context): + results[context]['e'].set() + +def results_event_clear(context): + results[context]['e'].clear() + +def results_event_wait(context, timeout): + results[context]['e'].wait(timeout) + +def results_set(context, r): + results[context]['r'] = r + +def results_append(context, r): + results[context]['r'].append(r) + +def is_results_context(context): + return context in results + +def is_results_more(context): + return 'm' in results[context] + +def results_more_set(context): + results[context]['m'] = True + +def results_prepare(context): + results[context] = {} + results[context]['e'] = threading.Event() + results[context]['e'].clear() + results[context]['r'] = [] + +def results_get(context): + return results[context]['r'] + +def plugin_register(name, func_table, name_to_id_table, version, msg_id_base_set): + plugins[name] = {} + p = plugins[name] + p['func_table'] = func_table + p['name_to_id_table'] = name_to_id_table + p['version'] = version + p['msg_id_base_set'] = msg_id_base_set + +def plugin_show(): + for p in plugins: + print(p) diff --git a/vpp-api/python/vpp_papi/vpp_papi.py b/vpp-api/python/vpp_papi/vpp_papi.py new file mode 100644 index 00000000000..6a7a358f6cd --- /dev/null +++ b/vpp-api/python/vpp_papi/vpp_papi.py @@ -0,0 +1,155 @@ +# +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Import C API shared object +# +from __future__ import print_function + +import signal, logging, os, sys +from struct import * + +scriptdir = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(scriptdir) +import vpp_api +from vpp_api_base import * + +# Import API definitions. The core VPE API is imported into main namespace +import memclnt +from vpe import * +vpe = sys.modules['vpe'] + +def msg_handler(msg): + if not msg: + logging.warning('vpp_api.read failed') + return + + id = unpack('>H', msg[0:2]) + logging.debug('Received message', id[0]) + if id[0] == memclnt.VL_API_RX_THREAD_EXIT: + logging.info("We got told to leave") + return; + + # + # Decode message and returns a tuple. + # + logging.debug('api_func', api_func_table[id[0]]) + r = api_func_table[id[0]](msg) + if not r: + logging.warning('Message decode failed', id[0]) + return + + if 'context' in r._asdict(): + if r.context > 0: + context = r.context + + # + # XXX: Call provided callback for event + # Are we guaranteed to not get an event during processing of other messages? + # How to differentiate what's a callback message and what not? Context = 0? + # + if not is_waiting_for_reply(): + event_callback_call(r) + return + + # + # Collect results until control ping + # + if id[0] == vpe.VL_API_CONTROL_PING_REPLY: + results_event_set(context) + waiting_for_reply_clear() + return + if not is_results_context(context): + logging.warning('Not expecting results for this context', context) + return + if is_results_more(context): + results_append(context, r) + return + + results_set(context, r) + results_event_set(context) + waiting_for_reply_clear() + +def connect(name): + signal.alarm(3) # 3 second + rv = vpp_api.connect(name, msg_handler) + signal.alarm(0) + logging.info("Connect:", rv) + + # + # Assign message id space for plugins + # + plugin_map_plugins() + + return rv + +def disconnect(): + rv = vpp_api.disconnect() + logging.info("Disconnected") + return rv + +def register_event_callback(callback): + event_callback_set(callback) + +def plugin_name_to_id(plugin, name_to_id_table, base): + try: + m = globals()[plugin] + except KeyError: + m = sys.modules[plugin] + for name in name_to_id_table: + setattr(m, name, name_to_id_table[name] + base) + +def plugin_map_plugins(): + for p in plugins: + if p == 'memclnt' or p == 'vpe': + continue + + # + # Find base + # Update api table + # + version = plugins[p]['version'] + name = p + '_' + format(version, '08x') + r = memclnt.get_first_msg_id(name.encode('ascii')) + + ## TODO: Add error handling + if r.retval != 0: + print('Failed getting first msg id for:', p) + continue + + # Set base + base = r.first_msg_id + msg_id_base_set = plugins[p]['msg_id_base_set'] + msg_id_base_set(base) + plugins[p]['base'] = base + func_table = plugins[p]['func_table'] + i = r.first_msg_id + for entry in func_table: + api_func_table.insert(i, entry) + i += 1 + plugin_name_to_id(p, plugins[p]['name_to_id_table'], base) + +# +# Set up core API +# +memclnt.msg_id_base_set(1) +plugins['memclnt']['base'] = 1 +msg_id_base_set(len(plugins['memclnt']['func_table']) + 1) +plugins['vpe']['base'] = len(plugins['memclnt']['func_table']) + 1 +api_func_table = [] +api_func_table.append(None) +api_func_table[1:] = plugins['memclnt']['func_table'] + plugins['vpe']['func_table'] +plugin_name_to_id('memclnt', plugins['memclnt']['name_to_id_table'], 1) +plugin_name_to_id('vpe', plugins['vpe']['name_to_id_table'], plugins['vpe']['base']) +#logging.basicConfig(level=logging.DEBUG) diff --git a/vpp/vpp-api/api.c b/vpp/vpp-api/api.c index 1de1b55f722..2ba5ee4045f 100644 --- a/vpp/vpp-api/api.c +++ b/vpp/vpp-api/api.c @@ -2887,12 +2887,12 @@ vl_api_vnet_get_summary_stats_t_handler (vl_api_vnet_get_summary_stats_t * mp) } vnet_interface_counter_unlock (im); - /* Note: in HOST byte order! */ - rmp->total_pkts[VLIB_RX] = total_pkts[VLIB_RX]; - rmp->total_bytes[VLIB_RX] = total_bytes[VLIB_RX]; - rmp->total_pkts[VLIB_TX] = total_pkts[VLIB_TX]; - rmp->total_bytes[VLIB_TX] = total_bytes[VLIB_TX]; - rmp->vector_rate = vlib_last_vector_length_per_node (sm->vlib_main); + rmp->total_pkts[VLIB_RX] = clib_host_to_net_u64 (total_pkts[VLIB_RX]); + rmp->total_bytes[VLIB_RX] = clib_host_to_net_u64 (total_bytes[VLIB_RX]); + rmp->total_pkts[VLIB_TX] = clib_host_to_net_u64 (total_pkts[VLIB_TX]); + rmp->total_bytes[VLIB_TX] = clib_host_to_net_u64 (total_bytes[VLIB_TX]); + rmp->vector_rate = + clib_host_to_net_u64 (vlib_last_vector_length_per_node (sm->vlib_main)); vl_msg_api_send_shmem (q, (u8 *) & rmp); } diff --git a/vppapigen/Makefile.am b/vppapigen/Makefile.am index 42530015ea9..066e1c30d27 100644 --- a/vppapigen/Makefile.am +++ b/vppapigen/Makefile.am @@ -14,6 +14,7 @@ AUTOMAKE_OPTIONS = foreign bin_PROGRAMS = vppapigen +bin_SCRIPTS = pyvppapigen.py BUILT_SOURCES = gram.h diff --git a/vppapigen/lex.c b/vppapigen/lex.c index 88744ff1029..b011044dd01 100644 --- a/vppapigen/lex.c +++ b/vppapigen/lex.c @@ -331,13 +331,17 @@ int main (int argc, char **argv) if (!strncmp (argv [curarg], "--python", 8)) { curarg++; if (curarg < argc) { - pythonfp = fopen (argv[curarg], "w"); + if (!strcmp(argv[curarg], "-")) { + pythonfp = stdout; + } else { + pythonfp = fopen(argv[curarg], "w"); + pythonfile = argv[curarg]; + } if (pythonfp == NULL) { fprintf (stderr, "Couldn't open python output file %s\n", argv[curarg]); exit (1); } - pythonfile = argv[curarg]; curarg++; } else { fprintf(stderr, "Missing filename after --python\n"); diff --git a/vppapigen/node.c b/vppapigen/node.c index e66fdce846a..420a128c00e 100644 --- a/vppapigen/node.c +++ b/vppapigen/node.c @@ -1339,6 +1339,11 @@ void generate_python (YYSTYPE a1, FILE *fp) np = np->peer; } fprintf (fp, "\n]\n"); + + /* + * API CRC signature + */ + fprintf (fp, "vl_api_version = 0x%08x\n\n", (unsigned int)input_crc); } void generate(YYSTYPE a1) diff --git a/vppapigen/pyvppapigen.py b/vppapigen/pyvppapigen.py new file mode 100755 index 00000000000..e216169de3b --- /dev/null +++ b/vppapigen/pyvppapigen.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function +import argparse, sys, os, importlib, pprint + +parser = argparse.ArgumentParser(description='VPP Python API generator') +parser.add_argument('-i', '--input', action="store", dest="inputfile", type=argparse.FileType('r')) +parser.add_argument('-c', '--cfile', action="store") +args = parser.parse_args() + +# +# Read API definitions file into vppapidefs +# +exec(args.inputfile.read()) + +# https://docs.python.org/3/library/struct.html +format_struct = {'u8': 'B', + 'u16' : 'H', + 'u32' : 'I', + 'i32' : 'i', + 'u64' : 'Q', + 'f64' : 'd', + 'vl_api_ip4_fib_counter_t' : 'IBQQ', + 'vl_api_ip6_fib_counter_t' : 'QQBQQ', + }; +# +# NB: If new types are introduced in vpe.api, these must be updated. +# +type_size = {'u8': 1, + 'u16' : 2, + 'u32' : 4, + 'i32' : 4, + 'u64' : 8, + 'f64' : 8, + 'vl_api_ip4_fib_counter_t' : 21, + 'vl_api_ip6_fib_counter_t' : 33, +}; + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +def get_args(t): + argslist = [] + for i in t: + if i[1][0] == '_': + argslist.append(i[1][1:]) + else: + argslist.append(i[1]) + + return argslist + +def get_pack(f): + zeroarray = False + bytecount = 0 + pack = '' + elements = 1 + if len(f) is 3 or len(f) is 4: # TODO: add support for variable length arrays (VPP-162) + size = type_size[f[0]] + bytecount += size * int(f[2]) + # Check if we have a zero length array + if f[2] == '0': + # If len 3 zero array + elements = 0; + pack += format_struct[f[0]] + bytecount = size + elif size == 1: + n = f[2] * size + pack += str(n) + 's' + else: + pack += format_struct[f[0]] * int(f[2]) + elements = int(f[2]) + else: + bytecount += type_size[f[0]] + pack += format_struct[f[0]] + return (pack, elements, bytecount) + + +''' +def get_reply_func(f): + if f['name']+'_reply' in func_name: + return func_name[f['name']+'_reply'] + if f['name'].find('_dump') > 0: + r = f['name'].replace('_dump','_details') + if r in func_name: + return func_name[r] + return None +''' + +def footer_print(): + print(''' +def msg_id_base_set(b): + global base + base = b + +import os +name = os.path.splitext(os.path.basename(__file__))[0] + ''') + print(u"plugin_register(name, api_func_table, api_name_to_id,", vl_api_version, ", msg_id_base_set)") + +def api_table_print(name, i): + msg_id_in = 'VL_API_' + name.upper() + fstr = name + '_decode' + print('api_func_table.append(' + fstr + ')') + print('api_name_to_id["' + msg_id_in + '"] =', i) + print('') + +def encode_print(name, id, t): + total = 0 + args = get_args(t) + pack = '>' + for i, f in enumerate(t): + p, elements, size = get_pack(f) + pack += p + total += size + + if name.find('_dump') > 0: + multipart = True + else: + multipart = False + + if len(args) < 4: + print(u"def", name + "(async = False):") + else: + print(u"def", name + "(" + ', '.join(args[3:]) + ", async = False):") + print(u" global base") + print(u" context = get_context(base + " + id + ")") + + print(''' + results_prepare(context) + waiting_for_reply_set() + ''') + if multipart == True: + print(u" results_more_set(context)") + + ### TODO deal with zeroarray!!! + #if zeroarray == True: + # print(u" vpp_api.write(pack('" + pack + "', " + id + ", 0, context, " + ', '.join(args[3:-1]) + ") + " + args[-1] + ")") + #else: + print(u" vpp_api.write(pack('" + pack + "', base + " + id + ", 0, context, " + ', '.join(args[3:]) + "))") + + if multipart == True: + print(u" vpp_api.write(pack('>HII', VL_API_CONTROL_PING, 0, context))") + + print(''' + if not async: + results_event_wait(context, 5) + return results_get(context) + return context + ''') + +def get_normal_pack(t, i, pack, offset): + while t: + f = t.pop(0) + i += 1 + if len(f) >= 3: + return t, i, pack, offset, f + p, elements, size = get_pack(f) + pack += p + offset += size + return t, i, pack, offset, None + +def decode_print(name, t): + # + # Generate code for each element + # + print(u'def ' + name + u'_decode(msg):') + total = 0 + args = get_args(t) + print(u" n = namedtuple('" + name + "', '" + ', '.join(args) + "')") + print(u" res = []") + + pack = '>' + start = 0 + end = 0 + offset = 0 + t = list(t) + i = 0 + while t: + t, i, pack, offset, array = get_normal_pack(t, i, pack, offset) + if array: + p, elements, size = get_pack(array) + + # Byte string + if elements > 0 and type_size[array[0]] == 1: + pack += p + offset += size * elements + continue + + # Dump current pack string + if pack != '>': + print(u" tr = unpack_from('" + pack + "', msg[" + str(start) + ":])") + print(u" res.extend(list(tr))") + start += offset + pack = '>' + + if elements == 0: + # This has to be the last element + if len(array) == 3: + print(u" res.append(msg[" + str(offset) + ":])") + if len(t) > 0: + eprint('WARNING: Variable length array must be last element in message', name, array) + + continue + if size == 1 or len(p) == 1: + # Do it as a bytestring. + if p == 'B': + p = 's' + # XXX: Assume that length parameter is the previous field. Add validation. + print(u" c = res[" + str(i - 2) + "]") + print(u" tr = unpack_from('>' + str(c) + '" + p + "', msg[" + str(start) + ":])") + print(u" res.append(tr)") + continue + print(u" tr2 = []") + print(u" offset = " + str(total)) + print(u" for j in range(res[" + str(i - 2) + "]):") + print(u" tr2.append(unpack_from('>" + p + "', msg[" + str(start) + ":], offset))") + print(u" offset += " + str(size)) + print(u" res.append(tr2)") + continue + + # Missing something!! + print(u" tr = unpack_from('>" + p + "', msg[" + str(start) + ":])") + start += size + + print(u" res.append(tr)") + + if pack != '>': + print(u" tr = unpack_from('" + pack + "', msg[" + str(start) + ":])") + print(u" res.extend(list(tr))") + print(u" return n._make(res)") + print('') + +# +# Generate the main Python file +# +def main(): + print(''' +# +# AUTO-GENERATED FILE. PLEASE DO NOT EDIT. +# +from vpp_api_base import * +from struct import * +from collections import namedtuple +import vpp_api +api_func_table = [] +api_name_to_id = {} + ''') + + for i, a in enumerate(vppapidef): + name = a[0] + encode_print(name, str(i), a[1:]) + decode_print(name, a[1:]) + api_table_print(name, i) + footer_print() + +if __name__ == "__main__": + main() -- 2.16.6