-#!/usr/bin/env python
+#!/usr/bin/env python3
-from __future__ import print_function
import ply.lex as lex
import ply.yacc as yacc
import sys
import argparse
+import keyword
import logging
import binascii
import os
+import sys
+
+log = logging.getLogger('vppapigen')
+
+# Ensure we don't leave temporary files around
+sys.dont_write_bytecode = True
#
# VPP API language
global_types = {}
-def global_type_add(name):
+def global_type_add(name, obj):
'''Add new type to the dictionary of types '''
type_name = 'vl_api_' + name + '_t'
- if type_name in global_types:
- raise KeyError('Type is already defined: {}'.format(name))
- global_types[type_name] = True
+ global_types[type_name] = obj
# All your trace are belong to us!
def exception_handler(exception_type, exception, traceback):
- print ("%s: %s" % (exception_type.__name__, exception))
+ print("%s: %s" % (exception_type.__name__, exception))
#
'import': 'IMPORT',
'true': 'TRUE',
'false': 'FALSE',
+ 'union': 'UNION',
}
tokens = ['STRING_LITERAL',
t_ignore_LINE_COMMENT = '//.*'
+ def t_FALSE(self, t):
+ r'false'
+ t.value = False
+ return t
+
+ def t_TRUE(self, t):
+ r'false'
+ t.value = True
+ return t
+
def t_NUM(self, t):
- r'0[xX][0-9a-fA-F]+|\d+'
+ r'0[xX][0-9a-fA-F]+|-?\d+\.?\d*'
base = 16 if t.value.startswith('0x') else 10
- t.value = int(t.value, base)
+ if '.' in t.value:
+ t.value = float(t.value)
+ else:
+ t.value = int(t.value, base)
return t
def t_ID(self, t):
# A string containing ignored characters (spaces and tabs)
t_ignore = ' \t'
+
+def crc_block_combine(block, crc):
+ s = str(block).encode()
+ return binascii.crc32(s, crc) & 0xffffffff
+
+
class Service():
- def __init__(self, caller, reply, events=[], stream=False):
+ def __init__(self, caller, reply, events=None, stream=False):
self.caller = caller
self.reply = reply
self.stream = stream
- self.events = events
+ self.events = [] if events is None else events
class Typedef():
self.name = name
self.flags = flags
self.block = block
- self.crc = binascii.crc32(str(block)) & 0xffffffff
- global_type_add(name)
+ self.crc = str(block).encode()
+ self.manual_print = False
+ self.manual_endian = False
+ for f in flags:
+ if f == 'manual_print':
+ self.manual_print = True
+ elif f == 'manual_endian':
+ self.manual_endian = True
+ for b in block:
+ # Tag length field of a VLA
+ if isinstance(b, Array):
+ if b.lengthfield:
+ for b2 in block:
+ if b2.fieldname == b.lengthfield:
+ b2.vla_len = True
+
+ global_type_add(name, self)
+
+ self.vla = False
+
+ for i, b in enumerate(block):
+ if isinstance(b, Array):
+ if b.length == 0:
+ self.vla = True
+ if i + 1 < len(block):
+ raise ValueError(
+ 'VLA field "{}" must be the last '
+ 'field in message "{}"'
+ .format(b.fieldname, name))
+ elif b.fieldtype == 'string':
+ self.vla = True
+ if i + 1 < len(block):
+ raise ValueError(
+ 'VLA field "{}" must be the last '
+ 'field in message "{}"'
+ .format(b.fieldname, name))
def __repr__(self):
return self.name + str(self.flags) + str(self.block)
+class Using():
+ def __init__(self, name, flags, alias):
+ self.name = name
+ self.vla = False
+
+ self.manual_print = False
+ self.manual_endian = False
+ for f in flags:
+ if f == 'manual_print':
+ self.manual_print = True
+ elif f == 'manual_endian':
+ self.manual_endian = True
+
+ if isinstance(alias, Array):
+ a = {'type': alias.fieldtype,
+ 'length': alias.length}
+ else:
+ a = {'type': alias.fieldtype}
+ self.alias = a
+ self.crc = str(alias).encode()
+ global_type_add(name, self)
+
+ def __repr__(self):
+ return self.name + str(self.alias)
+
+
+class Union():
+ def __init__(self, name, flags, block):
+ self.type = 'Union'
+ self.manual_print = False
+ self.manual_endian = False
+ self.name = name
+
+ self.manual_print = False
+ self.manual_endian = False
+ for f in flags:
+ if f == 'manual_print':
+ self.manual_print = True
+ elif f == 'manual_endian':
+ self.manual_endian = True
+
+ self.block = block
+ self.crc = str(block).encode()
+ global_type_add(name, self)
+
+ def __repr__(self):
+ return str(self.block)
+
+
class Define():
def __init__(self, name, flags, block):
self.name = name
self.flags = flags
self.block = block
- self.crc = binascii.crc32(str(block)) & 0xffffffff
- self.typeonly = False
+ self.crc = str(block).encode()
self.dont_trace = False
self.manual_print = False
self.manual_endian = False
self.autoreply = False
self.singular = False
for f in flags:
- if f == 'typeonly':
- self.typeonly = True
- global_type_add(name)
- elif f == 'dont_trace':
+ if f == 'dont_trace':
self.dont_trace = True
elif f == 'manual_print':
self.manual_print = True
elif f == 'autoreply':
self.autoreply = True
- for b in block:
+ for i, b in enumerate(block):
if isinstance(b, Option):
if b[1] == 'singular' and b[2] == 'true':
self.singular = True
block.remove(b)
+ if isinstance(b, Array) and b.vla and i + 1 < len(block):
+ raise ValueError(
+ 'VLA field "{}" must be the last field in message "{}"'
+ .format(b.fieldname, name))
+ elif b.fieldtype.startswith('vl_api_'):
+ if (global_types[b.fieldtype].vla and i + 1 < len(block)):
+ raise ValueError(
+ 'VLA field "{}" must be the last '
+ 'field in message "{}"'
+ .format(b.fieldname, name))
+ elif b.fieldtype == 'string' and b.length == 0:
+ if i + 1 < len(block):
+ raise ValueError(
+ 'VLA field "{}" must be the last '
+ 'field in message "{}"'
+ .format(b.fieldname, name))
+ # Tag length field of a VLA
+ if isinstance(b, Array):
+ if b.lengthfield:
+ for b2 in block:
+ if b2.fieldname == b.lengthfield:
+ b2.vla_len = True
+
def __repr__(self):
return self.name + str(self.flags) + str(self.block)
def __init__(self, name, block, enumtype='u32'):
self.name = name
self.enumtype = enumtype
+ self.vla = False
+
count = 0
for i, b in enumerate(block):
if type(b) is list:
block[i] = [b, count]
self.block = block
- self.crc = binascii.crc32(str(block)) & 0xffffffff
- global_type_add(name)
+ self.crc = str(block).encode()
+ global_type_add(name, self)
def __repr__(self):
return self.name + str(self.block)
f = os.path.join(dir, filename)
if os.path.exists(f):
break
- with open(f) as fd:
+
+ with open(f, encoding='utf-8') as fd:
self.result = parser.parse_file(fd, None)
def __repr__(self):
class Option():
- def __init__(self, option):
+ def __init__(self, option, value):
+ self.type = 'Option'
self.option = option
- self.crc = binascii.crc32(str(option)) & 0xffffffff
+ self.value = value
+ self.crc = str(option).encode()
def __repr__(self):
return str(self.option)
class Array():
- def __init__(self, fieldtype, name, length):
+ def __init__(self, fieldtype, name, length, modern_vla=False):
self.type = 'Array'
self.fieldtype = fieldtype
self.fieldname = name
+ self.modern_vla = modern_vla
if type(length) is str:
self.lengthfield = length
self.length = 0
+ self.vla = True
else:
self.length = length
self.lengthfield = None
+ self.vla = False
def __repr__(self):
return str([self.fieldtype, self.fieldname, self.length,
class Field():
- def __init__(self, fieldtype, name):
+ def __init__(self, fieldtype, name, limit=None):
self.type = 'Field'
self.fieldtype = fieldtype
+
+ if self.fieldtype == 'string':
+ raise ValueError("The string type {!r} is an "
+ "array type ".format(name))
+
+ if name in keyword.kwlist:
+ raise ValueError("Fieldname {!r} is a python keyword and is not "
+ "accessible via the python API. ".format(name))
self.fieldname = name
+ self.limit = limit
def __repr__(self):
return str([self.fieldtype, self.fieldname])
| option
| import
| enum
+ | union
| service'''
p[0] = p[1]
| RPC ID RETURNS ID EVENTS event_list ';' '''
if p[2] == p[4]:
# Verify that caller and reply differ
- self._parse_error('Reply ID ({}) should not be equal to Caller ID'.format(p[2]),
- self._token_coord(p, 1))
+ self._parse_error(
+ 'Reply ID ({}) should not be equal to Caller ID'.format(p[2]),
+ self._token_coord(p, 1))
if len(p) == 8:
p[0] = Service(p[2], p[4], p[6])
elif len(p) == 7:
def p_define_flist(self, p):
'''define : flist DEFINE ID '{' block_statements_opt '}' ';' '''
- p[0] = Define(p[3], p[1], p[5])
+ # Legacy typedef
+ if 'typeonly' in p[1]:
+ self._parse_error('legacy typedef. use typedef: {} {}[{}];'
+ .format(p[1], p[2], p[4]),
+ self._token_coord(p, 1))
+ else:
+ p[0] = Define(p[3], p[1], p[5])
def p_flist(self, p):
'''flist : flag
'''typedef : TYPEDEF ID '{' block_statements_opt '}' ';' '''
p[0] = Typedef(p[2], [], p[4])
+ def p_typedef_flist(self, p):
+ '''typedef : flist TYPEDEF ID '{' block_statements_opt '}' ';' '''
+ p[0] = Typedef(p[3], p[1], p[5])
+
+ def p_typedef_alias(self, p):
+ '''typedef : TYPEDEF declaration '''
+ p[0] = Using(p[2].fieldname, [], p[2])
+
+ def p_typedef_alias_flist(self, p):
+ '''typedef : flist TYPEDEF declaration '''
+ p[0] = Using(p[3].fieldname, p[1], p[3])
+
def p_block_statements_opt(self, p):
- '''block_statements_opt : block_statements'''
+ '''block_statements_opt : block_statements '''
p[0] = p[1]
def p_block_statements(self, p):
def p_enum_statements(self, p):
'''enum_statements : enum_statement
- | enum_statements enum_statement'''
+ | enum_statements enum_statement'''
if len(p) == 2:
p[0] = [p[1]]
else:
else:
p[0] = p[1]
+ def p_field_options(self, p):
+ '''field_options : field_option
+ | field_options field_option'''
+ if len(p) == 2:
+ p[0] = p[1]
+ else:
+ p[0] = {**p[1], **p[2]}
+
+ def p_field_option(self, p):
+ '''field_option : ID
+ | ID '=' assignee ','
+ | ID '=' assignee
+
+ '''
+ if len(p) == 2:
+ p[0] = {p[1]: None}
+ else:
+ p[0] = {p[1]: p[3]}
+
def p_declaration(self, p):
- '''declaration : type_specifier ID ';' '''
- if len(p) != 4:
+ '''declaration : type_specifier ID ';'
+ | type_specifier ID '[' field_options ']' ';' '''
+ if len(p) == 7:
+ p[0] = Field(p[1], p[2], p[4])
+ elif len(p) == 4:
+ p[0] = Field(p[1], p[2])
+ else:
self._parse_error('ERROR')
self.fields.append(p[2])
- p[0] = Field(p[1], p[2])
+
+ def p_declaration_array_vla(self, p):
+ '''declaration : type_specifier ID '[' ']' ';' '''
+ p[0] = Array(p[1], p[2], 0, modern_vla=True)
def p_declaration_array(self, p):
'''declaration : type_specifier ID '[' NUM ']' ';'
| type_specifier ID '[' ID ']' ';' '''
+
if len(p) != 7:
return self._parse_error(
'array: %s' % p.value,
def p_option(self, p):
'''option : OPTION ID '=' assignee ';' '''
- p[0] = Option([p[1], p[2], p[4]])
+ p[0] = Option(p[2], p[4])
def p_assignee(self, p):
'''assignee : NUM
self._token_coord(p, 1))
p[0] = p[1]
+ def p_union(self, p):
+ '''union : UNION ID '{' block_statements_opt '}' ';' '''
+ p[0] = Union(p[2], [], p[4])
+
+ def p_union_flist(self, p):
+ '''union : flist UNION ID '{' block_statements_opt '}' ';' '''
+ p[0] = Union(p[3], p[1], p[5])
+
# Error rule for syntax errors
def p_error(self, p):
if p:
def __init__(self, debug=False, filename='', logger=None):
self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug)
self.parser = yacc.yacc(module=VPPAPIParser(filename, logger),
- tabmodule='vppapigentab', debug=debug)
+ write_tables=False, debug=debug)
self.logger = logger
def parse_string(self, code, debug=0, lineno=1):
def process(self, objs):
s = {}
- s['defines'] = []
- s['typedefs'] = []
- s['imports'] = []
- s['options'] = {}
- s['enums'] = []
- s['services'] = []
-
+ s['Option'] = {}
+ s['Define'] = []
+ s['Service'] = []
+ s['types'] = []
+ s['Import'] = []
+ s['Alias'] = {}
+ crc = 0
for o in objs:
+ tname = o.__class__.__name__
+ try:
+ crc = binascii.crc32(o.crc, crc)
+ except AttributeError:
+ pass
if isinstance(o, Define):
- if o.typeonly:
- s['typedefs'].append(o)
- else:
- s['defines'].append(o)
- if o.autoreply:
- s['defines'].append(self.autoreply_block(o.name))
+ s[tname].append(o)
+ if o.autoreply:
+ s[tname].append(self.autoreply_block(o.name))
elif isinstance(o, Option):
- s['options'][o[1]] = o[2]
- elif isinstance(o, Enum):
- s['enums'].append(o)
- elif isinstance(o, Typedef):
- s['typedefs'].append(o)
+ s[tname][o[1]] = o[2]
elif type(o) is list:
for o2 in o:
if isinstance(o2, Service):
- s['services'].append(o2)
-
-
- msgs = {d.name: d for d in s['defines']}
- svcs = {s.caller: s for s in s['services']}
- replies = {s.reply: s for s in s['services']}
+ s['Service'].append(o2)
+ elif (isinstance(o, Enum) or
+ isinstance(o, Typedef) or
+ isinstance(o, Union)):
+ s['types'].append(o)
+ elif isinstance(o, Using):
+ s['Alias'][o.name] = o
+ else:
+ if tname not in s:
+ raise ValueError('Unknown class type: {} {}'
+ .format(tname, o))
+ s[tname].append(o)
+
+ msgs = {d.name: d for d in s['Define']}
+ svcs = {s.caller: s for s in s['Service']}
+ replies = {s.reply: s for s in s['Service']}
seen_services = {}
+ s['file_crc'] = crc
+
for service in svcs:
if service not in msgs:
- raise ValueError('Service definition refers to unknown message'
- ' definition: {}'.format(service))
- if svcs[service].reply != 'null' and svcs[service].reply not in msgs:
+ raise ValueError(
+ 'Service definition refers to unknown message'
+ ' definition: {}'.format(service))
+ if svcs[service].reply != 'null' and \
+ svcs[service].reply not in msgs:
raise ValueError('Service definition refers to unknown message'
' definition in reply: {}'
.format(svcs[service].reply))
if d[:-6] in svcs:
continue
if d[:-6] not in msgs:
- self.logger.warning('{} missing calling message'
- .format(d))
+ raise ValueError('{} missing calling message'
+ .format(d))
continue
if d.endswith('_dump'):
if d in svcs:
continue
if d[:-5]+'_details' in msgs:
- s['services'].append(Service(d, d[:-5]+'_details',
- stream=True))
+ s['Service'].append(Service(d, d[:-5]+'_details',
+ stream=True))
else:
- self.logger.error('{} missing details message'
- .format(d))
+ raise ValueError('{} missing details message'
+ .format(d))
continue
if d.endswith('_details'):
if d[:-8]+'_dump' not in msgs:
- self.logger.error('{} missing dump message'
- .format(d))
+ raise ValueError('{} missing dump message'
+ .format(d))
continue
if d in svcs:
continue
if d+'_reply' in msgs:
- s['services'].append(Service(d, d+'_reply'))
+ s['Service'].append(Service(d, d+'_reply'))
else:
- raise ValueError('{} missing reply message ({}) or service definition'
- .format(d, d+'_reply'))
+ raise ValueError(
+ '{} missing reply message ({}) or service definition'
+ .format(d, d+'_reply'))
return s
- def process_imports(self, objs, in_import):
+ def process_imports(self, objs, in_import, result):
imported_objs = []
for o in objs:
+ # Only allow the following object types from imported file
+ if in_import and not (isinstance(o, Enum) or
+ isinstance(o, Union) or
+ isinstance(o, Typedef) or
+ isinstance(o, Import) or
+ isinstance(o, Using)):
+ continue
if isinstance(o, Import):
- return objs + self.process_imports(o.result, True)
- if in_import:
- if isinstance(o, Define) and o.typeonly:
- imported_objs.append(o)
- if in_import:
- return imported_objs
- return objs
+ result.append(o)
+ self.process_imports(o.result, True, result)
+ else:
+ result.append(o)
# Add message ids to each message.
return s
-def getcrc(s):
- return binascii.crc32(str(s)) & 0xffffffff
-
-
dirlist = []
return dirlist
+def foldup_blocks(block, crc):
+ for b in block:
+ # Look up CRC in user defined types
+ if b.fieldtype.startswith('vl_api_'):
+ # Recursively
+ t = global_types[b.fieldtype]
+ try:
+ crc = crc_block_combine(t.block, crc)
+ return foldup_blocks(t.block, crc)
+ except AttributeError:
+ pass
+ return crc
+
+
+def foldup_crcs(s):
+ for f in s:
+ f.crc = foldup_blocks(f.block,
+ binascii.crc32(f.crc))
+
+
#
# Main
#
def main():
+ if sys.version_info < (3, 5,):
+ log.exception('vppapigen requires a supported version of python. '
+ 'Please use version 3.5 or greater. '
+ 'Using {}'.format(sys.version))
+ return 1
+
cliparser = argparse.ArgumentParser(description='VPP API generator')
cliparser.add_argument('--pluginpath', default=""),
cliparser.add_argument('--includedir', action='append'),
- cliparser.add_argument('--input', type=argparse.FileType('r'),
+ cliparser.add_argument('--input',
+ type=argparse.FileType('r', encoding='UTF-8'),
default=sys.stdin)
- cliparser.add_argument('--output', nargs='?', type=argparse.FileType('w'),
+ cliparser.add_argument('--output', nargs='?',
+ type=argparse.FileType('w', encoding='UTF-8'),
default=sys.stdout)
cliparser.add_argument('output_module', nargs='?', default='C')
logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
else:
logging.basicConfig()
- log = logging.getLogger('vppapigen')
-
parser = VPPAPI(debug=args.debug, filename=filename, logger=log)
- result = parser.parse_file(args.input, log)
+ parsed_objects = parser.parse_file(args.input, log)
# Build a list of objects. Hash of lists.
- result = parser.process_imports(result, False)
- s = parser.process(result)
+ result = []
+
+ if args.output_module == 'C':
+ s = parser.process(parsed_objects)
+ else:
+ parser.process_imports(parsed_objects, False, result)
+ s = parser.process(result)
# Add msg_id field
- s['defines'] = add_msg_id(s['defines'])
+ s['Define'] = add_msg_id(s['Define'])
- file_crc = getcrc(s)
+ # Fold up CRCs
+ foldup_crcs(s['Define'])
#
# Debug
if args.debug:
import pprint
- pp = pprint.PrettyPrinter(indent=4)
- for t in s['defines']:
- pp.pprint([t.name, t.flags, t.block])
- for t in s['typedefs']:
+ pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
+ for t in s['Define']:
pp.pprint([t.name, t.flags, t.block])
+ for t in s['types']:
+ pp.pprint([t.name, t.block])
#
# Generate representation
#
- import imp
+ from importlib.machinery import SourceFileLoader
# Default path
pluginpath = ''
if not args.pluginpath:
cand = []
cand.append(os.path.dirname(os.path.realpath(__file__)))
- cand.append(os.path.dirname(os.path.realpath(__file__)) + \
+ cand.append(os.path.dirname(os.path.realpath(__file__)) +
'/../share/vpp/')
for c in cand:
c += '/'
- if os.path.isfile(c + args.output_module + '.py'):
+ if os.path.isfile('{}vppapigen_{}.py'
+ .format(c, args.output_module.lower())):
pluginpath = c
break
else:
pluginpath = args.pluginpath + '/'
if pluginpath == '':
- raise Exception('Output plugin not found')
- module_path = pluginpath + args.output_module + '.py'
+ log.exception('Output plugin not found')
+ return 1
+ module_path = '{}vppapigen_{}.py'.format(pluginpath,
+ args.output_module.lower())
try:
- plugin = imp.load_source(args.output_module, module_path)
- except Exception, err:
- raise Exception('Error importing output plugin: {}, {}'
- .format(module_path, err))
-
- result = plugin.run(filename, s, file_crc)
+ plugin = SourceFileLoader(args.output_module,
+ module_path).load_module()
+ except Exception as err:
+ log.exception('Error importing output plugin: {}, {}'
+ .format(module_path, err))
+ return 1
+
+ result = plugin.run(filename, s)
if result:
- print (result, file=args.output)
+ print(result, file=args.output)
else:
- raise Exception('Running plugin failed: {} {}'
- .format(filename, result))
+ log.exception('Running plugin failed: {} {}'
+ .format(filename, result))
+ return 1
+ return 0
if __name__ == '__main__':
- main()
+ sys.exit(main())