Code Review
/
vpp.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
review
|
tree
raw
|
inline
| side by side
CSIT-1148: VPPAPIGEN 2nd level import of .api ignored
[vpp.git]
/
src
/
tools
/
vppapigen
/
vppapigen.py
diff --git
a/src/tools/vppapigen/vppapigen.py
b/src/tools/vppapigen/vppapigen.py
index
a230e22
..
f3094ec
100755
(executable)
--- a/
src/tools/vppapigen/vppapigen.py
+++ b/
src/tools/vppapigen/vppapigen.py
@@
-9,12
+9,16
@@
import logging
import binascii
import os
import binascii
import os
+# Ensure we don't leave temporary files around
+sys.dont_write_bytecode = True
+
#
# VPP API language
#
# Global dictionary of new types (including enums)
global_types = {}
#
# VPP API language
#
# Global dictionary of new types (including enums)
global_types = {}
+global_crc = 0
def global_type_add(name):
def global_type_add(name):
@@
-27,7
+31,7
@@
def global_type_add(name):
# All your trace are belong to us!
def exception_handler(exception_type, exception, traceback):
# All your trace are belong to us!
def exception_handler(exception_type, exception, traceback):
- print
("%s: %s" % (exception_type.__name__, exception))
+ print("%s: %s" % (exception_type.__name__, exception))
#
#
@@
-41,6
+45,7
@@
class VPPAPILexer(object):
'service': 'SERVICE',
'rpc': 'RPC',
'returns': 'RETURNS',
'service': 'SERVICE',
'rpc': 'RPC',
'returns': 'RETURNS',
+ 'null': 'NULL',
'stream': 'STREAM',
'events': 'EVENTS',
'define': 'DEFINE',
'stream': 'STREAM',
'events': 'EVENTS',
'define': 'DEFINE',
@@
-66,6
+71,7
@@
class VPPAPILexer(object):
'import': 'IMPORT',
'true': 'TRUE',
'false': 'FALSE',
'import': 'IMPORT',
'true': 'TRUE',
'false': 'FALSE',
+ 'union': 'UNION',
}
tokens = ['STRING_LITERAL',
}
tokens = ['STRING_LITERAL',
@@
-116,11
+122,6
@@
class VPPAPILexer(object):
t_ignore = ' \t'
t_ignore = ' \t'
-class Iterator(type):
- def __iter__(self):
- return self.iter()
-
-
class Service():
def __init__(self, caller, reply, events=[], stream=False):
self.caller = caller
class Service():
def __init__(self, caller, reply, events=[], stream=False):
self.caller = caller
@@
-131,33
+132,56
@@
class Service():
class Typedef():
def __init__(self, name, flags, block):
class Typedef():
def __init__(self, name, flags, block):
+ global global_crc
self.name = name
self.flags = flags
self.block = block
self.crc = binascii.crc32(str(block)) & 0xffffffff
self.name = name
self.flags = flags
self.block = block
self.crc = binascii.crc32(str(block)) & 0xffffffff
+ global_crc = binascii.crc32(str(block), global_crc)
+ self.manual_print = False
+ self.manual_endian = False
+ for f in flags:
+ if f == 'manual_print':
+ self.manual_print = True
+ elif f == 'manual_endian':
+ self.manual_endian = True
global_type_add(name)
def __repr__(self):
return self.name + str(self.flags) + str(self.block)
global_type_add(name)
def __repr__(self):
return self.name + str(self.flags) + str(self.block)
+class Union():
+ def __init__(self, name, block):
+ self.type = 'Union'
+ self.manual_print = False
+ self.manual_endian = False
+ global global_crc
+ self.name = name
+ self.block = block
+ self.crc = binascii.crc32(str(block)) & 0xffffffff
+ global_crc = binascii.crc32(str(block), global_crc)
+ global_type_add(name)
+
+ def __repr__(self):
+ return str(self.block)
+
+
class Define():
def __init__(self, name, flags, block):
class Define():
def __init__(self, name, flags, block):
+ global global_crc
self.name = name
self.flags = flags
self.block = block
self.crc = binascii.crc32(str(block)) & 0xffffffff
self.name = name
self.flags = flags
self.block = block
self.crc = binascii.crc32(str(block)) & 0xffffffff
- self.typeonly = False
+ global_crc = binascii.crc32(str(block), global_crc)
self.dont_trace = False
self.manual_print = False
self.manual_endian = False
self.autoreply = False
self.singular = False
for f in flags:
self.dont_trace = False
self.manual_print = False
self.manual_endian = False
self.autoreply = False
self.singular = False
for f in flags:
- if f == 'typeonly':
- self.typeonly = True
- global_type_add(name)
- elif f == 'dont_trace':
+ if f == 'dont_trace':
self.dont_trace = True
elif f == 'manual_print':
self.manual_print = True
self.dont_trace = True
elif f == 'manual_print':
self.manual_print = True
@@
-178,8
+202,10
@@
class Define():
class Enum():
def __init__(self, name, block, enumtype='u32'):
class Enum():
def __init__(self, name, block, enumtype='u32'):
+ global global_crc
self.name = name
self.enumtype = enumtype
self.name = name
self.enumtype = enumtype
+
count = 0
for i, b in enumerate(block):
if type(b) is list:
count = 0
for i, b in enumerate(block):
if type(b) is list:
@@
-190,6
+216,7
@@
class Enum():
self.block = block
self.crc = binascii.crc32(str(block)) & 0xffffffff
self.block = block
self.crc = binascii.crc32(str(block)) & 0xffffffff
+ global_crc = binascii.crc32(str(block), global_crc)
global_type_add(name)
def __repr__(self):
global_type_add(name)
def __repr__(self):
@@
-217,8
+244,10
@@
class Import():
class Option():
def __init__(self, option):
class Option():
def __init__(self, option):
+ global global_crc
self.option = option
self.crc = binascii.crc32(str(option)) & 0xffffffff
self.option = option
self.crc = binascii.crc32(str(option)) & 0xffffffff
+ global_crc = binascii.crc32(str(option), global_crc)
def __repr__(self):
return str(self.option)
def __repr__(self):
return str(self.option)
@@
-326,6
+355,7
@@
class VPPAPIParser(object):
| option
| import
| enum
| option
| import
| enum
+ | union
| service'''
p[0] = p[1]
| service'''
p[0] = p[1]
@@
-346,9
+376,15
@@
class VPPAPIParser(object):
p[0] = p[1] + [p[2]]
def p_service_statement(self, p):
p[0] = p[1] + [p[2]]
def p_service_statement(self, p):
- '''service_statement : RPC ID RETURNS ID ';'
+ '''service_statement : RPC ID RETURNS NULL ';'
+ | RPC ID RETURNS ID ';'
| RPC ID RETURNS STREAM ID ';'
| RPC ID RETURNS ID EVENTS event_list ';' '''
| RPC ID RETURNS STREAM ID ';'
| RPC ID RETURNS ID EVENTS event_list ';' '''
+ if p[2] == p[4]:
+ # Verify that caller and reply differ
+ self._parse_error(
+ 'Reply ID ({}) should not be equal to Caller ID'.format(p[2]),
+ self._token_coord(p, 1))
if len(p) == 8:
p[0] = Service(p[2], p[4], p[6])
elif len(p) == 7:
if len(p) == 8:
p[0] = Service(p[2], p[4], p[6])
elif len(p) == 7:
@@
-393,7
+429,11
@@
class VPPAPIParser(object):
def p_define_flist(self, p):
'''define : flist DEFINE ID '{' block_statements_opt '}' ';' '''
def p_define_flist(self, p):
'''define : flist DEFINE ID '{' block_statements_opt '}' ';' '''
- p[0] = Define(p[3], p[1], p[5])
+ # Legacy typedef
+ if 'typeonly' in p[1]:
+ p[0] = Typedef(p[3], p[1], p[5])
+ else:
+ p[0] = Define(p[3], p[1], p[5])
def p_flist(self, p):
'''flist : flag
def p_flist(self, p):
'''flist : flag
@@
-418,7
+458,7
@@
class VPPAPIParser(object):
p[0] = Typedef(p[2], [], p[4])
def p_block_statements_opt(self, p):
p[0] = Typedef(p[2], [], p[4])
def p_block_statements_opt(self, p):
- '''block_statements_opt : block_statements'''
+ '''block_statements_opt : block_statements
'''
p[0] = p[1]
def p_block_statements(self, p):
p[0] = p[1]
def p_block_statements(self, p):
@@
-512,6
+552,10
@@
class VPPAPIParser(object):
self._token_coord(p, 1))
p[0] = p[1]
self._token_coord(p, 1))
p[0] = p[1]
+ def p_union(self, p):
+ '''union : UNION ID '{' block_statements_opt '}' ';' '''
+ p[0] = Union(p[2], p[4])
+
# Error rule for syntax errors
def p_error(self, p):
if p:
# Error rule for syntax errors
def p_error(self, p):
if p:
@@
-527,7
+571,7
@@
class VPPAPI(object):
def __init__(self, debug=False, filename='', logger=None):
self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug)
self.parser = yacc.yacc(module=VPPAPIParser(filename, logger),
def __init__(self, debug=False, filename='', logger=None):
self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug)
self.parser = yacc.yacc(module=VPPAPIParser(filename, logger),
-
tabmodule='vppapigentab'
, debug=debug)
+
write_tables=False
, debug=debug)
self.logger = logger
def parse_string(self, code, debug=0, lineno=1):
self.logger = logger
def parse_string(self, code, debug=0, lineno=1):
@@
-545,96
+589,109
@@
class VPPAPI(object):
def process(self, objs):
s = {}
def process(self, objs):
s = {}
- s['defines'] = []
- s['typedefs'] = []
- s['imports'] = []
- s['options'] = {}
- s['enums'] = []
- s['services'] = []
-
+ s['Option'] = {}
+ s['Define'] = []
+ s['Service'] = []
+ s['types'] = []
+ s['Import'] = []
for o in objs:
for o in objs:
+ tname = o.__class__.__name__
if isinstance(o, Define):
if isinstance(o, Define):
- if o.typeonly:
- s['typedefs'].append(o)
- else:
- s['defines'].append(o)
- if o.autoreply:
- s['defines'].append(self.autoreply_block(o.name))
+ s[tname].append(o)
+ if o.autoreply:
+ s[tname].append(self.autoreply_block(o.name))
elif isinstance(o, Option):
elif isinstance(o, Option):
- s['options'][o[1]] = o[2]
- elif isinstance(o, Enum):
- s['enums'].append(o)
- elif isinstance(o, Typedef):
- s['typedefs'].append(o)
+ s[tname][o[1]] = o[2]
elif type(o) is list:
for o2 in o:
if isinstance(o2, Service):
elif type(o) is list:
for o2 in o:
if isinstance(o2, Service):
- s['services'].append(o2)
+ s['Service'].append(o2)
+ elif isinstance(o, Enum) or isinstance(o, Typedef) or isinstance(o, Union):
+ s['types'].append(o)
+ else:
+ if tname not in s:
+ raise ValueError('Unknown class type: {} {}'.format(tname, o))
+ s[tname].append(o)
- # Create services implicitly
- msgs = {d.name: d for d in s['defines']}
- svcs = {s.caller: s for s in s['services']}
+ msgs = {d.name: d for d in s['Define']}
+ svcs = {s.caller: s for s in s['Service']}
+ replies = {s.reply: s for s in s['Service']}
+ seen_services = {}
for service in svcs:
if service not in msgs:
for service in svcs:
if service not in msgs:
- raise ValueError('Service definition refers to unknown message'
- ' definition: {}'.format(service))
- if svcs[service].reply not in msgs:
+ raise ValueError(
+ 'Service definition refers to unknown message'
+ ' definition: {}'.format(service))
+ if svcs[service].reply != 'null' and \
+ svcs[service].reply not in msgs:
raise ValueError('Service definition refers to unknown message'
' definition in reply: {}'
.format(svcs[service].reply))
raise ValueError('Service definition refers to unknown message'
' definition in reply: {}'
.format(svcs[service].reply))
+ if service in replies:
+ raise ValueError('Service definition refers to message'
+ ' marked as reply: {}'.format(service))
for event in svcs[service].events:
if event not in msgs:
raise ValueError('Service definition refers to unknown '
'event: {} in message: {}'
.format(event, service))
for event in svcs[service].events:
if event not in msgs:
raise ValueError('Service definition refers to unknown '
'event: {} in message: {}'
.format(event, service))
+ seen_services[event] = True
+ # Create services implicitly
for d in msgs:
for d in msgs:
- if
msgs[d].singular is True
:
+ if
d in seen_services
:
continue
continue
- if
d.endswith('_counters')
:
+ if
msgs[d].singular is True
:
continue
if d.endswith('_reply'):
if d[:-6] in svcs:
continue
if d[:-6] not in msgs:
continue
if d.endswith('_reply'):
if d[:-6] in svcs:
continue
if d[:-6] not in msgs:
-
self.logger.warning
('{} missing calling message'
-
.format(d))
+
raise ValueError
('{} missing calling message'
+ .format(d))
continue
if d.endswith('_dump'):
if d in svcs:
continue
if d[:-5]+'_details' in msgs:
continue
if d.endswith('_dump'):
if d in svcs:
continue
if d[:-5]+'_details' in msgs:
- s['
services
'].append(Service(d, d[:-5]+'_details',
+ s['
Service
'].append(Service(d, d[:-5]+'_details',
stream=True))
else:
stream=True))
else:
-
self.logger.e
rror('{} missing details message'
-
.format(d))
+
raise ValueE
rror('{} missing details message'
+ .format(d))
continue
if d.endswith('_details'):
if d[:-8]+'_dump' not in msgs:
continue
if d.endswith('_details'):
if d[:-8]+'_dump' not in msgs:
-
self.logger.e
rror('{} missing dump message'
-
.format(d))
+
raise ValueE
rror('{} missing dump message'
+ .format(d))
continue
if d in svcs:
continue
if d+'_reply' in msgs:
continue
if d in svcs:
continue
if d+'_reply' in msgs:
- s['
services
'].append(Service(d, d+'_reply'))
+ s['
Service
'].append(Service(d, d+'_reply'))
else:
else:
- self.logger.warning('{} missing reply message ({})'
- .format(d, d+'_reply'))
-
s['services'].append(Service(d, None
))
+ raise ValueError(
+ '{} missing reply message ({}) or service definition'
+
.format(d, d+'_reply'
))
return s
return s
- def process_imports(self, objs):
+ def process_imports(self, objs, in_import, result):
+ imported_objs = []
for o in objs:
for o in objs:
+ # Only allow the following object types from imported file
+ if in_import and not (isinstance(o, Enum) or
+ isinstance(o, Union) or
+ isinstance(o, Typedef) or
+ isinstance(o, Import)):
+ continue
if isinstance(o, Import):
if isinstance(o, Import):
-
return objs + self.process_imports(o.
result)
- return objs
-
+
self.process_imports(o.result, True,
result)
+ else:
+ result.append(o)
# Add message ids to each message.
def add_msg_id(s):
# Add message ids to each message.
def add_msg_id(s):
@@
-643,10
+700,6
@@
def add_msg_id(s):
return s
return s
-def getcrc(s):
- return binascii.crc32(str(s)) & 0xffffffff
-
-
dirlist = []
dirlist = []
@@
-664,9
+717,6
@@
def dirlist_get():
# Main
#
def main():
# Main
#
def main():
- logging.basicConfig()
- log = logging.getLogger('vppapigen')
-
cliparser = argparse.ArgumentParser(description='VPP API generator')
cliparser.add_argument('--pluginpath', default=""),
cliparser.add_argument('--includedir', action='append'),
cliparser = argparse.ArgumentParser(description='VPP API generator')
cliparser.add_argument('--pluginpath', default=""),
cliparser.add_argument('--includedir', action='append'),
@@
-692,27
+742,34
@@
def main():
else:
filename = ''
else:
filename = ''
+ if args.debug:
+ logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
+ else:
+ logging.basicConfig()
+ log = logging.getLogger('vppapigen')
+
parser = VPPAPI(debug=args.debug, filename=filename, logger=log)
parser = VPPAPI(debug=args.debug, filename=filename, logger=log)
-
result
= parser.parse_file(args.input, log)
+
parsed_objects
= parser.parse_file(args.input, log)
# Build a list of objects. Hash of lists.
# Build a list of objects. Hash of lists.
- result = parser.process_imports(result)
+ result = []
+ parser.process_imports(parsed_objects, False, result)
s = parser.process(result)
# Add msg_id field
s = parser.process(result)
# Add msg_id field
- s['
defines'] = add_msg_id(s['defines
'])
+ s['
Define'] = add_msg_id(s['Define
'])
- file_crc = g
etcrc(s)
+ file_crc = g
lobal_crc & 0xffffffff
#
# Debug
if args.debug:
import pprint
#
# Debug
if args.debug:
import pprint
- pp = pprint.PrettyPrinter(indent=4)
- for t in s['defines']:
- pp.pprint([t.name, t.flags, t.block])
- for t in s['typedefs']:
+ pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
+ for t in s['Define']:
pp.pprint([t.name, t.flags, t.block])
pp.pprint([t.name, t.flags, t.block])
+ for t in s['types']:
+ pp.pprint([t.name, t.block])
#
# Generate representation
#
# Generate representation
@@
-724,7
+781,7
@@
def main():
if not args.pluginpath:
cand = []
cand.append(os.path.dirname(os.path.realpath(__file__)))
if not args.pluginpath:
cand = []
cand.append(os.path.dirname(os.path.realpath(__file__)))
- cand.append(os.path.dirname(os.path.realpath(__file__)) +
\
+ cand.append(os.path.dirname(os.path.realpath(__file__)) +
'/../share/vpp/')
for c in cand:
c += '/'
'/../share/vpp/')
for c in cand:
c += '/'
@@
-745,7
+802,7
@@
def main():
result = plugin.run(filename, s, file_crc)
if result:
result = plugin.run(filename, s, file_crc)
if result:
- print
(result, file=args.output)
+ print(result, file=args.output)
else:
raise Exception('Running plugin failed: {} {}'
.format(filename, result))
else:
raise Exception('Running plugin failed: {} {}'
.format(filename, result))