X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Ftools%2Fvppapigen%2Fvppapigen.py;h=576fa54842483514cb7eda88417298f89a9a5ce2;hb=097fa66b9;hp=628bb09def742c95f8e18e10b7cdc05f71d66f42;hpb=d6743b1d64931627905520081ff3218a2c4bac3e;p=vpp.git diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py index 628bb09def7..576fa548424 100755 --- a/src/tools/vppapigen/vppapigen.py +++ b/src/tools/vppapigen/vppapigen.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 from __future__ import print_function import ply.lex as lex @@ -20,17 +20,15 @@ sys.dont_write_bytecode = True global_types = {} -def global_type_add(name): +def global_type_add(name, obj): '''Add new type to the dictionary of types ''' type_name = 'vl_api_' + name + '_t' - if type_name in global_types: - raise KeyError('Type is already defined: {}'.format(name)) - global_types[type_name] = True + global_types[type_name] = obj # All your trace are belong to us! def exception_handler(exception_type, exception, traceback): - print ("%s: %s" % (exception_type.__name__, exception)) + print("%s: %s" % (exception_type.__name__, exception)) # @@ -70,6 +68,7 @@ class VPPAPILexer(object): 'import': 'IMPORT', 'true': 'TRUE', 'false': 'FALSE', + 'union': 'UNION', } tokens = ['STRING_LITERAL', @@ -119,12 +118,17 @@ class VPPAPILexer(object): # A string containing ignored characters (spaces and tabs) t_ignore = ' \t' + +def crc_block_combine(block, crc): + s = str(block).encode() + return binascii.crc32(s, crc) & 0xffffffff + class Service(): - def __init__(self, caller, reply, events=[], stream=False): + def __init__(self, caller, reply, events=None, stream=False): self.caller = caller self.reply = reply self.stream = stream - self.events = events + self.events = [] if events is None else events class Typedef(): @@ -132,30 +136,64 @@ class Typedef(): self.name = name self.flags = flags self.block = block - self.crc = binascii.crc32(str(block)) & 0xffffffff - global_type_add(name) + self.crc = str(block).encode() + self.manual_print = False + self.manual_endian = False + for f in flags: + if f == 'manual_print': + self.manual_print = True + elif f == 'manual_endian': + self.manual_endian = True + global_type_add(name, self) def __repr__(self): return self.name + str(self.flags) + str(self.block) +class Using(): + def __init__(self, name, alias): + self.name = name + + if isinstance(alias, Array): + a = { 'type': alias.fieldtype, # noqa: E201 + 'length': alias.length } # noqa: E202 + else: + a = { 'type': alias.fieldtype } # noqa: E201,E202 + self.alias = a + self.crc = str(alias).encode() + global_type_add(name, self) + + def __repr__(self): + return self.name + str(self.alias) + + +class Union(): + def __init__(self, name, block): + self.type = 'Union' + self.manual_print = False + self.manual_endian = False + self.name = name + self.block = block + self.crc = str(block).encode() + global_type_add(name, self) + + def __repr__(self): + return str(self.block) + + class Define(): def __init__(self, name, flags, block): self.name = name self.flags = flags self.block = block - self.crc = binascii.crc32(str(block)) & 0xffffffff - self.typeonly = False + self.crc = str(block).encode() self.dont_trace = False self.manual_print = False self.manual_endian = False self.autoreply = False self.singular = False for f in flags: - if f == 'typeonly': - self.typeonly = True - global_type_add(name) - elif f == 'dont_trace': + if f == 'dont_trace': self.dont_trace = True elif f == 'manual_print': self.manual_print = True @@ -178,6 +216,7 @@ class Enum(): def __init__(self, name, block, enumtype='u32'): self.name = name self.enumtype = enumtype + count = 0 for i, b in enumerate(block): if type(b) is list: @@ -187,8 +226,8 @@ class Enum(): block[i] = [b, count] self.block = block - self.crc = binascii.crc32(str(block)) & 0xffffffff - global_type_add(name) + self.crc = str(block).encode() + global_type_add(name, self) def __repr__(self): return self.name + str(self.block) @@ -206,8 +245,12 @@ class Import(): f = os.path.join(dir, filename) if os.path.exists(f): break - with open(f) as fd: - self.result = parser.parse_file(fd, None) + if sys.version[0] == '2': + with open(f) as fd: + self.result = parser.parse_file(fd, None) + else: + with open(f, encoding='utf-8') as fd: + self.result = parser.parse_file(fd, None) def __repr__(self): return self.filename @@ -216,7 +259,7 @@ class Import(): class Option(): def __init__(self, option): self.option = option - self.crc = binascii.crc32(str(option)) & 0xffffffff + self.crc = str(option).encode() def __repr__(self): return str(self.option) @@ -243,10 +286,11 @@ class Array(): class Field(): - def __init__(self, fieldtype, name): + def __init__(self, fieldtype, name, limit=None): self.type = 'Field' self.fieldtype = fieldtype self.fieldname = name + self.limit = limit def __repr__(self): return str([self.fieldtype, self.fieldname]) @@ -324,6 +368,7 @@ class VPPAPIParser(object): | option | import | enum + | union | service''' p[0] = p[1] @@ -350,8 +395,9 @@ class VPPAPIParser(object): | RPC ID RETURNS ID EVENTS event_list ';' ''' if p[2] == p[4]: # Verify that caller and reply differ - self._parse_error('Reply ID ({}) should not be equal to Caller ID'.format(p[2]), - self._token_coord(p, 1)) + self._parse_error( + 'Reply ID ({}) should not be equal to Caller ID'.format(p[2]), + self._token_coord(p, 1)) if len(p) == 8: p[0] = Service(p[2], p[4], p[6]) elif len(p) == 7: @@ -396,7 +442,11 @@ class VPPAPIParser(object): def p_define_flist(self, p): '''define : flist DEFINE ID '{' block_statements_opt '}' ';' ''' - p[0] = Define(p[3], p[1], p[5]) + # Legacy typedef + if 'typeonly' in p[1]: + p[0] = Typedef(p[3], p[1], p[5]) + else: + p[0] = Define(p[3], p[1], p[5]) def p_flist(self, p): '''flist : flag @@ -420,8 +470,12 @@ class VPPAPIParser(object): '''typedef : TYPEDEF ID '{' block_statements_opt '}' ';' ''' p[0] = Typedef(p[2], [], p[4]) + def p_typedef_alias(self, p): + '''typedef : TYPEDEF declaration ''' + p[0] = Using(p[2].fieldname, p[2]) + def p_block_statements_opt(self, p): - '''block_statements_opt : block_statements''' + '''block_statements_opt : block_statements ''' p[0] = p[1] def p_block_statements(self, p): @@ -439,7 +493,7 @@ class VPPAPIParser(object): def p_enum_statements(self, p): '''enum_statements : enum_statement - | enum_statements enum_statement''' + | enum_statements enum_statement''' if len(p) == 2: p[0] = [p[1]] else: @@ -453,12 +507,30 @@ class VPPAPIParser(object): else: p[0] = p[1] + def p_field_options(self, p): + '''field_options : field_option + | field_options field_option''' + if len(p) == 2: + p[0] = p[1] + else: + p[0] = { **p[1], **p[2] } + + def p_field_option(self, p): + '''field_option : ID '=' assignee ',' + | ID '=' assignee + ''' + p[0] = { p[1]: p[3] } + def p_declaration(self, p): - '''declaration : type_specifier ID ';' ''' - if len(p) != 4: + '''declaration : type_specifier ID ';' + | type_specifier ID '[' field_options ']' ';' ''' + if len(p) == 7: + p[0] = Field(p[1], p[2], p[4]) + elif len(p) == 4: + p[0] = Field(p[1], p[2]) + else: self._parse_error('ERROR') self.fields.append(p[2]) - p[0] = Field(p[1], p[2]) def p_declaration_array(self, p): '''declaration : type_specifier ID '[' NUM ']' ';' @@ -515,6 +587,10 @@ class VPPAPIParser(object): self._token_coord(p, 1)) p[0] = p[1] + def p_union(self, p): + '''union : UNION ID '{' block_statements_opt '}' ';' ''' + p[0] = Union(p[2], p[4]) + # Error rule for syntax errors def p_error(self, p): if p: @@ -548,43 +624,55 @@ class VPPAPI(object): def process(self, objs): s = {} - s['defines'] = [] - s['typedefs'] = [] - s['imports'] = [] - s['options'] = {} - s['enums'] = [] - s['services'] = [] - + s['Option'] = {} + s['Define'] = [] + s['Service'] = [] + s['types'] = [] + s['Import'] = [] + s['Alias'] = {} + crc = 0 for o in objs: + tname = o.__class__.__name__ + try: + crc = binascii.crc32(o.crc, crc) + except AttributeError: + pass if isinstance(o, Define): - if o.typeonly: - s['typedefs'].append(o) - else: - s['defines'].append(o) - if o.autoreply: - s['defines'].append(self.autoreply_block(o.name)) + s[tname].append(o) + if o.autoreply: + s[tname].append(self.autoreply_block(o.name)) elif isinstance(o, Option): - s['options'][o[1]] = o[2] - elif isinstance(o, Enum): - s['enums'].append(o) - elif isinstance(o, Typedef): - s['typedefs'].append(o) + s[tname][o[1]] = o[2] elif type(o) is list: for o2 in o: if isinstance(o2, Service): - s['services'].append(o2) - - - msgs = {d.name: d for d in s['defines']} - svcs = {s.caller: s for s in s['services']} - replies = {s.reply: s for s in s['services']} + s['Service'].append(o2) + elif (isinstance(o, Enum) or + isinstance(o, Typedef) or + isinstance(o, Union)): + s['types'].append(o) + elif isinstance(o, Using): + s['Alias'][o.name] = o.alias + else: + if tname not in s: + raise ValueError('Unknown class type: {} {}' + .format(tname, o)) + s[tname].append(o) + + msgs = {d.name: d for d in s['Define']} + svcs = {s.caller: s for s in s['Service']} + replies = {s.reply: s for s in s['Service']} seen_services = {} + s['file_crc'] = crc + for service in svcs: if service not in msgs: - raise ValueError('Service definition refers to unknown message' - ' definition: {}'.format(service)) - if svcs[service].reply != 'null' and svcs[service].reply not in msgs: + raise ValueError( + 'Service definition refers to unknown message' + ' definition: {}'.format(service)) + if svcs[service].reply != 'null' and \ + svcs[service].reply not in msgs: raise ValueError('Service definition refers to unknown message' ' definition in reply: {}' .format(svcs[service].reply)) @@ -608,47 +696,51 @@ class VPPAPI(object): if d[:-6] in svcs: continue if d[:-6] not in msgs: - self.logger.warning('{} missing calling message' - .format(d)) + raise ValueError('{} missing calling message' + .format(d)) continue if d.endswith('_dump'): if d in svcs: continue if d[:-5]+'_details' in msgs: - s['services'].append(Service(d, d[:-5]+'_details', - stream=True)) + s['Service'].append(Service(d, d[:-5]+'_details', + stream=True)) else: - self.logger.error('{} missing details message' - .format(d)) + raise ValueError('{} missing details message' + .format(d)) continue if d.endswith('_details'): if d[:-8]+'_dump' not in msgs: - self.logger.error('{} missing dump message' - .format(d)) + raise ValueError('{} missing dump message' + .format(d)) continue if d in svcs: continue if d+'_reply' in msgs: - s['services'].append(Service(d, d+'_reply')) + s['Service'].append(Service(d, d+'_reply')) else: - raise ValueError('{} missing reply message ({}) or service definition' - .format(d, d+'_reply')) + raise ValueError( + '{} missing reply message ({}) or service definition' + .format(d, d+'_reply')) return s - def process_imports(self, objs, in_import): + def process_imports(self, objs, in_import, result): imported_objs = [] for o in objs: + # Only allow the following object types from imported file + if in_import and not (isinstance(o, Enum) or + isinstance(o, Union) or + isinstance(o, Typedef) or + isinstance(o, Import) or + isinstance(o, Using)): + continue if isinstance(o, Import): - return objs + self.process_imports(o.result, True) - if in_import: - if isinstance(o, Define) and o.typeonly: - imported_objs.append(o) - if in_import: - return imported_objs - return objs + self.process_imports(o.result, True, result) + else: + result.append(o) # Add message ids to each message. @@ -658,10 +750,6 @@ def add_msg_id(s): return s -def getcrc(s): - return binascii.crc32(str(s)) & 0xffffffff - - dirlist = [] @@ -674,6 +762,23 @@ def dirlist_add(dirs): def dirlist_get(): return dirlist +def foldup_blocks(block, crc): + for b in block: + # Look up CRC in user defined types + if b.fieldtype.startswith('vl_api_'): + # Recursively + t = global_types[b.fieldtype] + try: + crc = crc_block_combine(t.block, crc) + return foldup_blocks(t.block, crc) + except: + pass + return crc + +def foldup_crcs(s): + for f in s: + f.crc = foldup_blocks(f.block, + binascii.crc32(f.crc)) # # Main @@ -682,10 +787,20 @@ def main(): cliparser = argparse.ArgumentParser(description='VPP API generator') cliparser.add_argument('--pluginpath', default=""), cliparser.add_argument('--includedir', action='append'), - cliparser.add_argument('--input', type=argparse.FileType('r'), - default=sys.stdin) - cliparser.add_argument('--output', nargs='?', type=argparse.FileType('w'), - default=sys.stdout) + if sys.version[0] == '2': + cliparser.add_argument('--input', type=argparse.FileType('r'), + default=sys.stdin) + cliparser.add_argument('--output', nargs='?', + type=argparse.FileType('w'), + default=sys.stdout) + + else: + cliparser.add_argument('--input', + type=argparse.FileType('r', encoding='UTF-8'), + default=sys.stdin) + cliparser.add_argument('--output', nargs='?', + type=argparse.FileType('w', encoding='UTF-8'), + default=sys.stdout) cliparser.add_argument('output_module', nargs='?', default='C') cliparser.add_argument('--debug', action='store_true') @@ -710,61 +825,65 @@ def main(): logging.basicConfig() log = logging.getLogger('vppapigen') - parser = VPPAPI(debug=args.debug, filename=filename, logger=log) - result = parser.parse_file(args.input, log) + parsed_objects = parser.parse_file(args.input, log) # Build a list of objects. Hash of lists. - result = parser.process_imports(result, False) + result = [] + parser.process_imports(parsed_objects, False, result) s = parser.process(result) # Add msg_id field - s['defines'] = add_msg_id(s['defines']) + s['Define'] = add_msg_id(s['Define']) - file_crc = getcrc(s) + # Fold up CRCs + foldup_crcs(s['Define']) # # Debug if args.debug: import pprint - pp = pprint.PrettyPrinter(indent=4) - for t in s['defines']: - pp.pprint([t.name, t.flags, t.block]) - for t in s['typedefs']: + pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr) + for t in s['Define']: pp.pprint([t.name, t.flags, t.block]) + for t in s['types']: + pp.pprint([t.name, t.block]) # # Generate representation # - import imp + from importlib.machinery import SourceFileLoader # Default path pluginpath = '' if not args.pluginpath: cand = [] cand.append(os.path.dirname(os.path.realpath(__file__))) - cand.append(os.path.dirname(os.path.realpath(__file__)) + \ + cand.append(os.path.dirname(os.path.realpath(__file__)) + '/../share/vpp/') for c in cand: c += '/' - if os.path.isfile(c + args.output_module + '.py'): + if os.path.isfile('{}vppapigen_{}.py' + .format(c, args.output_module.lower())): pluginpath = c break else: pluginpath = args.pluginpath + '/' if pluginpath == '': raise Exception('Output plugin not found') - module_path = pluginpath + args.output_module + '.py' + module_path = '{}vppapigen_{}.py'.format(pluginpath, + args.output_module.lower()) try: - plugin = imp.load_source(args.output_module, module_path) - except Exception, err: + plugin = SourceFileLoader(args.output_module, + module_path).load_module() + except Exception as err: raise Exception('Error importing output plugin: {}, {}' .format(module_path, err)) - result = plugin.run(filename, s, file_crc) + result = plugin.run(filename, s) if result: - print (result, file=args.output) + print(result, file=args.output) else: raise Exception('Running plugin failed: {} {}' .format(filename, result))