3 from __future__ import print_function
5 import ply.yacc as yacc
12 # Ensure we don't leave temporary files around
13 sys.dont_write_bytecode = True
19 # Global dictionary of new types (including enums)
23 def global_type_add(name):
24 '''Add new type to the dictionary of types '''
25 type_name = 'vl_api_' + name + '_t'
26 if type_name in global_types:
27 raise KeyError('Type is already defined: {}'.format(name))
28 global_types[type_name] = True
31 # All your trace are belong to us!
32 def exception_handler(exception_type, exception, traceback):
33 print ("%s: %s" % (exception_type.__name__, exception))
39 class VPPAPILexer(object):
40 def __init__(self, filename):
41 self.filename = filename
53 'typeonly': 'TYPEONLY',
54 'manual_print': 'MANUAL_PRINT',
55 'manual_endian': 'MANUAL_ENDIAN',
56 'dont_trace': 'DONT_TRACE',
57 'autoreply': 'AUTOREPLY',
75 tokens = ['STRING_LITERAL',
76 'ID', 'NUM'] + list(reserved.values())
78 t_ignore_LINE_COMMENT = '//.*'
81 r'0[xX][0-9a-fA-F]+|\d+'
82 base = 16 if t.value.startswith('0x') else 10
83 t.value = int(t.value, base)
87 r'[a-zA-Z_][a-zA-Z_0-9]*'
88 # Check for reserved words
89 t.type = VPPAPILexer.reserved.get(t.value, 'ID')
93 def t_STRING_LITERAL(self, t):
94 r'\"([^\\\n]|(\\.))*?\"'
95 t.value = str(t.value).replace("\"", "")
98 # C or C++ comment (ignore)
99 def t_comment(self, t):
100 r'(/\*(.|\n)*?\*/)|(//.*)'
101 t.lexer.lineno += t.value.count('\n')
103 # Error handling rule
104 def t_error(self, t):
105 raise ParseError("Illegal character '{}' ({})"
106 "in {}: line {}".format(t.value[0],
107 hex(ord(t.value[0])),
112 # Define a rule so we can track line numbers
113 def t_newline(self, t):
115 t.lexer.lineno += len(t.value)
117 literals = ":{}[];=.,"
119 # A string containing ignored characters (spaces and tabs)
123 def __init__(self, caller, reply, events=[], stream=False):
131 def __init__(self, name, flags, block):
135 self.crc = binascii.crc32(str(block)) & 0xffffffff
136 global_type_add(name)
139 return self.name + str(self.flags) + str(self.block)
143 def __init__(self, name, flags, block):
147 self.crc = binascii.crc32(str(block)) & 0xffffffff
148 self.typeonly = False
149 self.dont_trace = False
150 self.manual_print = False
151 self.manual_endian = False
152 self.autoreply = False
153 self.singular = False
157 global_type_add(name)
158 elif f == 'dont_trace':
159 self.dont_trace = True
160 elif f == 'manual_print':
161 self.manual_print = True
162 elif f == 'manual_endian':
163 self.manual_endian = True
164 elif f == 'autoreply':
165 self.autoreply = True
168 if isinstance(b, Option):
169 if b[1] == 'singular' and b[2] == 'true':
174 return self.name + str(self.flags) + str(self.block)
178 def __init__(self, name, block, enumtype='u32'):
180 self.enumtype = enumtype
182 for i, b in enumerate(block):
187 block[i] = [b, count]
190 self.crc = binascii.crc32(str(block)) & 0xffffffff
191 global_type_add(name)
194 return self.name + str(self.block)
198 def __init__(self, filename):
199 self.filename = filename
202 parser = VPPAPI(filename=filename)
203 dirlist = dirlist_get()
206 f = os.path.join(dir, filename)
207 if os.path.exists(f):
210 self.result = parser.parse_file(fd, None)
217 def __init__(self, option):
219 self.crc = binascii.crc32(str(option)) & 0xffffffff
222 return str(self.option)
224 def __getitem__(self, index):
225 return self.option[index]
229 def __init__(self, fieldtype, name, length):
231 self.fieldtype = fieldtype
232 self.fieldname = name
233 if type(length) is str:
234 self.lengthfield = length
238 self.lengthfield = None
241 return str([self.fieldtype, self.fieldname, self.length,
246 def __init__(self, fieldtype, name):
248 self.fieldtype = fieldtype
249 self.fieldname = name
252 return str([self.fieldtype, self.fieldname])
256 """ Coordinates of a syntactic element. Consists of:
259 - (optional) column number, for the Lexer
261 __slots__ = ('file', 'line', 'column', '__weakref__')
263 def __init__(self, file, line, column=None):
269 str = "%s:%s" % (self.file, self.line)
271 str += ":%s" % self.column
275 class ParseError(Exception):
282 class VPPAPIParser(object):
283 tokens = VPPAPILexer.tokens
285 def __init__(self, filename, logger):
286 self.filename = filename
290 def _parse_error(self, msg, coord):
291 raise ParseError("%s: %s" % (coord, msg))
293 def _parse_warning(self, msg, coord):
295 self.logger.warning("%s: %s" % (coord, msg))
297 def _coord(self, lineno, column=None):
300 line=lineno, column=column)
302 def _token_coord(self, p, token_idx):
303 """ Returns the coordinates for the YaccProduction object 'p' indexed
304 with 'token_idx'. The coordinate includes the 'lineno' and
305 'column'. Both follow the lex semantic, starting from 1.
307 last_cr = p.lexer.lexdata.rfind('\n', 0, p.lexpos(token_idx))
310 column = (p.lexpos(token_idx) - (last_cr))
311 return self._coord(p.lineno(token_idx), column)
313 def p_slist(self, p):
330 def p_import(self, p):
331 '''import : IMPORT STRING_LITERAL ';' '''
334 def p_service(self, p):
335 '''service : SERVICE '{' service_statements '}' ';' '''
338 def p_service_statements(self, p):
339 '''service_statements : service_statement
340 | service_statements service_statement'''
346 def p_service_statement(self, p):
347 '''service_statement : RPC ID RETURNS NULL ';'
348 | RPC ID RETURNS ID ';'
349 | RPC ID RETURNS STREAM ID ';'
350 | RPC ID RETURNS ID EVENTS event_list ';' '''
352 # Verify that caller and reply differ
353 self._parse_error('Reply ID ({}) should not be equal to Caller ID'.format(p[2]),
354 self._token_coord(p, 1))
356 p[0] = Service(p[2], p[4], p[6])
358 p[0] = Service(p[2], p[5], stream=True)
360 p[0] = Service(p[2], p[4])
362 def p_event_list(self, p):
363 '''event_list : events
364 | event_list events '''
370 def p_event(self, p):
376 '''enum : ENUM ID '{' enum_statements '}' ';' '''
377 p[0] = Enum(p[2], p[4])
379 def p_enum_type(self, p):
380 ''' enum : ENUM ID ':' enum_size '{' enum_statements '}' ';' '''
382 p[0] = Enum(p[2], p[6], enumtype=p[4])
384 p[0] = Enum(p[2], p[4])
386 def p_enum_size(self, p):
392 def p_define(self, p):
393 '''define : DEFINE ID '{' block_statements_opt '}' ';' '''
395 p[0] = Define(p[2], [], p[4])
397 def p_define_flist(self, p):
398 '''define : flist DEFINE ID '{' block_statements_opt '}' ';' '''
399 p[0] = Define(p[3], p[1], p[5])
401 def p_flist(self, p):
410 '''flag : MANUAL_PRINT
419 def p_typedef(self, p):
420 '''typedef : TYPEDEF ID '{' block_statements_opt '}' ';' '''
421 p[0] = Typedef(p[2], [], p[4])
423 def p_block_statements_opt(self, p):
424 '''block_statements_opt : block_statements'''
427 def p_block_statements(self, p):
428 '''block_statements : block_statement
429 | block_statements block_statement'''
435 def p_block_statement(self, p):
436 '''block_statement : declaration
440 def p_enum_statements(self, p):
441 '''enum_statements : enum_statement
442 | enum_statements enum_statement'''
448 def p_enum_statement(self, p):
449 '''enum_statement : ID '=' NUM ','
456 def p_declaration(self, p):
457 '''declaration : type_specifier ID ';' '''
459 self._parse_error('ERROR')
460 self.fields.append(p[2])
461 p[0] = Field(p[1], p[2])
463 def p_declaration_array(self, p):
464 '''declaration : type_specifier ID '[' NUM ']' ';'
465 | type_specifier ID '[' ID ']' ';' '''
467 return self._parse_error(
468 'array: %s' % p.value,
469 self._coord(lineno=p.lineno))
471 # Make this error later
472 if type(p[4]) is int and p[4] == 0:
473 # XXX: Line number is wrong
474 self._parse_warning('Old Style VLA: {} {}[{}];'
475 .format(p[1], p[2], p[4]),
476 self._token_coord(p, 1))
478 if type(p[4]) is str and p[4] not in self.fields:
479 # Verify that length field exists
480 self._parse_error('Missing length field: {} {}[{}];'
481 .format(p[1], p[2], p[4]),
482 self._token_coord(p, 1))
483 p[0] = Array(p[1], p[2], p[4])
485 def p_option(self, p):
486 '''option : OPTION ID '=' assignee ';' '''
487 p[0] = Option([p[1], p[2], p[4]])
489 def p_assignee(self, p):
496 def p_type_specifier(self, p):
497 '''type_specifier : U8
510 # Do a second pass later to verify that user defined types are defined
511 def p_typedef_specifier(self, p):
512 '''type_specifier : ID '''
513 if p[1] not in global_types:
514 self._parse_error('Undefined type: {}'.format(p[1]),
515 self._token_coord(p, 1))
518 # Error rule for syntax errors
519 def p_error(self, p):
522 'before: %s' % p.value,
523 self._coord(lineno=p.lineno))
525 self._parse_error('At end of input', self.filename)
528 class VPPAPI(object):
530 def __init__(self, debug=False, filename='', logger=None):
531 self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug)
532 self.parser = yacc.yacc(module=VPPAPIParser(filename, logger),
533 write_tables=False, debug=debug)
536 def parse_string(self, code, debug=0, lineno=1):
537 self.lexer.lineno = lineno
538 return self.parser.parse(code, lexer=self.lexer, debug=debug)
540 def parse_file(self, fd, debug=0):
542 return self.parse_string(data, debug=debug)
544 def autoreply_block(self, name):
545 block = [Field('u32', 'context'),
546 Field('i32', 'retval')]
547 return Define(name + '_reply', [], block)
549 def process(self, objs):
559 if isinstance(o, Define):
561 s['typedefs'].append(o)
563 s['defines'].append(o)
565 s['defines'].append(self.autoreply_block(o.name))
566 elif isinstance(o, Option):
567 s['options'][o[1]] = o[2]
568 elif isinstance(o, Enum):
570 elif isinstance(o, Typedef):
571 s['typedefs'].append(o)
572 elif type(o) is list:
574 if isinstance(o2, Service):
575 s['services'].append(o2)
578 msgs = {d.name: d for d in s['defines']}
579 svcs = {s.caller: s for s in s['services']}
580 replies = {s.reply: s for s in s['services']}
584 if service not in msgs:
585 raise ValueError('Service definition refers to unknown message'
586 ' definition: {}'.format(service))
587 if svcs[service].reply != 'null' and svcs[service].reply not in msgs:
588 raise ValueError('Service definition refers to unknown message'
589 ' definition in reply: {}'
590 .format(svcs[service].reply))
591 if service in replies:
592 raise ValueError('Service definition refers to message'
593 ' marked as reply: {}'.format(service))
594 for event in svcs[service].events:
595 if event not in msgs:
596 raise ValueError('Service definition refers to unknown '
597 'event: {} in message: {}'
598 .format(event, service))
599 seen_services[event] = True
601 # Create services implicitly
603 if d in seen_services:
605 if msgs[d].singular is True:
607 if d.endswith('_reply'):
610 if d[:-6] not in msgs:
611 self.logger.warning('{} missing calling message'
614 if d.endswith('_dump'):
617 if d[:-5]+'_details' in msgs:
618 s['services'].append(Service(d, d[:-5]+'_details',
621 self.logger.error('{} missing details message'
625 if d.endswith('_details'):
626 if d[:-8]+'_dump' not in msgs:
627 self.logger.error('{} missing dump message'
633 if d+'_reply' in msgs:
634 s['services'].append(Service(d, d+'_reply'))
636 raise ValueError('{} missing reply message ({}) or service definition'
637 .format(d, d+'_reply'))
641 def process_imports(self, objs, in_import):
644 if isinstance(o, Import):
645 return objs + self.process_imports(o.result, True)
647 if isinstance(o, Define) and o.typeonly:
648 imported_objs.append(o)
654 # Add message ids to each message.
657 o.block.insert(0, Field('u16', '_vl_msg_id'))
662 return binascii.crc32(str(s)) & 0xffffffff
668 def dirlist_add(dirs):
671 dirlist = dirlist + dirs
682 cliparser = argparse.ArgumentParser(description='VPP API generator')
683 cliparser.add_argument('--pluginpath', default=""),
684 cliparser.add_argument('--includedir', action='append'),
685 cliparser.add_argument('--input', type=argparse.FileType('r'),
687 cliparser.add_argument('--output', nargs='?', type=argparse.FileType('w'),
690 cliparser.add_argument('output_module', nargs='?', default='C')
691 cliparser.add_argument('--debug', action='store_true')
692 cliparser.add_argument('--show-name', nargs=1)
693 args = cliparser.parse_args()
695 dirlist_add(args.includedir)
697 sys.excepthook = exception_handler
701 filename = args.show_name[0]
702 elif args.input != sys.stdin:
703 filename = args.input.name
708 logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
710 logging.basicConfig()
711 log = logging.getLogger('vppapigen')
714 parser = VPPAPI(debug=args.debug, filename=filename, logger=log)
715 result = parser.parse_file(args.input, log)
717 # Build a list of objects. Hash of lists.
718 result = parser.process_imports(result, False)
719 s = parser.process(result)
722 s['defines'] = add_msg_id(s['defines'])
730 pp = pprint.PrettyPrinter(indent=4)
731 for t in s['defines']:
732 pp.pprint([t.name, t.flags, t.block])
733 for t in s['typedefs']:
734 pp.pprint([t.name, t.flags, t.block])
737 # Generate representation
743 if not args.pluginpath:
745 cand.append(os.path.dirname(os.path.realpath(__file__)))
746 cand.append(os.path.dirname(os.path.realpath(__file__)) + \
750 if os.path.isfile(c + args.output_module + '.py'):
754 pluginpath = args.pluginpath + '/'
756 raise Exception('Output plugin not found')
757 module_path = pluginpath + args.output_module + '.py'
760 plugin = imp.load_source(args.output_module, module_path)
761 except Exception, err:
762 raise Exception('Error importing output plugin: {}, {}'
763 .format(module_path, err))
765 result = plugin.run(filename, s, file_crc)
767 print (result, file=args.output)
769 raise Exception('Running plugin failed: {} {}'
770 .format(filename, result))
773 if __name__ == '__main__':