3 from __future__ import print_function
5 import ply.yacc as yacc
16 # Global dictionary of new types (including enums)
20 def global_type_add(name):
21 '''Add new type to the dictionary of types '''
22 type_name = 'vl_api_' + name + '_t'
23 if type_name in global_types:
24 raise KeyError('Type is already defined: {}'.format(name))
25 global_types[type_name] = True
28 # All your trace are belong to us!
29 def exception_handler(exception_type, exception, traceback):
30 print ("%s: %s" % (exception_type.__name__, exception))
36 class VPPAPILexer(object):
37 def __init__(self, filename):
38 self.filename = filename
50 'typeonly': 'TYPEONLY',
51 'manual_print': 'MANUAL_PRINT',
52 'manual_endian': 'MANUAL_ENDIAN',
53 'dont_trace': 'DONT_TRACE',
54 'autoreply': 'AUTOREPLY',
72 tokens = ['STRING_LITERAL',
73 'ID', 'NUM'] + list(reserved.values())
75 t_ignore_LINE_COMMENT = '//.*'
78 r'0[xX][0-9a-fA-F]+|\d+'
79 base = 16 if t.value.startswith('0x') else 10
80 t.value = int(t.value, base)
84 r'[a-zA-Z_][a-zA-Z_0-9]*'
85 # Check for reserved words
86 t.type = VPPAPILexer.reserved.get(t.value, 'ID')
90 def t_STRING_LITERAL(self, t):
91 r'\"([^\\\n]|(\\.))*?\"'
92 t.value = str(t.value).replace("\"", "")
95 # C or C++ comment (ignore)
96 def t_comment(self, t):
97 r'(/\*(.|\n)*?\*/)|(//.*)'
98 t.lexer.lineno += t.value.count('\n')
100 # Error handling rule
101 def t_error(self, t):
102 raise ParseError("Illegal character '{}' ({})"
103 "in {}: line {}".format(t.value[0],
104 hex(ord(t.value[0])),
109 # Define a rule so we can track line numbers
110 def t_newline(self, t):
112 t.lexer.lineno += len(t.value)
114 literals = ":{}[];=.,"
116 # A string containing ignored characters (spaces and tabs)
120 def __init__(self, caller, reply, events=[], stream=False):
128 def __init__(self, name, flags, block):
132 self.crc = binascii.crc32(str(block)) & 0xffffffff
133 global_type_add(name)
136 return self.name + str(self.flags) + str(self.block)
140 def __init__(self, name, flags, block):
144 self.crc = binascii.crc32(str(block)) & 0xffffffff
145 self.typeonly = False
146 self.dont_trace = False
147 self.manual_print = False
148 self.manual_endian = False
149 self.autoreply = False
150 self.singular = False
154 global_type_add(name)
155 elif f == 'dont_trace':
156 self.dont_trace = True
157 elif f == 'manual_print':
158 self.manual_print = True
159 elif f == 'manual_endian':
160 self.manual_endian = True
161 elif f == 'autoreply':
162 self.autoreply = True
165 if isinstance(b, Option):
166 if b[1] == 'singular' and b[2] == 'true':
171 return self.name + str(self.flags) + str(self.block)
175 def __init__(self, name, block, enumtype='u32'):
177 self.enumtype = enumtype
179 for i, b in enumerate(block):
184 block[i] = [b, count]
187 self.crc = binascii.crc32(str(block)) & 0xffffffff
188 global_type_add(name)
191 return self.name + str(self.block)
195 def __init__(self, filename):
196 self.filename = filename
199 parser = VPPAPI(filename=filename)
200 dirlist = dirlist_get()
203 f = os.path.join(dir, filename)
204 if os.path.exists(f):
207 self.result = parser.parse_file(fd, None)
214 def __init__(self, option):
216 self.crc = binascii.crc32(str(option)) & 0xffffffff
219 return str(self.option)
221 def __getitem__(self, index):
222 return self.option[index]
226 def __init__(self, fieldtype, name, length):
228 self.fieldtype = fieldtype
229 self.fieldname = name
230 if type(length) is str:
231 self.lengthfield = length
235 self.lengthfield = None
238 return str([self.fieldtype, self.fieldname, self.length,
243 def __init__(self, fieldtype, name):
245 self.fieldtype = fieldtype
246 self.fieldname = name
249 return str([self.fieldtype, self.fieldname])
253 """ Coordinates of a syntactic element. Consists of:
256 - (optional) column number, for the Lexer
258 __slots__ = ('file', 'line', 'column', '__weakref__')
260 def __init__(self, file, line, column=None):
266 str = "%s:%s" % (self.file, self.line)
268 str += ":%s" % self.column
272 class ParseError(Exception):
279 class VPPAPIParser(object):
280 tokens = VPPAPILexer.tokens
282 def __init__(self, filename, logger):
283 self.filename = filename
287 def _parse_error(self, msg, coord):
288 raise ParseError("%s: %s" % (coord, msg))
290 def _parse_warning(self, msg, coord):
292 self.logger.warning("%s: %s" % (coord, msg))
294 def _coord(self, lineno, column=None):
297 line=lineno, column=column)
299 def _token_coord(self, p, token_idx):
300 """ Returns the coordinates for the YaccProduction object 'p' indexed
301 with 'token_idx'. The coordinate includes the 'lineno' and
302 'column'. Both follow the lex semantic, starting from 1.
304 last_cr = p.lexer.lexdata.rfind('\n', 0, p.lexpos(token_idx))
307 column = (p.lexpos(token_idx) - (last_cr))
308 return self._coord(p.lineno(token_idx), column)
310 def p_slist(self, p):
327 def p_import(self, p):
328 '''import : IMPORT STRING_LITERAL ';' '''
331 def p_service(self, p):
332 '''service : SERVICE '{' service_statements '}' ';' '''
335 def p_service_statements(self, p):
336 '''service_statements : service_statement
337 | service_statements service_statement'''
343 def p_service_statement(self, p):
344 '''service_statement : RPC ID RETURNS NULL ';'
345 | RPC ID RETURNS ID ';'
346 | RPC ID RETURNS STREAM ID ';'
347 | RPC ID RETURNS ID EVENTS event_list ';' '''
349 # Verify that caller and reply differ
350 self._parse_error('Reply ID ({}) should not be equal to Caller ID'.format(p[2]),
351 self._token_coord(p, 1))
353 p[0] = Service(p[2], p[4], p[6])
355 p[0] = Service(p[2], p[5], stream=True)
357 p[0] = Service(p[2], p[4])
359 def p_event_list(self, p):
360 '''event_list : events
361 | event_list events '''
367 def p_event(self, p):
373 '''enum : ENUM ID '{' enum_statements '}' ';' '''
374 p[0] = Enum(p[2], p[4])
376 def p_enum_type(self, p):
377 ''' enum : ENUM ID ':' enum_size '{' enum_statements '}' ';' '''
379 p[0] = Enum(p[2], p[6], enumtype=p[4])
381 p[0] = Enum(p[2], p[4])
383 def p_enum_size(self, p):
389 def p_define(self, p):
390 '''define : DEFINE ID '{' block_statements_opt '}' ';' '''
392 p[0] = Define(p[2], [], p[4])
394 def p_define_flist(self, p):
395 '''define : flist DEFINE ID '{' block_statements_opt '}' ';' '''
396 p[0] = Define(p[3], p[1], p[5])
398 def p_flist(self, p):
407 '''flag : MANUAL_PRINT
416 def p_typedef(self, p):
417 '''typedef : TYPEDEF ID '{' block_statements_opt '}' ';' '''
418 p[0] = Typedef(p[2], [], p[4])
420 def p_block_statements_opt(self, p):
421 '''block_statements_opt : block_statements'''
424 def p_block_statements(self, p):
425 '''block_statements : block_statement
426 | block_statements block_statement'''
432 def p_block_statement(self, p):
433 '''block_statement : declaration
437 def p_enum_statements(self, p):
438 '''enum_statements : enum_statement
439 | enum_statements enum_statement'''
445 def p_enum_statement(self, p):
446 '''enum_statement : ID '=' NUM ','
453 def p_declaration(self, p):
454 '''declaration : type_specifier ID ';' '''
456 self._parse_error('ERROR')
457 self.fields.append(p[2])
458 p[0] = Field(p[1], p[2])
460 def p_declaration_array(self, p):
461 '''declaration : type_specifier ID '[' NUM ']' ';'
462 | type_specifier ID '[' ID ']' ';' '''
464 return self._parse_error(
465 'array: %s' % p.value,
466 self._coord(lineno=p.lineno))
468 # Make this error later
469 if type(p[4]) is int and p[4] == 0:
470 # XXX: Line number is wrong
471 self._parse_warning('Old Style VLA: {} {}[{}];'
472 .format(p[1], p[2], p[4]),
473 self._token_coord(p, 1))
475 if type(p[4]) is str and p[4] not in self.fields:
476 # Verify that length field exists
477 self._parse_error('Missing length field: {} {}[{}];'
478 .format(p[1], p[2], p[4]),
479 self._token_coord(p, 1))
480 p[0] = Array(p[1], p[2], p[4])
482 def p_option(self, p):
483 '''option : OPTION ID '=' assignee ';' '''
484 p[0] = Option([p[1], p[2], p[4]])
486 def p_assignee(self, p):
493 def p_type_specifier(self, p):
494 '''type_specifier : U8
507 # Do a second pass later to verify that user defined types are defined
508 def p_typedef_specifier(self, p):
509 '''type_specifier : ID '''
510 if p[1] not in global_types:
511 self._parse_error('Undefined type: {}'.format(p[1]),
512 self._token_coord(p, 1))
515 # Error rule for syntax errors
516 def p_error(self, p):
519 'before: %s' % p.value,
520 self._coord(lineno=p.lineno))
522 self._parse_error('At end of input', self.filename)
525 class VPPAPI(object):
527 def __init__(self, debug=False, filename='', logger=None):
528 self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug)
529 self.parser = yacc.yacc(module=VPPAPIParser(filename, logger),
530 tabmodule='vppapigentab', debug=debug)
533 def parse_string(self, code, debug=0, lineno=1):
534 self.lexer.lineno = lineno
535 return self.parser.parse(code, lexer=self.lexer, debug=debug)
537 def parse_file(self, fd, debug=0):
539 return self.parse_string(data, debug=debug)
541 def autoreply_block(self, name):
542 block = [Field('u32', 'context'),
543 Field('i32', 'retval')]
544 return Define(name + '_reply', [], block)
546 def process(self, objs):
556 if isinstance(o, Define):
558 s['typedefs'].append(o)
560 s['defines'].append(o)
562 s['defines'].append(self.autoreply_block(o.name))
563 elif isinstance(o, Option):
564 s['options'][o[1]] = o[2]
565 elif isinstance(o, Enum):
567 elif isinstance(o, Typedef):
568 s['typedefs'].append(o)
569 elif type(o) is list:
571 if isinstance(o2, Service):
572 s['services'].append(o2)
575 msgs = {d.name: d for d in s['defines']}
576 svcs = {s.caller: s for s in s['services']}
577 replies = {s.reply: s for s in s['services']}
581 if service not in msgs:
582 raise ValueError('Service definition refers to unknown message'
583 ' definition: {}'.format(service))
584 if svcs[service].reply != 'null' and svcs[service].reply not in msgs:
585 raise ValueError('Service definition refers to unknown message'
586 ' definition in reply: {}'
587 .format(svcs[service].reply))
588 if service in replies:
589 raise ValueError('Service definition refers to message'
590 ' marked as reply: {}'.format(service))
591 for event in svcs[service].events:
592 if event not in msgs:
593 raise ValueError('Service definition refers to unknown '
594 'event: {} in message: {}'
595 .format(event, service))
596 seen_services[event] = True
598 # Create services implicitly
600 if d in seen_services:
602 if msgs[d].singular is True:
604 #if d.endswith('_counters'):
606 #if d.endswith('_event'):
608 if d.endswith('_reply'):
611 if d[:-6] not in msgs:
612 self.logger.warning('{} missing calling message'
615 if d.endswith('_dump'):
618 if d[:-5]+'_details' in msgs:
619 s['services'].append(Service(d, d[:-5]+'_details',
622 self.logger.error('{} missing details message'
626 if d.endswith('_details'):
627 if d[:-8]+'_dump' not in msgs:
628 self.logger.error('{} missing dump message'
634 if d+'_reply' in msgs:
635 s['services'].append(Service(d, d+'_reply'))
637 self.logger.warning('{} missing reply message ({})'
638 .format(d, d+'_reply'))
639 s['services'].append(Service(d, None))
643 def process_imports(self, objs, in_import):
646 if isinstance(o, Import):
647 return objs + self.process_imports(o.result, True)
649 if isinstance(o, Define) and o.typeonly:
650 imported_objs.append(o)
656 # Add message ids to each message.
659 o.block.insert(0, Field('u16', '_vl_msg_id'))
664 return binascii.crc32(str(s)) & 0xffffffff
670 def dirlist_add(dirs):
673 dirlist = dirlist + dirs
684 cliparser = argparse.ArgumentParser(description='VPP API generator')
685 cliparser.add_argument('--pluginpath', default=""),
686 cliparser.add_argument('--includedir', action='append'),
687 cliparser.add_argument('--input', type=argparse.FileType('r'),
689 cliparser.add_argument('--output', nargs='?', type=argparse.FileType('w'),
692 cliparser.add_argument('output_module', nargs='?', default='C')
693 cliparser.add_argument('--debug', action='store_true')
694 cliparser.add_argument('--show-name', nargs=1)
695 args = cliparser.parse_args()
697 dirlist_add(args.includedir)
699 sys.excepthook = exception_handler
703 filename = args.show_name[0]
704 elif args.input != sys.stdin:
705 filename = args.input.name
710 logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
712 logging.basicConfig()
713 log = logging.getLogger('vppapigen')
716 parser = VPPAPI(debug=args.debug, filename=filename, logger=log)
717 result = parser.parse_file(args.input, log)
719 # Build a list of objects. Hash of lists.
720 result = parser.process_imports(result, False)
721 s = parser.process(result)
724 s['defines'] = add_msg_id(s['defines'])
732 pp = pprint.PrettyPrinter(indent=4)
733 for t in s['defines']:
734 pp.pprint([t.name, t.flags, t.block])
735 for t in s['typedefs']:
736 pp.pprint([t.name, t.flags, t.block])
739 # Generate representation
745 if not args.pluginpath:
747 cand.append(os.path.dirname(os.path.realpath(__file__)))
748 cand.append(os.path.dirname(os.path.realpath(__file__)) + \
752 if os.path.isfile(c + args.output_module + '.py'):
756 pluginpath = args.pluginpath + '/'
758 raise Exception('Output plugin not found')
759 module_path = pluginpath + args.output_module + '.py'
762 plugin = imp.load_source(args.output_module, module_path)
763 except Exception, err:
764 raise Exception('Error importing output plugin: {}, {}'
765 .format(module_path, err))
767 result = plugin.run(filename, s, file_crc)
769 print (result, file=args.output)
771 raise Exception('Running plugin failed: {} {}'
772 .format(filename, result))
775 if __name__ == '__main__':