gerrit.fd Code Review - vpp.git/blob - src/tools/vppapigen/vppapigen.py

1 #!/usr/bin/env python3

3 import sys

4 import argparse

5 import keyword

6 import logging

7 import binascii

8 import os

9 from subprocess import Popen, PIPE

10 import ply.lex as lex

11 import ply.yacc as yacc

12 from io import TextIOWrapper

14 assert sys.version_info >= (3, 5), "Not supported Python version: {}".format(

15 sys.version

16 )

17 log = logging.getLogger("vppapigen")

19 # Ensure we don't leave temporary files around

20 sys.dont_write_bytecode = True

22 #

23 # VPP API language

24 #

26 # Global dictionary of new types (including enums)

27 global_types = {}

29 seen_imports = {}

32 def global_type_add(name, obj):

33 """Add new type to the dictionary of types"""

34 type_name = "vl_api_" + name + "_t"

35 if type_name in global_types:

36 raise KeyError("Attempted redefinition of {!r} with {!r}.".format(name, obj))

37 global_types[type_name] = obj

40 # All your trace are belong to us!

41 def exception_handler(exception_type, exception, traceback):

42 print("%s: %s" % (exception_type.__name__, exception))

45 #

46 # Lexer

47 #

48 class VPPAPILexer:

49 def __init__(self, filename):

50 self.filename = filename

52 reserved = {

53 "service": "SERVICE",

54 "rpc": "RPC",

55 "returns": "RETURNS",

56 "null": "NULL",

57 "stream": "STREAM",

58 "events": "EVENTS",

59 "define": "DEFINE",

60 "typedef": "TYPEDEF",

61 "enum": "ENUM",

62 "enumflag": "ENUMFLAG",

63 "typeonly": "TYPEONLY",

64 "manual_print": "MANUAL_PRINT",

65 "manual_endian": "MANUAL_ENDIAN",

66 "dont_trace": "DONT_TRACE",

67 "autoreply": "AUTOREPLY",

68 "autoendian": "AUTOENDIAN",

69 "option": "OPTION",

70 "u8": "U8",

71 "u16": "U16",

72 "u32": "U32",

73 "u64": "U64",

74 "i8": "I8",

75 "i16": "I16",

76 "i32": "I32",

77 "i64": "I64",

78 "f64": "F64",

79 "bool": "BOOL",

80 "string": "STRING",

81 "import": "IMPORT",

82 "true": "TRUE",

83 "false": "FALSE",

84 "union": "UNION",

85 "counters": "COUNTERS",

86 "paths": "PATHS",

87 "units": "UNITS",

88 "severity": "SEVERITY",

89 "type": "TYPE",

90 "description": "DESCRIPTION",

91 }

93 tokens = ["STRING_LITERAL", "COMMENT", "ID", "NUM"] + list(reserved.values())

95 t_ignore_LINE_COMMENT = "//.*"

97 def t_FALSE(self, t):

98 r"false"

99 t.value = False

100 return t

101

102 def t_TRUE(self, t):

103 r"false"

104 t.value = True

105 return t

106

107 def t_NUM(self, t):

108 r"0[xX][0-9a-fA-F]+|-?\d+\.?\d*"

109 base = 16 if t.value.startswith("0x") else 10

110 if "." in t.value:

111 t.value = float(t.value)

112 else:

113 t.value = int(t.value, base)

114 return t

115

116 def t_ID(self, t):

117 r"[a-zA-Z_][a-zA-Z_0-9]*"

118 # Check for reserved words

119 t.type = VPPAPILexer.reserved.get(t.value, "ID")

120 return t

121

122 # C string

123 def t_STRING_LITERAL(self, t):

124 r"\"([^\\\n]|(\\.))*?\" "

125 t.value = str(t.value).replace('"', "")

126 return t

127

128 # C or C++ comment (ignore)

129 def t_COMMENT(self, t):

130 r"(/\*(.|\n)*?\*/)|(//.*)"

131 t.lexer.lineno += t.value.count("\n")

132 return t

133

134 # Error handling rule

135 def t_error(self, t):

136 raise ParseError(

137 "Illegal character '{}' ({})"

138 "in {}: line {}".format(

139 t.value[0], hex(ord(t.value[0])), self.filename, t.lexer.lineno

140 )

141 )

142

143 # Define a rule so we can track line numbers

144 def t_newline(self, t):

145 r"\n+"

146 t.lexer.lineno += len(t.value)

147

148 literals = ":{}[];=.,"

149

150 # A string containing ignored characters (spaces and tabs)

151 t_ignore = " \t"

152

153

154 def vla_mark_length_field(block):

155 if isinstance(block[-1], Array):

156 lengthfield = block[-1].lengthfield

157 for b in block:

158 if b.fieldname == lengthfield:

159 b.is_lengthfield = True

160

161

162 def vla_is_last_check(name, block):

163 vla = False

164 for i, b in enumerate(block):

165 if isinstance(b, Array) and b.vla:

166 vla = True

167 if i + 1 < len(block):

168 raise ValueError(

169 'VLA field "{}" must be the last field in message "{}"'.format(

170 b.fieldname, name

171 )

172 )

173 elif b.fieldtype.startswith("vl_api_"):

174 if global_types[b.fieldtype].vla:

175 vla = True

176 if i + 1 < len(block):

177 raise ValueError(

178 'VLA field "{}" must be the last '

179 'field in message "{}"'.format(b.fieldname, name)

180 )

181 elif b.fieldtype == "string" and b.length == 0:

182 vla = True

183 if i + 1 < len(block):

184 raise ValueError(

185 'VLA field "{}" must be the last '

186 'field in message "{}"'.format(b.fieldname, name)

187 )

188 return vla

189

190

191 class Processable:

192 type = "<Invalid>"

193

194 def process(self, result): # -> Dict

195 result[self.type].append(self)

196

197

198 class Service(Processable):

199 type = "Service"

200

201 def __init__(self, caller, reply, events=None, stream_message=None, stream=False):

202 self.caller = caller

203 self.reply = reply

204 self.stream = stream

205 self.stream_message = stream_message

206 self.events = [] if events is None else events

207

208

209 class Typedef(Processable):

210 type = "Typedef"

211

212 def __init__(self, name, flags, block):

213 self.name = name

214 self.flags = flags

215 self.block = block

216 self.crc = str(block).encode()

217 self.manual_print = False

218 self.manual_endian = False

219 for f in flags:

220 if f == "manual_print":

221 self.manual_print = True

222 elif f == "manual_endian":

223 self.manual_endian = True

224 global_type_add(name, self)

225

226 self.vla = vla_is_last_check(name, block)

227 vla_mark_length_field(self.block)

228

229 def process(self, result):

230 result["types"].append(self)

231

232 def __repr__(self):

233 return self.name + str(self.flags) + str(self.block)

234

235

236 class Using(Processable):

237 type = "Using"

238

239 def __init__(self, name, flags, alias):

240 self.name = name

241 self.vla = False

242 self.block = []

243 self.manual_print = True

244 self.manual_endian = True

245

246 self.manual_print = False

247 self.manual_endian = False

248 for f in flags:

249 if f == "manual_print":

250 self.manual_print = True

251 elif f == "manual_endian":

252 self.manual_endian = True

253

254 if isinstance(alias, Array):

255 a = {"type": alias.fieldtype, "length": alias.length}

256 else:

257 a = {"type": alias.fieldtype}

258 self.alias = a

259 self.using = alias

260

261 #

262 # Should have been:

263 # self.crc = str(alias).encode()

264 # but to be backwards compatible use the block ([])

265 #

266 self.crc = str(self.block).encode()

267 global_type_add(name, self)

268

269 def process(self, result): # -> Dict

270 result["types"].append(self)

271

272 def __repr__(self):

273 return self.name + str(self.alias)

274

275

276 class Union(Processable):

277 type = "Union"

278

279 def __init__(self, name, flags, block):

280 self.manual_print = False

281 self.manual_endian = False

282 self.name = name

283

284 for f in flags:

285 if f == "manual_print":

286 self.manual_print = True

287 elif f == "manual_endian":

288 self.manual_endian = True

289

290 self.block = block

291 self.crc = str(block).encode()

292 self.vla = vla_is_last_check(name, block)

293

294 global_type_add(name, self)

295

296 def process(self, result):

297 result["types"].append(self)

298

299 def __repr__(self):

300 return str(self.block)

301

302

303 class Define(Processable):

304 type = "Define"

305

306 def __init__(self, name, flags, block, comment=None):

307 self.name = name

308 self.flags = flags

309 self.block = block

310 self.dont_trace = False

311 self.manual_print = False

312 self.manual_endian = False

313 self.autoreply = False

314 self.autoendian = 0

315 self.options = {}

316 self.comment = comment

317 for f in flags:

318 if f == "dont_trace":

319 self.dont_trace = True

320 elif f == "manual_print":

321 self.manual_print = True

322 elif f == "manual_endian":

323 self.manual_endian = True

324 elif f == "autoreply":

325 self.autoreply = True

326 elif f == "autoendian":

327 self.autoendian = 1

328

329 remove = []

330 for b in block:

331 if isinstance(b, Option):

332 self.options[b.option] = b.value

333 remove.append(b)

334

335 block = [x for x in block if x not in remove]

336 self.block = block

337 self.vla = vla_is_last_check(name, block)

338 vla_mark_length_field(self.block)

339

340 self.crc = str(block).encode()

341

342 def autoreply_block(self, name, parent):

343 block = [Field("u32", "context"), Field("i32", "retval")]

344 # inherit the parent's options

345 for k, v in parent.options.items():

346 block.append(Option(k, v))

347 return Define(name + "_reply", [], block)

348

349 def process(self, result): # -> Dict

350 tname = self.__class__.__name__

351 result[tname].append(self)

352 if self.autoreply:

353 result[tname].append(self.autoreply_block(self.name, self))

354

355 def __repr__(self):

356 return self.name + str(self.flags) + str(self.block)

357

358

359 class Enum(Processable):

360 type = "Enum"

361

362 def __init__(self, name, block, enumtype="u32"):

363 self.name = name

364 self.enumtype = enumtype

365 self.vla = False

366 self.manual_print = False

367

368 count = -1

369 block2 = []

370 block3 = []

371 bc_set = False

372

373 for b in block:

374 if "value" in b:

375 count = b["value"]

376 else:

377 count += 1

378 block2.append([b["id"], count])

379 try:

380 if b["option"]["backwards_compatible"]:

381 pass

382 bc_set = True

383 except KeyError:

384 block3.append([b["id"], count])

385 if bc_set:

386 raise ValueError(

387 "Backward compatible enum must "

388 "be last {!r} {!r}".format(name, b["id"])

389 )

390 self.block = block2

391 self.crc = str(block3).encode()

392 global_type_add(name, self)

393

394 def process(self, result):

395 result["types"].append(self)

396

397 def __repr__(self):

398 return self.name + str(self.block)

399

400

401 class EnumFlag(Enum):

402 type = "EnumFlag"

403

404 def __init__(self, name, block, enumtype="u32"):

405 super(EnumFlag, self).__init__(name, block, enumtype)

406

407 for b in self.block:

408 if bin(b[1])[2:].count("1") > 1:

409 raise TypeError(

410 "%s is not a flag enum. No element in a "

411 "flag enum may have more than a "

412 "single bit set." % self.name

413 )

414

415

416 class Import(Processable):

417 type = "Import"

418 _initialized = False

419

420 def __new__(cls, *args, **kwargs):

421 if args[0] not in seen_imports:

422 instance = super().__new__(cls)

423 instance._initialized = False

424 seen_imports[args[0]] = instance

425

426 return seen_imports[args[0]]

427

428 def __init__(self, filename, revision):

429 if self._initialized:

430 return

431 self.filename = filename

432 # Deal with imports

433 parser = VPPAPI(filename=filename, revision=revision)

434 dirlist = dirlist_get()

435 f = filename

436 for dir in dirlist:

437 f = os.path.join(dir, filename)

438 if os.path.exists(f):

439 break

440 self.result = parser.parse_filename(f, None)

441 self._initialized = True

442

443 def __repr__(self):

444 return self.filename

445

446

447 class Option(Processable):

448 type = "Option"

449

450 def __init__(self, option, value=None):

451 self.option = option

452 self.value = value

453 self.crc = str(option).encode()

454

455 def process(self, result): # -> Dict

456 result[self.type][self.option] = self.value

457

458 def __repr__(self):

459 return str(self.option)

460

461 def __getitem__(self, index):

462 return self.option[index]

463

464

465 class Array(Processable):

466 type = "Array"

467

468 def __init__(self, fieldtype, name, length, modern_vla=False):

469 self.fieldtype = fieldtype

470 self.fieldname = name

471 self.modern_vla = modern_vla

472 if type(length) is str:

473 self.lengthfield = length

474 self.length = 0

475 self.vla = True

476 else:

477 self.length = length

478 self.lengthfield = None

479 self.vla = False

480

481 def __repr__(self):

482 return str([self.fieldtype, self.fieldname, self.length, self.lengthfield])

483

484

485 class Field(Processable):

486 type = "Field"

487

488 def __init__(self, fieldtype, name, limit=None):

489 # limit field has been expanded to an options dict.

490

491 self.fieldtype = fieldtype

492 self.is_lengthfield = False

493

494 if self.fieldtype == "string":

495 raise ValueError("The string type {!r} is an " "array type ".format(name))

496

497 if name in keyword.kwlist:

498 raise ValueError(

499 "Fieldname {!r} is a python keyword and is not "

500 "accessible via the python API. ".format(name)

501 )

502 self.fieldname = name

503 self.limit = limit

504

505 def __repr__(self):

506 return str([self.fieldtype, self.fieldname])

507

508

509 class Counter(Processable):

510 type = "Counter"

511

512 def __init__(self, path, counter):

513 self.name = path

514 self.block = counter

515

516 def process(self, result): # -> Dict

517 result["Counters"].append(self)

518

519

520 class Paths(Processable):

521 type = "Paths"

522

523 def __init__(self, pathset):

524 self.paths = pathset

525

526 def __repr__(self):

527 return "%s(paths=%s)" % (self.__class__.__name__, self.paths)

528

529

530 class Coord:

531 """Coordinates of a syntactic element. Consists of:

532 - File name

533 - Line number

534 - (optional) column number, for the Lexer

535 """

536

537 __slots__ = ("file", "line", "column", "__weakref__")

538

539 def __init__(self, file, line, column=None):

540 self.file = file

541 self.line = line

542 self.column = column

543

544 def __str__(self):

545 str = "%s:%s" % (self.file, self.line)

546 if self.column:

547 str += ":%s" % self.column

548 return str

549

550

551 class ParseError(Exception):

552 pass

553

554

555 #

556 # Grammar rules

557 #

558 class VPPAPIParser:

559 tokens = VPPAPILexer.tokens

560

561 def __init__(self, filename, logger, revision=None):

562 self.filename = filename

563 self.logger = logger

564 self.fields = []

565 self.revision = revision

566 self.last_comment = None

567

568 def _parse_error(self, msg, coord):

569 raise ParseError("%s: %s" % (coord, msg))

570

571 def _parse_warning(self, msg, coord):

572 if self.logger:

573 self.logger.warning("%s: %s" % (coord, msg))

574

575 def _coord(self, lineno, column=None):

576 return Coord(file=self.filename, line=lineno, column=column)

577

578 def _token_coord(self, p, token_idx):

579 """Returns the coordinates for the YaccProduction object 'p' indexed

580 with 'token_idx'. The coordinate includes the 'lineno' and

581 'column'. Both follow the lex semantic, starting from 1.

582 """

583 last_cr = p.lexer.lexdata.rfind("\n", 0, p.lexpos(token_idx))

584 if last_cr < 0:

585 last_cr = -1

586 column = p.lexpos(token_idx) - (last_cr)

587 return self._coord(p.lineno(token_idx), column)

588

589 def p_slist(self, p):

590 """slist : stmt

591 | slist stmt"""

592 if len(p) == 2:

593 p[0] = [p[1]]

594 else:

595 p[0] = p[1] + [p[2]]

596

597 def p_stmt(self, p):

598 """stmt : define

599 | typedef

600 | option

601 | import

602 | enum

603 | enumflag

604 | union

605 | service

606 | paths

607 | comment

608 | counters"""

609 p[0] = p[1]

610

611 def p_import(self, p):

612 """import : IMPORT STRING_LITERAL ';'"""

613 p[0] = Import(p[2], revision=self.revision)

614

615 def p_path_elements(self, p):

616 """path_elements : path_element

617 | path_elements path_element"""

618 if len(p) == 2:

619 p[0] = p[1]

620 else:

621 if type(p[1]) is dict:

622 p[0] = [p[1], p[2]]

623 else:

624 p[0] = p[1] + [p[2]]

625

626 def p_path_element(self, p):

627 """path_element : STRING_LITERAL STRING_LITERAL ';'"""

628 p[0] = {"path": p[1], "counter": p[2]}

629

630 def p_paths(self, p):

631 """paths : PATHS '{' path_elements '}' ';'"""

632 p[0] = Paths(p[3])

633

634 def p_counters(self, p):

635 """counters : COUNTERS ID '{' counter_elements '}' ';'"""

636 p[0] = Counter(p[2], p[4])

637

638 def p_counter_elements(self, p):

639 """counter_elements : counter_element

640 | counter_elements counter_element"""

641 if len(p) == 2:

642 p[0] = [p[1]]

643 else:

644 if type(p[1]) is dict:

645 p[0] = [p[1], p[2]]

646 else:

647 p[0] = p[1] + [p[2]]

648

649 def p_counter_element(self, p):

650 """counter_element : ID '{' counter_statements '}' ';'"""

651 p[0] = {**{"name": p[1]}, **p[3]}

652

653 def p_counter_statements(self, p):

654 """counter_statements : counter_statement

655 | counter_statements counter_statement"""

656 if len(p) == 2:

657 p[0] = p[1]

658 else:

659 p[0] = {**p[1], **p[2]}

660

661 def p_counter_statement(self, p):

662 """counter_statement : SEVERITY ID ';'

663 | UNITS STRING_LITERAL ';'

664 | DESCRIPTION STRING_LITERAL ';'

665 | TYPE ID ';'"""

666 p[0] = {p[1]: p[2]}

667

668 def p_service(self, p):

669 """service : SERVICE '{' service_statements '}' ';'"""

670 p[0] = p[3]

671

672 def p_service_statements(self, p):

673 """service_statements : service_statement

674 | service_statements service_statement"""

675 if len(p) == 2:

676 p[0] = [p[1]]

677 else:

678 p[0] = p[1] + [p[2]]

679

680 def p_service_statement(self, p):

681 """service_statement : RPC ID RETURNS NULL ';'

682 | RPC ID RETURNS ID ';'

683 | RPC ID RETURNS STREAM ID ';'

684 | RPC ID RETURNS ID EVENTS event_list ';'"""

685 if p[2] == p[4]:

686 # Verify that caller and reply differ

687 self._parse_error(

688 "Reply ID ({}) should not be equal to Caller ID".format(p[2]),

689 self._token_coord(p, 1),

690 )

691 if len(p) == 8:

692 p[0] = Service(p[2], p[4], p[6])

693 elif len(p) == 7:

694 p[0] = Service(p[2], p[5], stream=True)

695 else:

696 p[0] = Service(p[2], p[4])

697

698 def p_service_statement2(self, p):

699 """service_statement : RPC ID RETURNS ID STREAM ID ';'"""

700 p[0] = Service(p[2], p[4], stream_message=p[6], stream=True)

701

702 def p_event_list(self, p):

703 """event_list : events

704 | event_list events"""

705 if len(p) == 2:

706 p[0] = [p[1]]

707 else:

708 p[0] = p[1] + [p[2]]

709

710 def p_event(self, p):

711 """events : ID

712 | ID ','"""

713 p[0] = p[1]

714

715 def p_enum(self, p):

716 """enum : ENUM ID '{' enum_statements '}' ';'"""

717 p[0] = Enum(p[2], p[4])

718

719 def p_enum_type(self, p):

720 """enum : ENUM ID ':' enum_size '{' enum_statements '}' ';'"""

721 if len(p) == 9:

722 p[0] = Enum(p[2], p[6], enumtype=p[4])

723 else:

724 p[0] = Enum(p[2], p[4])

725

726 def p_enumflag(self, p):

727 """enumflag : ENUMFLAG ID '{' enum_statements '}' ';'"""

728 p[0] = EnumFlag(p[2], p[4])

729

730 def p_enumflag_type(self, p):

731 """enumflag : ENUMFLAG ID ':' enumflag_size '{' enum_statements '}' ';'""" # noqa : E502

732 if len(p) == 9:

733 p[0] = EnumFlag(p[2], p[6], enumtype=p[4])

734 else:

735 p[0] = EnumFlag(p[2], p[4])

736

737 def p_enum_size(self, p):

738 """enum_size : U8

739 | U16

740 | U32

741 | I8

742 | I16

743 | I32"""

744 p[0] = p[1]

745

746 def p_enumflag_size(self, p):

747 """enumflag_size : U8

748 | U16

749 | U32"""

750 p[0] = p[1]

751

752 def p_define(self, p):

753 """define : DEFINE ID '{' block_statements_opt '}' ';'"""

754 self.fields = []

755 p[0] = Define(p[2], [], p[4], self.last_comment)

756 self.last_comment = None

757

758 def p_define_flist(self, p):

759 """define : flist DEFINE ID '{' block_statements_opt '}' ';'"""

760 # Legacy typedef

761 if "typeonly" in p[1]:

762 self._parse_error(

763 "legacy typedef. use typedef: {} {}[{}];".format(p[1], p[2], p[4]),

764 self._token_coord(p, 1),

765 )

766 else:

767 p[0] = Define(p[3], p[1], p[5], self.last_comment)

768 self.last_comment = None

769

770 def p_flist(self, p):

771 """flist : flag

772 | flist flag"""

773 if len(p) == 2:

774 p[0] = [p[1]]

775 else:

776 p[0] = p[1] + [p[2]]

777

778 def p_flag(self, p):

779 """flag : MANUAL_PRINT

780 | MANUAL_ENDIAN

781 | DONT_TRACE

782 | TYPEONLY

783 | AUTOENDIAN

784 | AUTOREPLY"""

785 if len(p) == 1:

786 return

787 p[0] = p[1]

788

789 def p_typedef(self, p):

790 """typedef : TYPEDEF ID '{' block_statements_opt '}' ';'"""

791 p[0] = Typedef(p[2], [], p[4])

792

793 def p_typedef_flist(self, p):

794 """typedef : flist TYPEDEF ID '{' block_statements_opt '}' ';'"""

795 p[0] = Typedef(p[3], p[1], p[5])

796

797 def p_typedef_alias(self, p):

798 """typedef : TYPEDEF declaration"""

799 p[0] = Using(p[2].fieldname, [], p[2])

800

801 def p_typedef_alias_flist(self, p):

802 """typedef : flist TYPEDEF declaration"""

803 p[0] = Using(p[3].fieldname, p[1], p[3])

804

805 def p_block_statements_opt(self, p):

806 """block_statements_opt : block_statements"""

807 p[0] = p[1]

808

809 def p_block_statements(self, p):

810 """block_statements : block_statement

811 | block_statements block_statement"""

812 if len(p) == 2:

813 p[0] = [p[1]]

814 else:

815 p[0] = p[1] + [p[2]]

816

817 def p_block_statement(self, p):

818 """block_statement : declaration

819 | option"""

820 p[0] = p[1]

821

822 def p_enum_statements(self, p):

823 """enum_statements : enum_statement

824 | enum_statements enum_statement"""

825 if len(p) == 2:

826 p[0] = [p[1]]

827 else:

828 p[0] = p[1] + [p[2]]

829

830 def p_enum_statement(self, p):

831 """enum_statement : ID '=' NUM ','

832 | ID ','

833 | ID '[' field_options ']' ','

834 | ID '=' NUM '[' field_options ']' ','"""

835 if len(p) == 3:

836 p[0] = {"id": p[1]}

837 elif len(p) == 5:

838 p[0] = {"id": p[1], "value": p[3]}

839 elif len(p) == 6:

840 p[0] = {"id": p[1], "option": p[3]}

841 elif len(p) == 8:

842 p[0] = {"id": p[1], "value": p[3], "option": p[5]}

843 else:

844 self._parse_error("ERROR", self._token_coord(p, 1))

845

846 def p_field_options(self, p):

847 """field_options : field_option

848 | field_options field_option"""

849 if len(p) == 2:

850 p[0] = p[1]

851 else:

852 p[0] = {**p[1], **p[2]}

853

854 def p_field_option(self, p):

855 """field_option : ID

856 | ID '=' assignee ','

857 | ID '=' assignee

858

859 """

860 if len(p) == 2:

861 p[0] = {p[1]: None}

862 else:

863 p[0] = {p[1]: p[3]}

864

865 def p_variable_name(self, p):

866 """variable_name : ID

867 | TYPE

868 | SEVERITY

869 | DESCRIPTION

870 | COUNTERS

871 | PATHS

872 """

873 p[0] = p[1]

874

875 def p_comment(self, p):

876 """comment : COMMENT"""

877 self.last_comment = p[1]

878 p[0] = []

879

880 def p_declaration(self, p):

881 """declaration : type_specifier variable_name ';'

882 | type_specifier variable_name '[' field_options ']' ';'

883 """

884 if len(p) == 7:

885 p[0] = Field(p[1], p[2], p[4])

886 elif len(p) == 4:

887 p[0] = Field(p[1], p[2])

888 else:

889 self._parse_error("ERROR", self._token_coord(p, 1))

890 self.fields.append(p[2])

891

892 def p_declaration_array_vla(self, p):

893 """declaration : type_specifier variable_name '[' ']' ';'"""

894 p[0] = Array(p[1], p[2], 0, modern_vla=True)

895

896 def p_declaration_array(self, p):

897 """declaration : type_specifier variable_name '[' NUM ']' ';'

898 | type_specifier variable_name '[' ID ']' ';'"""

899

900 if len(p) != 7:

901 return self._parse_error(

902 "array: %s" % p.value, self._coord(lineno=p.lineno)

903 )

904

905 # Make this error later

906 if type(p[4]) is int and p[4] == 0:

907 # XXX: Line number is wrong

908 self._parse_warning(

909 "Old Style VLA: {} {}[{}];".format(p[1], p[2], p[4]),

910 self._token_coord(p, 1),

911 )

912

913 if type(p[4]) is str and p[4] not in self.fields:

914 # Verify that length field exists

915 self._parse_error(

916 "Missing length field: {} {}[{}];".format(p[1], p[2], p[4]),

917 self._token_coord(p, 1),

918 )

919 p[0] = Array(p[1], p[2], p[4])

920

921 def p_option(self, p):

922 """option : OPTION ID '=' assignee ';'

923 | OPTION ID ';'"""

924 if len(p) == 4:

925 p[0] = Option(p[2])

926 else:

927 p[0] = Option(p[2], p[4])

928

929 def p_assignee(self, p):

930 """assignee : NUM

931 | TRUE

932 | FALSE

933 | STRING_LITERAL"""

934 p[0] = p[1]

935

936 def p_type_specifier(self, p):

937 """type_specifier : U8

938 | U16

939 | U32

940 | U64

941 | I8

942 | I16

943 | I32

944 | I64

945 | F64

946 | BOOL

947 | STRING"""

948 p[0] = p[1]

949

950 # Do a second pass later to verify that user defined types are defined

951 def p_typedef_specifier(self, p):

952 """type_specifier : ID"""

953 if p[1] not in global_types:

954 self._parse_error(

955 "Undefined type: {}".format(p[1]), self._token_coord(p, 1)

956 )

957 p[0] = p[1]

958

959 def p_union(self, p):

960 """union : UNION ID '{' block_statements_opt '}' ';'"""

961 p[0] = Union(p[2], [], p[4])

962

963 def p_union_flist(self, p):

964 """union : flist UNION ID '{' block_statements_opt '}' ';'"""

965 p[0] = Union(p[3], p[1], p[5])

966

967 # Error rule for syntax errors

968 def p_error(self, p):

969 if p:

970 if p.type == "COMMENT":

971 self.parser.errok()

972 return

973 self._parse_error("before: %s" % p.value, self._coord(lineno=p.lineno))

974 else:

975 self._parse_error("At end of input", self.filename)

976

977 def build(self, **kwargs):

978 self.parser = yacc.yacc(module=self, **kwargs)

979

980

981 class VPPAPI:

982 def __init__(self, debug=False, filename="", logger=None, revision=None):

983 self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug)

984 self.parser = VPPAPIParser(filename, logger, revision=revision)

985 self.parser.build(write_tables=False, debug=debug)

986 self.logger = logger

987 self.revision = revision

988 self.filename = filename

989

990 def parse_string(self, code, debug=0, lineno=1):

991 self.lexer.lineno = lineno

992 return self.parser.parser.parse(code, lexer=self.lexer, debug=debug)

993

994 def parse_fd(self, fd, debug=0):

995 data = fd.read()

996 return self.parse_string(data, debug=debug)

997

998 def parse_filename(self, filename, debug=0):

999 if self.revision:

1000 git_show = "git show {}:{}".format(self.revision, filename)

1001 proc = Popen(git_show.split(), stdout=PIPE, encoding="utf-8")

1002 try:

1003 data, errs = proc.communicate()

1004 if proc.returncode != 0:

1005 print(

1006 "File not found: {}:{}".format(self.revision, filename),

1007 file=sys.stderr,

1008 )

1009 sys.exit(2)

1010 return self.parse_string(data, debug=debug)

1011 except Exception:

1012 sys.exit(3)

1013 else:

1014 try:

1015 with open(filename, encoding="utf-8") as fd:

1016 return self.parse_fd(fd, None)

1017 except FileNotFoundError:

1018 print("File not found: {}".format(filename), file=sys.stderr)

1019 sys.exit(2)

1020

1021 def process(self, objs):

1022 s = {}

1023 s["Option"] = {}

1024 s["Define"] = []

1025 s["Service"] = []

1026 s["types"] = []

1027 s["Import"] = []

1028 s["Counters"] = []

1029 s["Paths"] = []

1030 crc = 0

1031 for o in objs:

1032 try:

1033 crc = binascii.crc32(o.crc, crc) & 0xFFFFFFFF

1034 except AttributeError:

1035 pass

1036

1037 if type(o) is list:

1038 for o2 in o:

1039 if isinstance(o2, Service):

1040 o2.process(s)

1041 else:

1042 o.process(s)

1043

1044 msgs = {d.name: d for d in s["Define"]}

1045 svcs = {s.caller: s for s in s["Service"]}

1046 replies = {s.reply: s for s in s["Service"]}

1047 seen_services = {}

1048

1049 s["file_crc"] = crc

1050

1051 for service in svcs:

1052 if service not in msgs:

1053 raise ValueError(

1054 "Service definition refers to unknown message"

1055 " definition: {}".format(service)

1056 )

1057 if svcs[service].reply != "null" and svcs[service].reply not in msgs:

1058 raise ValueError(

1059 "Service definition refers to unknown message"

1060 " definition in reply: {}".format(svcs[service].reply)

1061 )

1062 if service in replies:

1063 raise ValueError(

1064 "Service definition refers to message"

1065 " marked as reply: {}".format(service)

1066 )

1067 for event in svcs[service].events:

1068 if event not in msgs:

1069 raise ValueError(

1070 "Service definition refers to unknown "

1071 "event: {} in message: {}".format(event, service)

1072 )

1073 seen_services[event] = True

1074

1075 # Create services implicitly

1076 for d in msgs:

1077 if d in seen_services:

1078 continue

1079 if d.endswith("_reply"):

1080 if d[:-6] in svcs:

1081 continue

1082 if d[:-6] not in msgs:

1083 raise ValueError("{} missing calling message".format(d))

1084 continue

1085 if d.endswith("_dump"):

1086 if d in svcs:

1087 continue

1088 if d[:-5] + "_details" in msgs:

1089 s["Service"].append(Service(d, d[:-5] + "_details", stream=True))

1090 else:

1091 raise ValueError("{} missing details message".format(d))

1092 continue

1093

1094 if d.endswith("_details"):

1095 if d[:-8] + "_get" in msgs:

1096 if d[:-8] + "_get" in svcs:

1097 continue

1098 raise ValueError(

1099 "{} should be in a stream service".format(d[:-8] + "_get")

1100 )

1101 if d[:-8] + "_dump" in msgs:

1102 continue

1103 raise ValueError("{} missing dump or get message".format(d))

1104

1105 if d in svcs:

1106 continue

1107 if d + "_reply" in msgs:

1108 s["Service"].append(Service(d, d + "_reply"))

1109 else:

1110 raise ValueError(

1111 "{} missing reply message ({}) or service definition".format(

1112 d, d + "_reply"

1113 )

1114 )

1115

1116 return s

1117

1118 def process_imports(self, objs, in_import, result): # -> List

1119 for o in objs:

1120 # Only allow the following object types from imported file

1121 if in_import and not isinstance(o, (Enum, Import, Typedef, Union, Using)):

1122 continue

1123 if isinstance(o, Import):

1124 result.append(o)

1125 result = self.process_imports(o.result, True, result)

1126 else:

1127 result.append(o)

1128 return result

1129

1130

1131 # Add message ids to each message.

1132 def add_msg_id(s):

1133 for o in s:

1134 o.block.insert(0, Field("u16", "_vl_msg_id"))

1135 return s

1138 dirlist = []

1141 def dirlist_add(dirs):

1142 global dirlist

1143 if dirs:

1144 dirlist = dirlist + dirs

1145

1146

1147 def dirlist_get():

1148 return dirlist

1149

1150

1151 def foldup_blocks(block, crc):

1152 for b in block:

1153 # Look up CRC in user defined types

1154 if b.fieldtype.startswith("vl_api_"):

1155 # Recursively

1156 t = global_types[b.fieldtype]

1157 try:

1158 crc = binascii.crc32(t.crc, crc) & 0xFFFFFFFF

1159 crc = foldup_blocks(t.block, crc)

1160 except AttributeError:

1161 pass

1162 return crc

1163

1164

1165 def foldup_crcs(s):

1166 for f in s:

1167 f.crc = foldup_blocks(f.block, binascii.crc32(f.crc) & 0xFFFFFFFF)

1168

1169

1170 def write_dependencies(output_file, dependency_file, imports):

1171 r = []

1172 for i in imports:

1173 for d in dirlist:

1174 f = os.path.abspath(os.path.join(d, i.filename))

1175 if os.path.exists(f):

1176 r.append(f)

1177 with open(dependency_file, "w", encoding="utf8") as f:

1178 print(f"{output_file}: \\", file=f)

1179 for i in r[:-1]:

1180 print(f" {i} \\", file=f)

1181 if imports:

1182 print(f" {r[-1]}", file=f)

1183

1184

1185 def run_vppapigen(

1186 input_file=None,

1187 output=sys.stdout,

1188 includedir=None,

1189 debug=False,

1190 show_name=None,

1191 output_module="C",

1192 outputdir=None,

1193 pluginpath="",

1194 git_revision=None,

1195 dependency_file=None,

1196 ):

1197 # reset globals

1198 dirlist.clear()

1199 global_types.clear()

1200 seen_imports.clear()

1201

1202 dirlist_add(includedir)

1203 if not debug:

1204 sys.excepthook = exception_handler

1205

1206 # Filename

1207 if show_name:

1208 filename = show_name[0]

1209 elif input_file:

1210 filename = input_file

1211 else:

1212 filename = ""

1213

1214 if debug:

1215 logging.basicConfig(stream=sys.stdout, level=logging.WARNING)

1216 else:

1217 logging.basicConfig()

1218

1219 #

1220 # Generate representation

1221 #

1222 from importlib.machinery import SourceFileLoader

1223

1224 # Default path

1225 pluginpath = ""

1226 if not pluginpath:

1227 cand = []

1228 cand.append(os.path.dirname(os.path.realpath(__file__)))

1229 cand.append(os.path.dirname(os.path.realpath(__file__)) + "/../share/vpp/")

1230 for c in cand:

1231 c += "/"

1232 if os.path.isfile("{}vppapigen_{}.py".format(c, output_module.lower())):

1233 pluginpath = c

1234 break

1235 else:

1236 pluginpath = pluginpath + "/"

1237 if pluginpath == "":

1238 log.exception("Output plugin not found")

1239 return 1

1240 module_path = "{}vppapigen_{}.py".format(pluginpath, output_module.lower())

1241

1242 try:

1243 plugin = SourceFileLoader(output_module, module_path).load_module()

1244 except Exception as err:

1245 log.exception("Error importing output plugin: %s, %s", module_path, err)

1246 return 1

1247

1248 parser = VPPAPI(debug=debug, filename=filename, logger=log, revision=git_revision)

1249

1250 try:

1251 if not input_file:

1252 parsed_objects = parser.parse_fd(sys.stdin, log)

1253 else:

1254 parsed_objects = parser.parse_filename(input_file, log)

1255 except ParseError as e:

1256 print("Parse error: ", e, file=sys.stderr)

1257 sys.exit(1)

1258

1259 # Build a list of objects. Hash of lists.

1260 result = []

1261

1262 # if the variable is not set in the plugin, assume it to be false.

1263 try:

1264 plugin.process_imports

1265 except AttributeError:

1266 plugin.process_imports = False

1267

1268 if plugin.process_imports:

1269 result = parser.process_imports(parsed_objects, False, result)

1270 s = parser.process(result)

1271 else:

1272 s = parser.process(parsed_objects)

1273 imports = parser.process_imports(parsed_objects, False, result)

1274 s["imported"] = parser.process(imports)

1275

1276 if dependency_file and isinstance(output, TextIOWrapper):

1277 write_dependencies(output.name, dependency_file[0], s["Import"])

1278

1279 # Add msg_id field

1280 s["Define"] = add_msg_id(s["Define"])

1281

1282 # Fold up CRCs

1283 foldup_crcs(s["Define"])

1284

1285 #

1286 # Debug

1287 if debug:

1288 import pprint

1289

1290 pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)

1291 for t in s["Define"]:

1292 pp.pprint([t.name, t.flags, t.block])

1293 for t in s["types"]:

1294 pp.pprint([t.name, t.block])

1295

1296 result = plugin.run(outputdir, filename, s)

1297 if result:

1298 if isinstance(output, str):

1299 with open(output, "w", encoding="UTF-8") as f:

1300 print(result, file=f)

1301 else:

1302 print(result, file=output)

1303 else:

1304 log.exception("Running plugin failed: %s %s", filename, result)

1305 return 1

1306 return 0

1307

1308

1309 def run_kw_vppapigen(kwargs):

1310 return run_vppapigen(**kwargs)

1311

1312

1313 #

1314 # Main

1315 #

1316 def main():

1317 if sys.version_info < (

1318 3,

1319 5,

1320 ):

1321 log.exception(

1322 "vppapigen requires a supported version of python. "

1323 "Please use version 3.5 or greater. "

1324 "Using %s",

1325 sys.version,

1326 )

1327 return 1

1328

1329 cliparser = argparse.ArgumentParser(description="VPP API generator")

1330 cliparser.add_argument("--pluginpath", default="")

1331 cliparser.add_argument("--includedir", action="append")

1332 cliparser.add_argument("--outputdir", action="store")

1333 cliparser.add_argument("--input")

1334 cliparser.add_argument(

1335 "--output",

1336 nargs="?",

1337 type=argparse.FileType("w", encoding="UTF-8"),

1338 default=sys.stdout,

1339 )

1340

1341 cliparser.add_argument("output_module", nargs="?", default="C")

1342 cliparser.add_argument("--debug", action="store_true")

1343 cliparser.add_argument("--show-name", nargs=1)

1344 cliparser.add_argument(

1345 "--git-revision", help="Git revision to use for opening files"

1346 )

1347 cliparser.add_argument("-MF", nargs=1, help="Dependency file")

1348 args = cliparser.parse_args()

1349

1350 return run_vppapigen(

1351 includedir=args.includedir,

1352 debug=args.debug,

1353 outputdir=args.outputdir,

1354 show_name=args.show_name,

1355 input_file=args.input,

1356 output_module=args.output_module,

1357 pluginpath=args.pluginpath,

1358 git_revision=args.git_revision,

1359 output=args.output,

1360 dependency_file=args.MF,

1361 )

1362

1363

1364 if __name__ == "__main__":

1365 sys.exit(main())