src/vpp-api/vapi/vapi_json_parser.py

   1 #!/usr/bin/env python3
   2
   3 import json
   4
   5
   6 class ParseError(Exception):
   7     pass
   8
   9
  10 magic_prefix = "vl_api_"
  11 magic_suffix = "_t"
  12
  13
  14 def remove_magic(what):
  15     if what.startswith(magic_prefix) and what.endswith(magic_suffix):
  16         return what[len(magic_prefix) : -len(magic_suffix)]
  17     return what
  18
  19
  20 class Field(object):
  21     def __init__(self, field_name, field_type, array_len=None, nelem_field=None):
  22         self.name = field_name
  23         self.type = field_type
  24         self.len = array_len
  25         self.nelem_field = nelem_field
  26
  27     def __str__(self):
  28         if self.len is None:
  29             return "Field(name: %s, type: %s)" % (self.name, self.type)
  30         elif type(self.len) == dict:
  31             return "Field(name: %s, type: %s, length: %s)" % (
  32                 self.name,
  33                 self.type,
  34                 self.len,
  35             )
  36         elif self.len > 0:
  37             return "Field(name: %s, type: %s, length: %s)" % (
  38                 self.name,
  39                 self.type,
  40                 self.len,
  41             )
  42         else:
  43             return "Field(name: %s, type: %s, variable length stored in: %s)" % (
  44                 self.name,
  45                 self.type,
  46                 self.nelem_field,
  47             )
  48
  49     def is_vla(self):
  50         return self.nelem_field is not None
  51
  52     def has_vla(self):
  53         return self.is_vla() or self.type.has_vla()
  54
  55
  56 class Alias(Field):
  57     pass
  58
  59
  60 class Type(object):
  61     def __init__(self, name):
  62         self.name = name
  63
  64     def __str__(self):
  65         return self.name
  66
  67
  68 class SimpleType(Type):
  69     def has_vla(self):
  70         return False
  71
  72
  73 def get_msg_header_defs(struct_type_class, field_class, json_parser, logger):
  74     return [
  75         struct_type_class(
  76             [
  77                 "msg_header1_t",
  78                 ["u16", "_vl_msg_id"],
  79                 ["u32", "context"],
  80             ],
  81             json_parser,
  82             field_class,
  83             logger,
  84         ),
  85         struct_type_class(
  86             [
  87                 "msg_header2_t",
  88                 ["u16", "_vl_msg_id"],
  89                 ["u32", "client_index"],
  90                 ["u32", "context"],
  91             ],
  92             json_parser,
  93             field_class,
  94             logger,
  95         ),
  96     ]
  97
  98
  99 class Struct(object):
 100     def __init__(self, name, fields):
 101         self.name = name
 102         self.fields = fields
 103         self.field_names = [n.name for n in self.fields]
 104         self.depends = [f.type for f in self.fields]
 105
 106     def __str__(self):
 107         return "[%s]" % "], [".join([str(f) for f in self.fields])
 108
 109     def has_vla(self):
 110         for f in self.fields:
 111             if f.has_vla():
 112                 return True
 113         return False
 114
 115
 116 class Enum(SimpleType):
 117     def __init__(self, name, value_pairs, enumtype):
 118         super(Enum, self).__init__(name)
 119         self.type = enumtype
 120         self.value_pairs = value_pairs
 121
 122     def __str__(self):
 123         return "Enum(%s, [%s])" % (
 124             self.name,
 125             "], [".join(["%s => %s" % (i, j) for i, j in self.value_pairs]),
 126         )
 127
 128
 129 class Union(Type):
 130     def __init__(self, name, type_pairs, crc):
 131         Type.__init__(self, name)
 132         self.crc = crc
 133         self.type_pairs = type_pairs
 134         self.depends = [t for t, _ in self.type_pairs]
 135
 136     def __str__(self):
 137         return "Union(%s, [%s])" % (
 138             self.name,
 139             "], [".join(["%s %s" % (i, j) for i, j in self.type_pairs]),
 140         )
 141
 142     def has_vla(self):
 143         return False
 144
 145
 146 class Message(object):
 147     def __init__(self, logger, definition, json_parser):
 148         struct_type_class = json_parser.struct_type_class
 149         field_class = json_parser.field_class
 150         self.request = None
 151         self.logger = logger
 152         m = definition
 153         logger.debug("Parsing message definition `%s'" % m)
 154         name = m[0]
 155         self.name = name
 156         logger.debug("Message name is `%s'" % name)
 157         ignore = True
 158         self.header = None
 159         self.is_reply = json_parser.is_reply(self.name)
 160         self.is_event = json_parser.is_event(self.name)
 161         self.is_stream = json_parser.is_stream(self.name)
 162         fields = []
 163         for header in get_msg_header_defs(
 164             struct_type_class, field_class, json_parser, logger
 165         ):
 166             logger.debug("Probing header `%s'" % header.name)
 167             if header.is_part_of_def(m[1:]):
 168                 self.header = header
 169                 logger.debug("Found header `%s'" % header.name)
 170                 fields.append(field_class(field_name="header", field_type=self.header))
 171                 ignore = False
 172                 break
 173         if ignore and not self.is_event and not self.is_reply:
 174             raise ParseError(
 175                 "While parsing message `%s': could not find all "
 176                 "common header fields" % name
 177             )
 178         for field in m[1:]:
 179             if isinstance(field, dict) and "crc" in field:
 180                 self.crc = field["crc"]
 181                 logger.debug("Found CRC `%s'" % self.crc)
 182                 continue
 183             else:
 184                 field_type = json_parser.lookup_type_like_id(field[0])
 185                 logger.debug("Parsing message field `%s'" % field)
 186                 l = len(field)
 187                 if any(type(n) is dict for n in field):
 188                     l -= 1
 189                 if l == 2:
 190                     if self.header is not None and self.header.has_field(field[1]):
 191                         continue
 192                     p = field_class(field_name=field[1], field_type=field_type)
 193                 elif l == 3:
 194                     if field[2] == 0 and field[0] != "string":
 195                         raise ParseError(
 196                             "While parsing message `%s': variable length "
 197                             "array `%s' doesn't have reference to member "
 198                             "containing the actual length" % (name, field[1])
 199                         )
 200                     if field[0] == "string" and field[2] == 0:
 201                         field_type = json_parser.lookup_type_like_id("vl_api_string_t")
 202                         p = field_class(field_name=field[1], field_type=field_type)
 203                     else:
 204                         if field[0] == "string" and field[2] > 0:
 205                             field_type = json_parser.lookup_type_like_id("u8")
 206
 207                         p = field_class(
 208                             field_name=field[1],
 209                             field_type=field_type,
 210                             array_len=field[2],
 211                         )
 212                 elif l == 4:
 213                     nelem_field = None
 214                     for f in fields:
 215                         if f.name == field[3]:
 216                             nelem_field = f
 217                     if nelem_field is None:
 218                         raise ParseError(
 219                             "While parsing message `%s': couldn't find "
 220                             "variable length array `%s' member containing "
 221                             "the actual length `%s'" % (name, field[1], field[3])
 222                         )
 223                     p = field_class(
 224                         field_name=field[1],
 225                         field_type=field_type,
 226                         array_len=field[2],
 227                         nelem_field=nelem_field,
 228                     )
 229                 else:
 230                     raise Exception(
 231                         "Don't know how to parse message "
 232                         "definition for message `%s': `%s'" % (m, m[1:])
 233                     )
 234                 logger.debug("Parsed field `%s'" % p)
 235                 fields.append(p)
 236         self.fields = fields
 237         self.depends = [f.type for f in self.fields]
 238         logger.debug("Parsed message: %s" % self)
 239
 240     def __str__(self):
 241         return "Message(%s, [%s], {crc: %s}" % (
 242             self.name,
 243             "], [".join([str(f) for f in self.fields]),
 244             self.crc,
 245         )
 246
 247
 248 class StructType(Type, Struct):
 249     def __init__(self, definition, json_parser, field_class, logger):
 250         t = definition
 251         logger.debug("Parsing struct definition `%s'" % t)
 252         name = t[0]
 253         fields = []
 254         for field in t[1:]:
 255             if len(field) == 1 and "crc" in field:
 256                 self.crc = field["crc"]
 257                 continue
 258             field_type = json_parser.lookup_type_like_id(field[0])
 259             logger.debug("Parsing type field `%s'" % field)
 260             if len(field) == 2:
 261                 p = field_class(field_name=field[1], field_type=field_type)
 262             elif len(field) == 3:
 263                 if field[2] == 0:
 264                     if name == "vl_api_string_t":
 265                         p = None
 266                         for f in fields:
 267                             if f.name == "length":
 268                                 nelem_field = f
 269                                 p = field_class(
 270                                     field_name=field[1],
 271                                     field_type=field_type,
 272                                     array_len=field[2],
 273                                     nelem_field=nelem_field,
 274                                 )
 275                                 break
 276                         if p is None:
 277                             raise ParseError(
 278                                 "While parsing type `%s': missing `length'" % name
 279                             )
 280                     else:
 281                         raise ParseError(
 282                             "While parsing type `%s': array `%s' has "
 283                             "variable length" % (name, field[1])
 284                         )
 285                 else:
 286                     p = field_class(
 287                         field_name=field[1], field_type=field_type, array_len=field[2]
 288                     )
 289             elif len(field) == 4:
 290                 nelem_field = None
 291                 for f in fields:
 292                     if f.name == field[3]:
 293                         nelem_field = f
 294                 if nelem_field is None:
 295                     raise ParseError(
 296                         "While parsing message `%s': couldn't find "
 297                         "variable length array `%s' member containing "
 298                         "the actual length `%s'" % (name, field[1], field[3])
 299                     )
 300                 p = field_class(
 301                     field_name=field[1],
 302                     field_type=field_type,
 303                     array_len=field[2],
 304                     nelem_field=nelem_field,
 305                 )
 306             else:
 307                 raise ParseError(
 308                     "Don't know how to parse field `%s' of type definition "
 309                     "for type `%s'" % (field, t)
 310                 )
 311             fields.append(p)
 312         Type.__init__(self, name)
 313         Struct.__init__(self, name, fields)
 314
 315     def __str__(self):
 316         return "StructType(%s, %s)" % (Type.__str__(self), Struct.__str__(self))
 317
 318     def has_field(self, name):
 319         return name in self.field_names
 320
 321     def is_part_of_def(self, definition):
 322         for idx in range(len(self.fields)):
 323             field = definition[idx]
 324             p = self.fields[idx]
 325             if field[1] != p.name:
 326                 return False
 327             if field[0] != p.type.name:
 328                 raise ParseError(
 329                     "Unexpected field type `%s' (should be `%s'), "
 330                     "while parsing msg/def/field `%s/%s/%s'"
 331                     % (field[0], p.type, p.name, definition, field)
 332                 )
 333         return True
 334
 335
 336 class JsonParser(object):
 337     def __init__(
 338         self,
 339         logger,
 340         files,
 341         simple_type_class=SimpleType,
 342         enum_class=Enum,
 343         union_class=Union,
 344         struct_type_class=StructType,
 345         field_class=Field,
 346         message_class=Message,
 347         alias_class=Alias,
 348     ):
 349         self.services = {}
 350         self.messages = {}
 351         self.enums = {}
 352         self.enumflags = {}
 353         self.unions = {}
 354         self.aliases = {}
 355         self.types = {
 356             x: simple_type_class(x)
 357             for x in [
 358                 "i8",
 359                 "i16",
 360                 "i32",
 361                 "i64",
 362                 "u8",
 363                 "u16",
 364                 "u32",
 365                 "u64",
 366                 "f64",
 367                 "bool",
 368             ]
 369         }
 370
 371         self.types["string"] = simple_type_class("u8")
 372         self.types["vl_api_string_t"] = struct_type_class(
 373             ["vl_api_string_t", ["u32", "length"], ["u8", "buf", 0]],
 374             self,
 375             field_class,
 376             logger,
 377         )
 378         self.replies = set()
 379         self.events = set()
 380         self.streams = set()
 381         self.simple_type_class = simple_type_class
 382         self.enum_class = enum_class
 383         self.union_class = union_class
 384         self.struct_type_class = struct_type_class
 385         self.field_class = field_class
 386         self.alias_class = alias_class
 387         self.message_class = message_class
 388
 389         self.exceptions = []
 390         self.json_files = []
 391         self.types_by_json = {}
 392         self.enums_by_json = {}
 393         self.unions_by_json = {}
 394         self.aliases_by_json = {}
 395         self.messages_by_json = {}
 396         self.logger = logger
 397         for f in files:
 398             self.parse_json_file(f)
 399         self.finalize_parsing()
 400
 401     def parse_json_file(self, path):
 402         self.logger.info("Parsing json api file: `%s'" % path)
 403         self.json_files.append(path)
 404         self.types_by_json[path] = []
 405         self.enums_by_json[path] = []
 406         self.unions_by_json[path] = []
 407         self.aliases_by_json[path] = []
 408         self.messages_by_json[path] = {}
 409         with open(path) as f:
 410             j = json.load(f)
 411             for k in j["services"]:
 412                 if k in self.services:
 413                     raise ParseError("Duplicate service `%s'" % k)
 414                 self.services[k] = j["services"][k]
 415                 self.replies.add(self.services[k]["reply"])
 416                 if "events" in self.services[k]:
 417                     for x in self.services[k]["events"]:
 418                         self.events.add(x)
 419                 if "stream_msg" in self.services[k]:
 420                     self.streams.add(self.services[k]["stream_msg"])
 421             for e in j["enums"]:
 422                 name = e[0]
 423                 value_pairs = e[1:-1]
 424                 enumtype = self.types[e[-1]["enumtype"]]
 425                 enum = self.enum_class(name, value_pairs, enumtype)
 426                 self.enums[enum.name] = enum
 427                 self.logger.debug("Parsed enum: %s" % enum)
 428                 self.enums_by_json[path].append(enum)
 429             for e in j["enumflags"]:
 430                 name = e[0]
 431                 value_pairs = e[1:-1]
 432                 enumtype = self.types[e[-1]["enumtype"]]
 433                 enum = self.enum_class(name, value_pairs, enumtype)
 434                 self.enums[enum.name] = enum
 435                 self.logger.debug("Parsed enumflag: %s" % enum)
 436                 self.enums_by_json[path].append(enum)
 437             exceptions = []
 438             progress = 0
 439             last_progress = 0
 440             while True:
 441                 for u in j["unions"]:
 442                     name = u[0]
 443                     if name in self.unions:
 444                         progress = progress + 1
 445                         continue
 446                     try:
 447                         type_pairs = [
 448                             [self.lookup_type_like_id(t), n] for t, n in u[1:]
 449                         ]
 450                         union = self.union_class(name, type_pairs, 0)
 451                         progress = progress + 1
 452                     except ParseError as e:
 453                         exceptions.append(e)
 454                         continue
 455                     self.unions[union.name] = union
 456                     self.logger.debug("Parsed union: %s" % union)
 457                     self.unions_by_json[path].append(union)
 458                 for t in j["types"]:
 459                     if t[0] in self.types:
 460                         progress = progress + 1
 461                         continue
 462                     try:
 463                         type_ = self.struct_type_class(
 464                             t, self, self.field_class, self.logger
 465                         )
 466                         if type_.name in self.types:
 467                             raise ParseError("Duplicate type `%s'" % type_.name)
 468                         progress = progress + 1
 469                     except ParseError as e:
 470                         exceptions.append(e)
 471                         continue
 472                     self.types[type_.name] = type_
 473                     self.types_by_json[path].append(type_)
 474                     self.logger.debug("Parsed type: %s" % type_)
 475                 for name, body in j["aliases"].items():
 476                     if name in self.aliases:
 477                         progress = progress + 1
 478                         continue
 479                     if "length" in body:
 480                         array_len = body["length"]
 481                     else:
 482                         array_len = None
 483                     try:
 484                         t = self.lookup_type_like_id(body["type"])
 485                     except ParseError as e:
 486                         exceptions.append(e)
 487                         continue
 488                     alias = self.alias_class(name, t, array_len)
 489                     self.aliases[name] = alias
 490                     self.logger.debug("Parsed alias: %s" % alias)
 491                     self.aliases_by_json[path].append(alias)
 492                 if not exceptions:
 493                     # finished parsing
 494                     break
 495                 if progress <= last_progress:
 496                     # cannot make forward progress
 497                     self.exceptions.extend(exceptions)
 498                     break
 499                 exceptions = []
 500                 last_progress = progress
 501                 progress = 0
 502             prev_length = len(self.messages)
 503             processed = []
 504             while True:
 505                 exceptions = []
 506                 for m in j["messages"]:
 507                     if m in processed:
 508                         continue
 509                     try:
 510                         msg = self.message_class(self.logger, m, self)
 511                         if msg.name in self.messages:
 512                             raise ParseError("Duplicate message `%s'" % msg.name)
 513                     except ParseError as e:
 514                         exceptions.append(e)
 515                         continue
 516                     self.messages[msg.name] = msg
 517                     self.messages_by_json[path][msg.name] = msg
 518                     processed.append(m)
 519                 if prev_length == len(self.messages):
 520                     # cannot make forward progress ...
 521                     self.exceptions.extend(exceptions)
 522                     break
 523                 prev_length = len(self.messages)
 524
 525     def lookup_type_like_id(self, name):
 526         mundane_name = remove_magic(name)
 527         if name in self.types:
 528             return self.types[name]
 529         elif name in self.enums:
 530             return self.enums[name]
 531         elif name in self.enumflags:
 532             return self.enumflags[name]
 533         elif name in self.unions:
 534             return self.unions[name]
 535         elif name in self.aliases:
 536             return self.aliases[name]
 537         elif mundane_name in self.types:
 538             return self.types[mundane_name]
 539         elif mundane_name in self.enums:
 540             return self.enums[mundane_name]
 541         elif mundane_name in self.enumflags:
 542             return self.enumflags[mundane_name]
 543         elif mundane_name in self.unions:
 544             return self.unions[mundane_name]
 545         elif mundane_name in self.aliases:
 546             return self.aliases[mundane_name]
 547         raise ParseError(
 548             "Could not find type, enum or union by magic name `%s' nor by "
 549             "mundane name `%s'" % (name, mundane_name)
 550         )
 551
 552     def is_reply(self, message):
 553         return message in self.replies
 554
 555     def is_event(self, message):
 556         return message in self.events
 557
 558     def is_stream(self, message):
 559         return message in self.streams
 560
 561     def has_stream_msg(self, message):
 562         return (
 563             message.name in self.services
 564             and "stream_msg" in self.services[message.name]
 565         )
 566
 567     def get_stream_msg(self, message):
 568         if not self.has_stream_msg(message):
 569             return None
 570         return self.messages[self.services[message.name]["stream_msg"]]
 571
 572     def get_reply(self, message):
 573         return self.messages[self.services[message]["reply"]]
 574
 575     def finalize_parsing(self):
 576         if len(self.messages) == 0:
 577             for e in self.exceptions:
 578                 self.logger.warning(e)
 579         for jn, j in self.messages_by_json.items():
 580             remove = []
 581             for n, m in j.items():
 582                 try:
 583                     if not m.is_reply and not m.is_event and not m.is_stream:
 584                         try:
 585                             m.reply = self.get_reply(n)
 586                             m.reply_is_stream = False
 587                             m.has_stream_msg = self.has_stream_msg(m)
 588                             if "stream" in self.services[m.name]:
 589                                 m.reply_is_stream = self.services[m.name]["stream"]
 590                             if m.has_stream_msg:
 591                                 m.stream_msg = self.get_stream_msg(m)
 592                             m.reply.request = m
 593                         except:
 594                             raise ParseError("Cannot find reply to message `%s'" % n)
 595                 except ParseError as e:
 596                     self.exceptions.append(e)
 597                     remove.append(n)
 598
 599             self.messages_by_json[jn] = {k: v for k, v in j.items() if k not in remove}