src/vpp-api/vapi/vapi_json_parser.py

   1 #!/usr/bin/env python3
   2
   3 import json
   4
   5
   6 class ParseError (Exception):
   7     pass
   8
   9
  10 magic_prefix = "vl_api_"
  11 magic_suffix = "_t"
  12
  13
  14 def remove_magic(what):
  15     if what.startswith(magic_prefix) and what.endswith(magic_suffix):
  16         return what[len(magic_prefix): - len(magic_suffix)]
  17     return what
  18
  19
  20 class Field(object):
  21
  22     def __init__(self, field_name, field_type, array_len=None,
  23                  nelem_field=None):
  24         self.name = field_name
  25         self.type = field_type
  26         self.len = array_len
  27         self.nelem_field = nelem_field
  28
  29     def __str__(self):
  30         if self.len is None:
  31             return "Field(name: %s, type: %s)" % (self.name, self.type)
  32         elif type(self.len) == dict:
  33             return "Field(name: %s, type: %s, length: %s)" % (self.name,
  34                                                               self.type,
  35                                                               self.len)
  36         elif self.len > 0:
  37             return "Field(name: %s, type: %s, length: %s)" % (self.name,
  38                                                               self.type,
  39                                                               self.len)
  40         else:
  41             return (
  42                 "Field(name: %s, type: %s, variable length stored in: %s)" %
  43                 (self.name, self.type, self.nelem_field))
  44
  45     def is_vla(self):
  46         return self.nelem_field is not None
  47
  48     def has_vla(self):
  49         return self.is_vla() or self.type.has_vla()
  50
  51
  52 class Alias(Field):
  53     pass
  54
  55
  56 class Type(object):
  57     def __init__(self, name):
  58         self.name = name
  59
  60     def __str__(self):
  61         return self.name
  62
  63
  64 class SimpleType (Type):
  65
  66     def has_vla(self):
  67         return False
  68
  69
  70 def get_msg_header_defs(struct_type_class, field_class, json_parser, logger):
  71     return [
  72         struct_type_class(['msg_header1_t',
  73                            ['u16', '_vl_msg_id'],
  74                            ['u32', 'context'],
  75                            ],
  76                           json_parser, field_class, logger
  77                           ),
  78         struct_type_class(['msg_header2_t',
  79                            ['u16', '_vl_msg_id'],
  80                            ['u32', 'client_index'],
  81                            ['u32', 'context'],
  82                            ],
  83                           json_parser, field_class, logger
  84                           ),
  85     ]
  86
  87
  88 class Struct(object):
  89
  90     def __init__(self, name, fields):
  91         self.name = name
  92         self.fields = fields
  93         self.field_names = [n.name for n in self.fields]
  94         self.depends = [f.type for f in self.fields]
  95
  96     def __str__(self):
  97         return "[%s]" % "], [".join([str(f) for f in self.fields])
  98
  99     def has_vla(self):
 100         for f in self.fields:
 101             if f.has_vla():
 102                 return True
 103         return False
 104
 105
 106 class Enum(SimpleType):
 107     def __init__(self, name, value_pairs, enumtype):
 108         super(Enum, self).__init__(name)
 109         self.type = enumtype
 110         self.value_pairs = value_pairs
 111
 112     def __str__(self):
 113         return "Enum(%s, [%s])" % (
 114             self.name,
 115             "], [" .join(["%s => %s" % (i, j) for i, j in self.value_pairs])
 116         )
 117
 118
 119 class Union(Type):
 120     def __init__(self, name, type_pairs, crc):
 121         Type.__init__(self, name)
 122         self.crc = crc
 123         self.type_pairs = type_pairs
 124         self.depends = [t for t, _ in self.type_pairs]
 125
 126     def __str__(self):
 127         return "Union(%s, [%s])" % (
 128             self.name,
 129             "], [" .join(["%s %s" % (i, j) for i, j in self.type_pairs])
 130         )
 131
 132     def has_vla(self):
 133         return False
 134
 135
 136 class Message(object):
 137
 138     def __init__(self, logger, definition, json_parser):
 139         struct_type_class = json_parser.struct_type_class
 140         field_class = json_parser.field_class
 141         self.request = None
 142         self.logger = logger
 143         m = definition
 144         logger.debug("Parsing message definition `%s'" % m)
 145         name = m[0]
 146         self.name = name
 147         logger.debug("Message name is `%s'" % name)
 148         ignore = True
 149         self.header = None
 150         self.is_reply = json_parser.is_reply(self.name)
 151         self.is_event = json_parser.is_event(self.name)
 152         fields = []
 153         for header in get_msg_header_defs(struct_type_class, field_class,
 154                                           json_parser, logger):
 155             logger.debug("Probing header `%s'" % header.name)
 156             if header.is_part_of_def(m[1:]):
 157                 self.header = header
 158                 logger.debug("Found header `%s'" % header.name)
 159                 fields.append(field_class(field_name='header',
 160                                           field_type=self.header))
 161                 ignore = False
 162                 break
 163         if ignore and not self.is_event and not self.is_reply:
 164             raise ParseError("While parsing message `%s': could not find all "
 165                              "common header fields" % name)
 166         for field in m[1:]:
 167             if len(field) == 1 and 'crc' in field:
 168                 self.crc = field['crc']
 169                 logger.debug("Found CRC `%s'" % self.crc)
 170                 continue
 171             else:
 172                 field_type = json_parser.lookup_type_like_id(field[0])
 173                 logger.debug("Parsing message field `%s'" % field)
 174                 l = len(field)
 175                 if any(type(n) is dict for n in field):
 176                     l -= 1
 177                 if l == 2:
 178                     if self.header is not None and\
 179                             self.header.has_field(field[1]):
 180                         continue
 181                     p = field_class(field_name=field[1],
 182                                     field_type=field_type)
 183                 elif l == 3:
 184                     if field[2] == 0 and field[0] != 'string':
 185                         raise ParseError(
 186                             "While parsing message `%s': variable length "
 187                             "array `%s' doesn't have reference to member "
 188                             "containing the actual length" % (
 189                                 name, field[1]))
 190                     if field[0] == 'string' and field[2] > 0:
 191                         field_type = json_parser.lookup_type_like_id('u8')
 192
 193                     p = field_class(
 194                         field_name=field[1],
 195                         field_type=field_type,
 196                         array_len=field[2])
 197                 elif l == 4:
 198                     nelem_field = None
 199                     for f in fields:
 200                         if f.name == field[3]:
 201                             nelem_field = f
 202                     if nelem_field is None:
 203                         raise ParseError(
 204                             "While parsing message `%s': couldn't find "
 205                             "variable length array `%s' member containing "
 206                             "the actual length `%s'" % (
 207                                 name, field[1], field[3]))
 208                     p = field_class(
 209                         field_name=field[1],
 210                         field_type=field_type,
 211                         array_len=field[2],
 212                         nelem_field=nelem_field)
 213                 else:
 214                     raise Exception("Don't know how to parse message "
 215                                     "definition for message `%s': `%s'" %
 216                                     (m, m[1:]))
 217                 logger.debug("Parsed field `%s'" % p)
 218                 fields.append(p)
 219         self.fields = fields
 220         self.depends = [f.type for f in self.fields]
 221         logger.debug("Parsed message: %s" % self)
 222
 223     def __str__(self):
 224         return "Message(%s, [%s], {crc: %s}" % \
 225             (self.name,
 226              "], [".join([str(f) for f in self.fields]),
 227              self.crc)
 228
 229
 230 class StructType (Type, Struct):
 231
 232     def __init__(self, definition, json_parser, field_class, logger):
 233         t = definition
 234         logger.debug("Parsing struct definition `%s'" % t)
 235         name = t[0]
 236         fields = []
 237         for field in t[1:]:
 238             if len(field) == 1 and 'crc' in field:
 239                 self.crc = field['crc']
 240                 continue
 241             field_type = json_parser.lookup_type_like_id(field[0])
 242             logger.debug("Parsing type field `%s'" % field)
 243             if len(field) == 2:
 244                 p = field_class(field_name=field[1],
 245                                 field_type=field_type)
 246             elif len(field) == 3:
 247                 if field[2] == 0:
 248                     raise ParseError("While parsing type `%s': array `%s' has "
 249                                      "variable length" % (name, field[1]))
 250                 p = field_class(field_name=field[1],
 251                                 field_type=field_type,
 252                                 array_len=field[2])
 253             elif len(field) == 4:
 254                 nelem_field = None
 255                 for f in fields:
 256                     if f.name == field[3]:
 257                         nelem_field = f
 258                 if nelem_field is None:
 259                     raise ParseError(
 260                         "While parsing message `%s': couldn't find "
 261                         "variable length array `%s' member containing "
 262                         "the actual length `%s'" % (
 263                             name, field[1], field[3]))
 264                 p = field_class(field_name=field[1],
 265                                 field_type=field_type,
 266                                 array_len=field[2],
 267                                 nelem_field=nelem_field)
 268             else:
 269                 raise ParseError(
 270                     "Don't know how to parse field `%s' of type definition "
 271                     "for type `%s'" % (field, t))
 272             fields.append(p)
 273         Type.__init__(self, name)
 274         Struct.__init__(self, name, fields)
 275
 276     def __str__(self):
 277         return "StructType(%s, %s)" % (Type.__str__(self),
 278                                        Struct.__str__(self))
 279
 280     def has_field(self, name):
 281         return name in self.field_names
 282
 283     def is_part_of_def(self, definition):
 284         for idx in range(len(self.fields)):
 285             field = definition[idx]
 286             p = self.fields[idx]
 287             if field[1] != p.name:
 288                 return False
 289             if field[0] != p.type.name:
 290                 raise ParseError(
 291                     "Unexpected field type `%s' (should be `%s'), "
 292                     "while parsing msg/def/field `%s/%s/%s'" %
 293                     (field[0], p.type, p.name, definition, field))
 294         return True
 295
 296
 297 class JsonParser(object):
 298     def __init__(self, logger, files, simple_type_class=SimpleType,
 299                  enum_class=Enum, union_class=Union,
 300                  struct_type_class=StructType, field_class=Field,
 301                  message_class=Message, alias_class=Alias):
 302         self.services = {}
 303         self.messages = {}
 304         self.enums = {}
 305         self.unions = {}
 306         self.aliases = {}
 307         self.types = {
 308             x: simple_type_class(x) for x in [
 309                 'i8', 'i16', 'i32', 'i64',
 310                 'u8', 'u16', 'u32', 'u64',
 311                 'f64', 'bool'
 312             ]
 313         }
 314
 315         self.types['string'] = simple_type_class('vl_api_string_t')
 316         self.replies = set()
 317         self.events = set()
 318         self.simple_type_class = simple_type_class
 319         self.enum_class = enum_class
 320         self.union_class = union_class
 321         self.struct_type_class = struct_type_class
 322         self.field_class = field_class
 323         self.alias_class = alias_class
 324         self.message_class = message_class
 325
 326         self.exceptions = []
 327         self.json_files = []
 328         self.types_by_json = {}
 329         self.enums_by_json = {}
 330         self.unions_by_json = {}
 331         self.aliases_by_json = {}
 332         self.messages_by_json = {}
 333         self.logger = logger
 334         for f in files:
 335             self.parse_json_file(f)
 336         self.finalize_parsing()
 337
 338     def parse_json_file(self, path):
 339         self.logger.info("Parsing json api file: `%s'" % path)
 340         self.json_files.append(path)
 341         self.types_by_json[path] = []
 342         self.enums_by_json[path] = []
 343         self.unions_by_json[path] = []
 344         self.aliases_by_json[path] = []
 345         self.messages_by_json[path] = {}
 346         with open(path) as f:
 347             j = json.load(f)
 348             for k in j['services']:
 349                 if k in self.services:
 350                     raise ParseError("Duplicate service `%s'" % k)
 351                 self.services[k] = j['services'][k]
 352                 self.replies.add(self.services[k]["reply"])
 353                 if "events" in self.services[k]:
 354                     for x in self.services[k]["events"]:
 355                         self.events.add(x)
 356             for e in j['enums']:
 357                 name = e[0]
 358                 value_pairs = e[1:-1]
 359                 enumtype = self.types[e[-1]["enumtype"]]
 360                 enum = self.enum_class(name, value_pairs, enumtype)
 361                 self.enums[enum.name] = enum
 362                 self.logger.debug("Parsed enum: %s" % enum)
 363                 self.enums_by_json[path].append(enum)
 364             exceptions = []
 365             progress = 0
 366             last_progress = 0
 367             while True:
 368                 for u in j['unions']:
 369                     name = u[0]
 370                     if name in self.unions:
 371                         progress = progress + 1
 372                         continue
 373                     try:
 374                         type_pairs = [[self.lookup_type_like_id(t), n]
 375                                       for t, n in u[1:]]
 376                         union = self.union_class(name, type_pairs, 0)
 377                         progress = progress + 1
 378                     except ParseError as e:
 379                         exceptions.append(e)
 380                         continue
 381                     self.unions[union.name] = union
 382                     self.logger.debug("Parsed union: %s" % union)
 383                     self.unions_by_json[path].append(union)
 384                 for t in j['types']:
 385                     if t[0] in self.types:
 386                         progress = progress + 1
 387                         continue
 388                     try:
 389                         type_ = self.struct_type_class(t, self,
 390                                                        self.field_class,
 391                                                        self.logger)
 392                         if type_.name in self.types:
 393                             raise ParseError(
 394                                 "Duplicate type `%s'" % type_.name)
 395                         progress = progress + 1
 396                     except ParseError as e:
 397                         exceptions.append(e)
 398                         continue
 399                     self.types[type_.name] = type_
 400                     self.types_by_json[path].append(type_)
 401                     self.logger.debug("Parsed type: %s" % type_)
 402                 for name, body in j['aliases'].items():
 403                     if name in self.aliases:
 404                         progress = progress + 1
 405                         continue
 406                     if 'length' in body:
 407                         array_len = body['length']
 408                     else:
 409                         array_len = None
 410                     try:
 411                         t = self.lookup_type_like_id(body['type'])
 412                     except ParseError as e:
 413                         exceptions.append(e)
 414                         continue
 415                     alias = self.alias_class(name, t, array_len)
 416                     self.aliases[name] = alias
 417                     self.logger.debug("Parsed alias: %s" % alias)
 418                     self.aliases_by_json[path].append(alias)
 419                 if not exceptions:
 420                     # finished parsing
 421                     break
 422                 if progress <= last_progress:
 423                     # cannot make forward progress
 424                     self.exceptions.extend(exceptions)
 425                     break
 426                 exceptions = []
 427                 last_progress = progress
 428                 progress = 0
 429             prev_length = len(self.messages)
 430             processed = []
 431             while True:
 432                 exceptions = []
 433                 for m in j['messages']:
 434                     if m in processed:
 435                         continue
 436                     try:
 437                         msg = self.message_class(self.logger, m, self)
 438                         if msg.name in self.messages:
 439                             raise ParseError(
 440                                 "Duplicate message `%s'" % msg.name)
 441                     except ParseError as e:
 442                         exceptions.append(e)
 443                         continue
 444                     self.messages[msg.name] = msg
 445                     self.messages_by_json[path][msg.name] = msg
 446                     processed.append(m)
 447                 if prev_length == len(self.messages):
 448                     # cannot make forward progress ...
 449                     self.exceptions.extend(exceptions)
 450                     break
 451                 prev_length = len(self.messages)
 452
 453     def lookup_type_like_id(self, name):
 454         mundane_name = remove_magic(name)
 455         if name in self.types:
 456             return self.types[name]
 457         elif name in self.enums:
 458             return self.enums[name]
 459         elif name in self.unions:
 460             return self.unions[name]
 461         elif name in self.aliases:
 462             return self.aliases[name]
 463         elif mundane_name in self.types:
 464             return self.types[mundane_name]
 465         elif mundane_name in self.enums:
 466             return self.enums[mundane_name]
 467         elif mundane_name in self.unions:
 468             return self.unions[mundane_name]
 469         elif mundane_name in self.aliases:
 470             return self.aliases[mundane_name]
 471         raise ParseError(
 472             "Could not find type, enum or union by magic name `%s' nor by "
 473             "mundane name `%s'" % (name, mundane_name))
 474
 475     def is_reply(self, message):
 476         return message in self.replies
 477
 478     def is_event(self, message):
 479         return message in self.events
 480
 481     def get_reply(self, message):
 482         return self.messages[self.services[message]['reply']]
 483
 484     def finalize_parsing(self):
 485         if len(self.messages) == 0:
 486             for e in self.exceptions:
 487                 self.logger.warning(e)
 488         for jn, j in self.messages_by_json.items():
 489             remove = []
 490             for n, m in j.items():
 491                 try:
 492                     if not m.is_reply and not m.is_event:
 493                         try:
 494                             m.reply = self.get_reply(n)
 495                             if "stream" in self.services[m.name]:
 496                                 m.reply_is_stream = \
 497                                     self.services[m.name]["stream"]
 498                             else:
 499                                 m.reply_is_stream = False
 500                             m.reply.request = m
 501                         except:
 502                             raise ParseError(
 503                                 "Cannot find reply to message `%s'" % n)
 504                 except ParseError as e:
 505                     self.exceptions.append(e)
 506                     remove.append(n)
 507
 508             self.messages_by_json[jn] = {
 509                 k: v for k, v in j.items() if k not in remove}