src/vpp-api/vapi/vapi_json_parser.py

   1 #!/usr/bin/env python2
   2
   3 import json
   4
   5
   6 class ParseError (Exception):
   7     pass
   8
   9
  10 magic_prefix = "vl_api_"
  11 magic_suffix = "_t"
  12
  13
  14 def remove_magic(what):
  15     if what.startswith(magic_prefix) and what.endswith(magic_suffix):
  16         return what[len(magic_prefix): - len(magic_suffix)]
  17     return what
  18
  19
  20 class Field(object):
  21
  22     def __init__(self, field_name, field_type, array_len=None,
  23                  nelem_field=None):
  24         self.name = field_name
  25         self.type = field_type
  26         self.len = array_len
  27         self.nelem_field = nelem_field
  28
  29     def __str__(self):
  30         if self.len is None:
  31             return "name: %s, type: %s" % (self.name, self.type)
  32         elif self.len > 0:
  33             return "name: %s, type: %s, length: %s" % (self.name, self.type,
  34                                                        self.len)
  35         else:
  36             return ("name: %s, type: %s, variable length stored in: %s" %
  37                     (self.name, self.type, self.nelem_field))
  38
  39
  40 class Type(object):
  41     def __init__(self, name):
  42         self.name = name
  43
  44     def __str__(self):
  45         return self.name
  46
  47
  48 class SimpleType (Type):
  49
  50     def __init__(self, name):
  51         super(SimpleType, self).__init__(name)
  52
  53     def __str__(self):
  54         return self.name
  55
  56
  57 def get_msg_header_defs(struct_type_class, field_class, json_parser, logger):
  58     return [
  59         struct_type_class(['msg_header1_t',
  60                            ['u16', '_vl_msg_id'],
  61                            ['u32', 'context'],
  62                            ],
  63                           json_parser, field_class, logger
  64                           ),
  65         struct_type_class(['msg_header2_t',
  66                            ['u16', '_vl_msg_id'],
  67                            ['u32', 'client_index'],
  68                            ['u32', 'context'],
  69                            ],
  70                           json_parser, field_class, logger
  71                           ),
  72     ]
  73
  74
  75 class Struct(object):
  76
  77     def __init__(self, name, fields):
  78         self.name = name
  79         self.fields = fields
  80         self.field_names = [n.name for n in self.fields]
  81         self.depends = [f.type for f in self.fields]
  82
  83     def __str__(self):
  84         return "[%s]" % "], [".join([str(f) for f in self.fields])
  85
  86
  87 class Enum(SimpleType):
  88     def __init__(self, name, value_pairs, enumtype):
  89         super(Enum, self).__init__(name)
  90         self.type = enumtype
  91         self.value_pairs = value_pairs
  92
  93     def __str__(self):
  94         return "Enum(%s, [%s])" % (
  95             self.name,
  96             "], [" .join(["%s => %s" % (i, j) for i, j in self.value_pairs])
  97         )
  98
  99
 100 class Union(Type):
 101     def __init__(self, name, type_pairs, crc):
 102         Type.__init__(self, name)
 103         self.crc = crc
 104         self.type_pairs = type_pairs
 105         self.depends = [t for t, _ in self.type_pairs]
 106
 107     def __str__(self):
 108         return "Union(%s, [%s])" % (
 109             self.name,
 110             "], [" .join(["%s %s" % (i, j) for i, j in self.type_pairs])
 111         )
 112
 113
 114 class Message(object):
 115
 116     def __init__(self, logger, definition, json_parser):
 117         struct_type_class = json_parser.struct_type_class
 118         field_class = json_parser.field_class
 119         self.request = None
 120         self.logger = logger
 121         m = definition
 122         logger.debug("Parsing message definition `%s'" % m)
 123         name = m[0]
 124         self.name = name
 125         logger.debug("Message name is `%s'" % name)
 126         ignore = True
 127         self.header = None
 128         self.is_reply = json_parser.is_reply(self.name)
 129         self.is_event = json_parser.is_event(self.name)
 130         fields = []
 131         for header in get_msg_header_defs(struct_type_class, field_class,
 132                                           json_parser, logger):
 133             logger.debug("Probing header `%s'" % header.name)
 134             if header.is_part_of_def(m[1:]):
 135                 self.header = header
 136                 logger.debug("Found header `%s'" % header.name)
 137                 fields.append(field_class(field_name='header',
 138                                           field_type=self.header))
 139                 ignore = False
 140                 break
 141         if ignore and not self.is_event and not self.is_reply:
 142             raise ParseError("While parsing message `%s': could not find all "
 143                              "common header fields" % name)
 144         for field in m[1:]:
 145             if len(field) == 1 and 'crc' in field:
 146                 self.crc = field['crc']
 147                 logger.debug("Found CRC `%s'" % self.crc)
 148                 continue
 149             else:
 150                 field_type = json_parser.lookup_type_like_id(field[0])
 151                 logger.debug("Parsing message field `%s'" % field)
 152                 if len(field) == 2:
 153                     if self.header is not None and\
 154                             self.header.has_field(field[1]):
 155                         continue
 156                     p = field_class(field_name=field[1],
 157                                     field_type=field_type)
 158                 elif len(field) == 3:
 159                     if field[2] == 0:
 160                         raise ParseError(
 161                             "While parsing message `%s': variable length "
 162                             "array `%s' doesn't have reference to member "
 163                             "containing the actual length" % (
 164                                 name, field[1]))
 165                     p = field_class(
 166                         field_name=field[1],
 167                         field_type=field_type,
 168                         array_len=field[2])
 169                 elif len(field) == 4:
 170                     nelem_field = None
 171                     for f in fields:
 172                         if f.name == field[3]:
 173                             nelem_field = f
 174                     if nelem_field is None:
 175                         raise ParseError(
 176                             "While parsing message `%s': couldn't find "
 177                             "variable length array `%s' member containing "
 178                             "the actual length `%s'" % (
 179                                 name, field[1], field[3]))
 180                     p = field_class(
 181                         field_name=field[1],
 182                         field_type=field_type,
 183                         array_len=field[2],
 184                         nelem_field=nelem_field)
 185                 else:
 186                     raise Exception("Don't know how to parse message "
 187                                     "definition for message `%s': `%s'" %
 188                                     (m, m[1:]))
 189                 logger.debug("Parsed field `%s'" % p)
 190                 fields.append(p)
 191         self.fields = fields
 192         self.depends = [f.type for f in self.fields]
 193
 194
 195 class StructType (Type, Struct):
 196
 197     def __init__(self, definition, json_parser, field_class, logger):
 198         t = definition
 199         logger.debug("Parsing struct definition `%s'" % t)
 200         name = t[0]
 201         fields = []
 202         for field in t[1:]:
 203             if len(field) == 1 and 'crc' in field:
 204                 self.crc = field['crc']
 205                 continue
 206             field_type = json_parser.lookup_type_like_id(field[0])
 207             logger.debug("Parsing type field `%s'" % field)
 208             if len(field) == 2:
 209                 p = field_class(field_name=field[1],
 210                                 field_type=field_type)
 211             elif len(field) == 3:
 212                 if field[2] == 0:
 213                     raise ParseError("While parsing type `%s': array `%s' has "
 214                                      "variable length" % (name, field[1]))
 215                 p = field_class(field_name=field[1],
 216                                 field_type=field_type,
 217                                 array_len=field[2])
 218             elif len(field) == 4:
 219                 nelem_field = None
 220                 for f in fields:
 221                     if f.name == field[3]:
 222                         nelem_field = f
 223                 if nelem_field is None:
 224                     raise ParseError(
 225                         "While parsing message `%s': couldn't find "
 226                         "variable length array `%s' member containing "
 227                         "the actual length `%s'" % (
 228                             name, field[1], field[3]))
 229                 p = field_class(field_name=field[1],
 230                                 field_type=field_type,
 231                                 array_len=field[2],
 232                                 nelem_field=nelem_field)
 233             else:
 234                 raise ParseError(
 235                     "Don't know how to parse field `%s' of type definition "
 236                     "for type `%s'" % (field, t))
 237             fields.append(p)
 238         Type.__init__(self, name)
 239         Struct.__init__(self, name, fields)
 240
 241     def __str__(self):
 242         return "StructType(%s, %s)" % (Type.__str__(self),
 243                                        Struct.__str__(self))
 244
 245     def has_field(self, name):
 246         return name in self.field_names
 247
 248     def is_part_of_def(self, definition):
 249         for idx in range(len(self.fields)):
 250             field = definition[idx]
 251             p = self.fields[idx]
 252             if field[1] != p.name:
 253                 return False
 254             if field[0] != p.type.name:
 255                 raise ParseError(
 256                     "Unexpected field type `%s' (should be `%s'), "
 257                     "while parsing msg/def/field `%s/%s/%s'" %
 258                     (field[0], p.type, p.name, definition, field))
 259         return True
 260
 261
 262 class JsonParser(object):
 263     def __init__(self, logger, files, simple_type_class=SimpleType,
 264                  enum_class=Enum, union_class=Union,
 265                  struct_type_class=StructType, field_class=Field,
 266                  message_class=Message):
 267         self.services = {}
 268         self.messages = {}
 269         self.enums = {}
 270         self.unions = {}
 271         self.types = {
 272             x: simple_type_class(x) for x in [
 273                 'i8', 'i16', 'i32', 'i64',
 274                 'u8', 'u16', 'u32', 'u64',
 275                 'f64'
 276             ]
 277         }
 278
 279         self.replies = set()
 280         self.events = set()
 281         self.simple_type_class = simple_type_class
 282         self.enum_class = enum_class
 283         self.union_class = union_class
 284         self.struct_type_class = struct_type_class
 285         self.field_class = field_class
 286         self.message_class = message_class
 287
 288         self.exceptions = []
 289         self.json_files = []
 290         self.types_by_json = {}
 291         self.enums_by_json = {}
 292         self.unions_by_json = {}
 293         self.messages_by_json = {}
 294         self.logger = logger
 295         for f in files:
 296             self.parse_json_file(f)
 297         self.finalize_parsing()
 298
 299     def parse_json_file(self, path):
 300         self.logger.info("Parsing json api file: `%s'" % path)
 301         self.json_files.append(path)
 302         self.types_by_json[path] = []
 303         self.enums_by_json[path] = []
 304         self.unions_by_json[path] = []
 305         self.messages_by_json[path] = {}
 306         with open(path) as f:
 307             j = json.load(f)
 308             for k in j['services']:
 309                 if k in self.services:
 310                     raise ParseError("Duplicate service `%s'" % k)
 311                 self.services[k] = j['services'][k]
 312                 self.replies.add(self.services[k]["reply"])
 313                 if "events" in self.services[k]:
 314                     for x in self.services[k]["events"]:
 315                         self.events.add(x)
 316             for e in j['enums']:
 317                 name = e[0]
 318                 value_pairs = e[1:-1]
 319                 enumtype = self.types[e[-1]["enumtype"]]
 320                 enum = self.enum_class(name, value_pairs, enumtype)
 321                 self.enums[enum.name] = enum
 322                 self.logger.debug("Parsed enum: %s" % enum)
 323                 self.enums_by_json[path].append(enum)
 324             exceptions = []
 325             progress = 0
 326             last_progress = 0
 327             while True:
 328                 for u in j['unions']:
 329                     name = u[0]
 330                     if name in self.unions:
 331                         progress = progress + 1
 332                         continue
 333                     try:
 334                         type_pairs = [[self.lookup_type_like_id(t), n]
 335                                       for t, n in u[1:-1]]
 336                         crc = u[-1]["crc"]
 337                         union = self.union_class(name, type_pairs, crc)
 338                         progress = progress + 1
 339                     except ParseError as e:
 340                         exceptions.append(e)
 341                         continue
 342                     self.unions[union.name] = union
 343                     self.logger.debug("Parsed union: %s" % union)
 344                     self.unions_by_json[path].append(union)
 345                 for t in j['types']:
 346                     if t[0] in self.types:
 347                         progress = progress + 1
 348                         continue
 349                     try:
 350                         type_ = self.struct_type_class(t, self,
 351                                                        self.field_class,
 352                                                        self.logger)
 353                         if type_.name in self.types:
 354                             raise ParseError(
 355                                 "Duplicate type `%s'" % type_.name)
 356                         progress = progress + 1
 357                     except ParseError as e:
 358                         exceptions.append(e)
 359                         continue
 360                     self.types[type_.name] = type_
 361                     self.types_by_json[path].append(type_)
 362                     self.logger.debug("Parsed type: %s" % type_)
 363                 if not exceptions:
 364                     # finished parsing
 365                     break
 366                 if progress <= last_progress:
 367                     # cannot make forward progress
 368                     self.exceptions.extend(exceptions)
 369                 exceptions = []
 370                 last_progress = progress
 371                 progress = 0
 372             prev_length = len(self.messages)
 373             processed = []
 374             while True:
 375                 exceptions = []
 376                 for m in j['messages']:
 377                     if m in processed:
 378                         continue
 379                     try:
 380                         msg = self.message_class(self.logger, m, self)
 381                         if msg.name in self.messages:
 382                             raise ParseError(
 383                                 "Duplicate message `%s'" % msg.name)
 384                     except ParseError as e:
 385                         exceptions.append(e)
 386                         continue
 387                     self.messages[msg.name] = msg
 388                     self.messages_by_json[path][msg.name] = msg
 389                     processed.append(m)
 390                 if prev_length == len(self.messages):
 391                     # cannot make forward progress ...
 392                     self.exceptions.extend(exceptions)
 393                     break
 394                 prev_length = len(self.messages)
 395
 396     def lookup_type_like_id(self, name):
 397         mundane_name = remove_magic(name)
 398         if name in self.types:
 399             return self.types[name]
 400         elif name in self.enums:
 401             return self.enums[name]
 402         elif name in self.unions:
 403             return self.unions[name]
 404         elif mundane_name in self.types:
 405             return self.types[mundane_name]
 406         elif mundane_name in self.enums:
 407             return self.enums[mundane_name]
 408         elif mundane_name in self.unions:
 409             return self.unions[mundane_name]
 410         raise ParseError(
 411             "Could not find type, enum or union by magic name `%s' nor by "
 412             "mundane name `%s'" % (name, mundane_name))
 413
 414     def is_reply(self, message):
 415         return message in self.replies
 416
 417     def is_event(self, message):
 418         return message in self.events
 419
 420     def get_reply(self, message):
 421         return self.messages[self.services[message]['reply']]
 422
 423     def finalize_parsing(self):
 424         if len(self.messages) == 0:
 425             for e in self.exceptions:
 426                 self.logger.warning(e)
 427         for jn, j in self.messages_by_json.items():
 428             remove = []
 429             for n, m in j.items():
 430                 try:
 431                     if not m.is_reply and not m.is_event:
 432                         try:
 433                             m.reply = self.get_reply(n)
 434                             if "stream" in self.services[m.name]:
 435                                 m.reply_is_stream = \
 436                                     self.services[m.name]["stream"]
 437                             else:
 438                                 m.reply_is_stream = False
 439                             m.reply.request = m
 440                         except:
 441                             raise ParseError(
 442                                 "Cannot find reply to message `%s'" % n)
 443                 except ParseError as e:
 444                     self.exceptions.append(e)
 445                     remove.append(n)
 446
 447             self.messages_by_json[jn] = {
 448                 k: v for k, v in j.items() if k not in remove}