src/vpp-api/vapi/vapi_json_parser.py

   1 #!/usr/bin/env python2
   2
   3 import json
   4
   5
   6 class ParseError (Exception):
   7     pass
   8
   9
  10 magic_prefix = "vl_api_"
  11 magic_suffix = "_t"
  12
  13
  14 def remove_magic(what):
  15     if what.startswith(magic_prefix) and what.endswith(magic_suffix):
  16         return what[len(magic_prefix): - len(magic_suffix)]
  17     return what
  18
  19
  20 class Field(object):
  21
  22     def __init__(self, field_name, field_type, array_len=None,
  23                  nelem_field=None):
  24         self.name = field_name
  25         self.type = field_type
  26         self.len = array_len
  27         self.nelem_field = nelem_field
  28
  29     def __str__(self):
  30         if self.len is None:
  31             return "Field(name: %s, type: %s)" % (self.name, self.type)
  32         elif self.len > 0:
  33             return "Field(name: %s, type: %s, length: %s)" % (self.name,
  34                                                               self.type,
  35                                                               self.len)
  36         else:
  37             return (
  38                 "Field(name: %s, type: %s, variable length stored in: %s)" %
  39                 (self.name, self.type, self.nelem_field))
  40
  41     def is_vla(self):
  42         return self.nelem_field is not None
  43
  44     def has_vla(self):
  45         return self.is_vla() or self.type.has_vla()
  46
  47
  48 class Alias(Field):
  49     pass
  50
  51
  52 class Type(object):
  53     def __init__(self, name):
  54         self.name = name
  55
  56     def __str__(self):
  57         return self.name
  58
  59
  60 class SimpleType (Type):
  61
  62     def has_vla(self):
  63         return False
  64
  65
  66 def get_msg_header_defs(struct_type_class, field_class, json_parser, logger):
  67     return [
  68         struct_type_class(['msg_header1_t',
  69                            ['u16', '_vl_msg_id'],
  70                            ['u32', 'context'],
  71                            ],
  72                           json_parser, field_class, logger
  73                           ),
  74         struct_type_class(['msg_header2_t',
  75                            ['u16', '_vl_msg_id'],
  76                            ['u32', 'client_index'],
  77                            ['u32', 'context'],
  78                            ],
  79                           json_parser, field_class, logger
  80                           ),
  81     ]
  82
  83
  84 class Struct(object):
  85
  86     def __init__(self, name, fields):
  87         self.name = name
  88         self.fields = fields
  89         self.field_names = [n.name for n in self.fields]
  90         self.depends = [f.type for f in self.fields]
  91
  92     def __str__(self):
  93         return "[%s]" % "], [".join([str(f) for f in self.fields])
  94
  95     def has_vla(self):
  96         for f in self.fields:
  97             if f.has_vla():
  98                 return True
  99         return False
 100
 101
 102 class Enum(SimpleType):
 103     def __init__(self, name, value_pairs, enumtype):
 104         super(Enum, self).__init__(name)
 105         self.type = enumtype
 106         self.value_pairs = value_pairs
 107
 108     def __str__(self):
 109         return "Enum(%s, [%s])" % (
 110             self.name,
 111             "], [" .join(["%s => %s" % (i, j) for i, j in self.value_pairs])
 112         )
 113
 114
 115 class Union(Type):
 116     def __init__(self, name, type_pairs, crc):
 117         Type.__init__(self, name)
 118         self.crc = crc
 119         self.type_pairs = type_pairs
 120         self.depends = [t for t, _ in self.type_pairs]
 121
 122     def __str__(self):
 123         return "Union(%s, [%s])" % (
 124             self.name,
 125             "], [" .join(["%s %s" % (i, j) for i, j in self.type_pairs])
 126         )
 127
 128     def has_vla(self):
 129         return False
 130
 131
 132 class Message(object):
 133
 134     def __init__(self, logger, definition, json_parser):
 135         struct_type_class = json_parser.struct_type_class
 136         field_class = json_parser.field_class
 137         self.request = None
 138         self.logger = logger
 139         m = definition
 140         logger.debug("Parsing message definition `%s'" % m)
 141         name = m[0]
 142         self.name = name
 143         logger.debug("Message name is `%s'" % name)
 144         ignore = True
 145         self.header = None
 146         self.is_reply = json_parser.is_reply(self.name)
 147         self.is_event = json_parser.is_event(self.name)
 148         fields = []
 149         for header in get_msg_header_defs(struct_type_class, field_class,
 150                                           json_parser, logger):
 151             logger.debug("Probing header `%s'" % header.name)
 152             if header.is_part_of_def(m[1:]):
 153                 self.header = header
 154                 logger.debug("Found header `%s'" % header.name)
 155                 fields.append(field_class(field_name='header',
 156                                           field_type=self.header))
 157                 ignore = False
 158                 break
 159         if ignore and not self.is_event and not self.is_reply:
 160             raise ParseError("While parsing message `%s': could not find all "
 161                              "common header fields" % name)
 162         for field in m[1:]:
 163             if len(field) == 1 and 'crc' in field:
 164                 self.crc = field['crc']
 165                 logger.debug("Found CRC `%s'" % self.crc)
 166                 continue
 167             else:
 168                 field_type = json_parser.lookup_type_like_id(field[0])
 169                 logger.debug("Parsing message field `%s'" % field)
 170                 l = len(field)
 171                 if any(type(n) is dict for n in field):
 172                     l -= 1
 173                 if l == 2:
 174                     if self.header is not None and\
 175                             self.header.has_field(field[1]):
 176                         continue
 177                     p = field_class(field_name=field[1],
 178                                     field_type=field_type)
 179                 elif l == 3:
 180                     if field[2] == 0:
 181                         raise ParseError(
 182                             "While parsing message `%s': variable length "
 183                             "array `%s' doesn't have reference to member "
 184                             "containing the actual length" % (
 185                                 name, field[1]))
 186                     p = field_class(
 187                         field_name=field[1],
 188                         field_type=field_type,
 189                         array_len=field[2])
 190                 elif l == 4:
 191                     nelem_field = None
 192                     for f in fields:
 193                         if f.name == field[3]:
 194                             nelem_field = f
 195                     if nelem_field is None:
 196                         raise ParseError(
 197                             "While parsing message `%s': couldn't find "
 198                             "variable length array `%s' member containing "
 199                             "the actual length `%s'" % (
 200                                 name, field[1], field[3]))
 201                     p = field_class(
 202                         field_name=field[1],
 203                         field_type=field_type,
 204                         array_len=field[2],
 205                         nelem_field=nelem_field)
 206                 else:
 207                     raise Exception("Don't know how to parse message "
 208                                     "definition for message `%s': `%s'" %
 209                                     (m, m[1:]))
 210                 logger.debug("Parsed field `%s'" % p)
 211                 fields.append(p)
 212         self.fields = fields
 213         self.depends = [f.type for f in self.fields]
 214         logger.debug("Parsed message: %s" % self)
 215
 216     def __str__(self):
 217         return "Message(%s, [%s], {crc: %s}" % \
 218             (self.name,
 219              "], [".join([str(f) for f in self.fields]),
 220              self.crc)
 221
 222
 223 class StructType (Type, Struct):
 224
 225     def __init__(self, definition, json_parser, field_class, logger):
 226         t = definition
 227         logger.debug("Parsing struct definition `%s'" % t)
 228         name = t[0]
 229         fields = []
 230         for field in t[1:]:
 231             if len(field) == 1 and 'crc' in field:
 232                 self.crc = field['crc']
 233                 continue
 234             field_type = json_parser.lookup_type_like_id(field[0])
 235             logger.debug("Parsing type field `%s'" % field)
 236             if len(field) == 2:
 237                 p = field_class(field_name=field[1],
 238                                 field_type=field_type)
 239             elif len(field) == 3:
 240                 if field[2] == 0:
 241                     raise ParseError("While parsing type `%s': array `%s' has "
 242                                      "variable length" % (name, field[1]))
 243                 p = field_class(field_name=field[1],
 244                                 field_type=field_type,
 245                                 array_len=field[2])
 246             elif len(field) == 4:
 247                 nelem_field = None
 248                 for f in fields:
 249                     if f.name == field[3]:
 250                         nelem_field = f
 251                 if nelem_field is None:
 252                     raise ParseError(
 253                         "While parsing message `%s': couldn't find "
 254                         "variable length array `%s' member containing "
 255                         "the actual length `%s'" % (
 256                             name, field[1], field[3]))
 257                 p = field_class(field_name=field[1],
 258                                 field_type=field_type,
 259                                 array_len=field[2],
 260                                 nelem_field=nelem_field)
 261             else:
 262                 raise ParseError(
 263                     "Don't know how to parse field `%s' of type definition "
 264                     "for type `%s'" % (field, t))
 265             fields.append(p)
 266         Type.__init__(self, name)
 267         Struct.__init__(self, name, fields)
 268
 269     def __str__(self):
 270         return "StructType(%s, %s)" % (Type.__str__(self),
 271                                        Struct.__str__(self))
 272
 273     def has_field(self, name):
 274         return name in self.field_names
 275
 276     def is_part_of_def(self, definition):
 277         for idx in range(len(self.fields)):
 278             field = definition[idx]
 279             p = self.fields[idx]
 280             if field[1] != p.name:
 281                 return False
 282             if field[0] != p.type.name:
 283                 raise ParseError(
 284                     "Unexpected field type `%s' (should be `%s'), "
 285                     "while parsing msg/def/field `%s/%s/%s'" %
 286                     (field[0], p.type, p.name, definition, field))
 287         return True
 288
 289
 290 class JsonParser(object):
 291     def __init__(self, logger, files, simple_type_class=SimpleType,
 292                  enum_class=Enum, union_class=Union,
 293                  struct_type_class=StructType, field_class=Field,
 294                  message_class=Message, alias_class=Alias):
 295         self.services = {}
 296         self.messages = {}
 297         self.enums = {}
 298         self.unions = {}
 299         self.aliases = {}
 300         self.types = {
 301             x: simple_type_class(x) for x in [
 302                 'i8', 'i16', 'i32', 'i64',
 303                 'u8', 'u16', 'u32', 'u64',
 304                 'f64', 'bool'
 305             ]
 306         }
 307
 308         self.types['string'] = simple_type_class('vl_api_string_t')
 309         self.replies = set()
 310         self.events = set()
 311         self.simple_type_class = simple_type_class
 312         self.enum_class = enum_class
 313         self.union_class = union_class
 314         self.struct_type_class = struct_type_class
 315         self.field_class = field_class
 316         self.alias_class = alias_class
 317         self.message_class = message_class
 318
 319         self.exceptions = []
 320         self.json_files = []
 321         self.types_by_json = {}
 322         self.enums_by_json = {}
 323         self.unions_by_json = {}
 324         self.aliases_by_json = {}
 325         self.messages_by_json = {}
 326         self.logger = logger
 327         for f in files:
 328             self.parse_json_file(f)
 329         self.finalize_parsing()
 330
 331     def parse_json_file(self, path):
 332         self.logger.info("Parsing json api file: `%s'" % path)
 333         self.json_files.append(path)
 334         self.types_by_json[path] = []
 335         self.enums_by_json[path] = []
 336         self.unions_by_json[path] = []
 337         self.aliases_by_json[path] = []
 338         self.messages_by_json[path] = {}
 339         with open(path) as f:
 340             j = json.load(f)
 341             for k in j['services']:
 342                 if k in self.services:
 343                     raise ParseError("Duplicate service `%s'" % k)
 344                 self.services[k] = j['services'][k]
 345                 self.replies.add(self.services[k]["reply"])
 346                 if "events" in self.services[k]:
 347                     for x in self.services[k]["events"]:
 348                         self.events.add(x)
 349             for e in j['enums']:
 350                 name = e[0]
 351                 value_pairs = e[1:-1]
 352                 enumtype = self.types[e[-1]["enumtype"]]
 353                 enum = self.enum_class(name, value_pairs, enumtype)
 354                 self.enums[enum.name] = enum
 355                 self.logger.debug("Parsed enum: %s" % enum)
 356                 self.enums_by_json[path].append(enum)
 357             exceptions = []
 358             progress = 0
 359             last_progress = 0
 360             while True:
 361                 for u in j['unions']:
 362                     name = u[0]
 363                     if name in self.unions:
 364                         progress = progress + 1
 365                         continue
 366                     try:
 367                         type_pairs = [[self.lookup_type_like_id(t), n]
 368                                       for t, n in u[1:]]
 369                         union = self.union_class(name, type_pairs, 0)
 370                         progress = progress + 1
 371                     except ParseError as e:
 372                         exceptions.append(e)
 373                         continue
 374                     self.unions[union.name] = union
 375                     self.logger.debug("Parsed union: %s" % union)
 376                     self.unions_by_json[path].append(union)
 377                 for name, body in j['aliases'].iteritems():
 378                     if name in self.aliases:
 379                         progress = progress + 1
 380                         continue
 381                     if 'length' in body:
 382                         array_len = body['length']
 383                     else:
 384                         array_len = None
 385                     t = self.types[body['type']]
 386                     alias = self.alias_class(name, t, array_len)
 387                     self.aliases[name] = alias
 388                     self.logger.debug("Parsed alias: %s" % alias)
 389                     self.aliases_by_json[path].append(alias)
 390                 for t in j['types']:
 391                     if t[0] in self.types:
 392                         progress = progress + 1
 393                         continue
 394                     try:
 395                         type_ = self.struct_type_class(t, self,
 396                                                        self.field_class,
 397                                                        self.logger)
 398                         if type_.name in self.types:
 399                             raise ParseError(
 400                                 "Duplicate type `%s'" % type_.name)
 401                         progress = progress + 1
 402                     except ParseError as e:
 403                         exceptions.append(e)
 404                         continue
 405                     self.types[type_.name] = type_
 406                     self.types_by_json[path].append(type_)
 407                     self.logger.debug("Parsed type: %s" % type_)
 408                 if not exceptions:
 409                     # finished parsing
 410                     break
 411                 if progress <= last_progress:
 412                     # cannot make forward progress
 413                     self.exceptions.extend(exceptions)
 414                     break
 415                 exceptions = []
 416                 last_progress = progress
 417                 progress = 0
 418             prev_length = len(self.messages)
 419             processed = []
 420             while True:
 421                 exceptions = []
 422                 for m in j['messages']:
 423                     if m in processed:
 424                         continue
 425                     try:
 426                         msg = self.message_class(self.logger, m, self)
 427                         if msg.name in self.messages:
 428                             raise ParseError(
 429                                 "Duplicate message `%s'" % msg.name)
 430                     except ParseError as e:
 431                         exceptions.append(e)
 432                         continue
 433                     self.messages[msg.name] = msg
 434                     self.messages_by_json[path][msg.name] = msg
 435                     processed.append(m)
 436                 if prev_length == len(self.messages):
 437                     # cannot make forward progress ...
 438                     self.exceptions.extend(exceptions)
 439                     break
 440                 prev_length = len(self.messages)
 441
 442     def lookup_type_like_id(self, name):
 443         mundane_name = remove_magic(name)
 444         if name in self.types:
 445             return self.types[name]
 446         elif name in self.enums:
 447             return self.enums[name]
 448         elif name in self.unions:
 449             return self.unions[name]
 450         elif name in self.aliases:
 451             return self.aliases[name]
 452         elif mundane_name in self.types:
 453             return self.types[mundane_name]
 454         elif mundane_name in self.enums:
 455             return self.enums[mundane_name]
 456         elif mundane_name in self.unions:
 457             return self.unions[mundane_name]
 458         elif mundane_name in self.aliases:
 459             return self.aliases[mundane_name]
 460         raise ParseError(
 461             "Could not find type, enum or union by magic name `%s' nor by "
 462             "mundane name `%s'" % (name, mundane_name))
 463
 464     def is_reply(self, message):
 465         return message in self.replies
 466
 467     def is_event(self, message):
 468         return message in self.events
 469
 470     def get_reply(self, message):
 471         return self.messages[self.services[message]['reply']]
 472
 473     def finalize_parsing(self):
 474         if len(self.messages) == 0:
 475             for e in self.exceptions:
 476                 self.logger.warning(e)
 477         for jn, j in self.messages_by_json.items():
 478             remove = []
 479             for n, m in j.items():
 480                 try:
 481                     if not m.is_reply and not m.is_event:
 482                         try:
 483                             m.reply = self.get_reply(n)
 484                             if "stream" in self.services[m.name]:
 485                                 m.reply_is_stream = \
 486                                     self.services[m.name]["stream"]
 487                             else:
 488                                 m.reply_is_stream = False
 489                             m.reply.request = m
 490                         except:
 491                             raise ParseError(
 492                                 "Cannot find reply to message `%s'" % n)
 493                 except ParseError as e:
 494                     self.exceptions.append(e)
 495                     remove.append(n)
 496
 497             self.messages_by_json[jn] = {
 498                 k: v for k, v in j.items() if k not in remove}