src/vpp-api/vapi/vapi_json_parser.py

   1 #!/usr/bin/env python2
   2
   3 import json
   4
   5
   6 class ParseError (Exception):
   7     pass
   8
   9
  10 magic_prefix = "vl_api_"
  11 magic_suffix = "_t"
  12
  13
  14 def remove_magic(what):
  15     if what.startswith(magic_prefix) and what.endswith(magic_suffix):
  16         return what[len(magic_prefix): - len(magic_suffix)]
  17     return what
  18
  19
  20 class Field(object):
  21
  22     def __init__(self, field_name, field_type, array_len=None,
  23                  nelem_field=None):
  24         self.name = field_name
  25         self.type = field_type
  26         self.len = array_len
  27         self.nelem_field = nelem_field
  28
  29     def __str__(self):
  30         if self.len is None:
  31             return "Field(name: %s, type: %s)" % (self.name, self.type)
  32         elif self.len > 0:
  33             return "Field(name: %s, type: %s, length: %s)" % (self.name,
  34                                                               self.type,
  35                                                               self.len)
  36         else:
  37             return (
  38                 "Field(name: %s, type: %s, variable length stored in: %s)" %
  39                 (self.name, self.type, self.nelem_field))
  40
  41     def is_vla(self):
  42         return self.nelem_field is not None
  43
  44     def has_vla(self):
  45         return self.is_vla() or self.type.has_vla()
  46
  47
  48 class Type(object):
  49     def __init__(self, name):
  50         self.name = name
  51
  52     def __str__(self):
  53         return self.name
  54
  55
  56 class SimpleType (Type):
  57
  58     def __init__(self, name):
  59         super(SimpleType, self).__init__(name)
  60
  61     def __str__(self):
  62         return self.name
  63
  64     def has_vla(self):
  65         return False
  66
  67
  68 def get_msg_header_defs(struct_type_class, field_class, json_parser, logger):
  69     return [
  70         struct_type_class(['msg_header1_t',
  71                            ['u16', '_vl_msg_id'],
  72                            ['u32', 'context'],
  73                            ],
  74                           json_parser, field_class, logger
  75                           ),
  76         struct_type_class(['msg_header2_t',
  77                            ['u16', '_vl_msg_id'],
  78                            ['u32', 'client_index'],
  79                            ['u32', 'context'],
  80                            ],
  81                           json_parser, field_class, logger
  82                           ),
  83     ]
  84
  85
  86 class Struct(object):
  87
  88     def __init__(self, name, fields):
  89         self.name = name
  90         self.fields = fields
  91         self.field_names = [n.name for n in self.fields]
  92         self.depends = [f.type for f in self.fields]
  93
  94     def __str__(self):
  95         return "[%s]" % "], [".join([str(f) for f in self.fields])
  96
  97     def has_vla(self):
  98         for f in self.fields:
  99             if f.has_vla():
 100                 return True
 101         return False
 102
 103
 104 class Enum(SimpleType):
 105     def __init__(self, name, value_pairs, enumtype):
 106         super(Enum, self).__init__(name)
 107         self.type = enumtype
 108         self.value_pairs = value_pairs
 109
 110     def __str__(self):
 111         return "Enum(%s, [%s])" % (
 112             self.name,
 113             "], [" .join(["%s => %s" % (i, j) for i, j in self.value_pairs])
 114         )
 115
 116
 117 class Union(Type):
 118     def __init__(self, name, type_pairs, crc):
 119         Type.__init__(self, name)
 120         self.crc = crc
 121         self.type_pairs = type_pairs
 122         self.depends = [t for t, _ in self.type_pairs]
 123
 124     def __str__(self):
 125         return "Union(%s, [%s])" % (
 126             self.name,
 127             "], [" .join(["%s %s" % (i, j) for i, j in self.type_pairs])
 128         )
 129
 130     def has_vla(self):
 131         return False
 132
 133
 134 class Message(object):
 135
 136     def __init__(self, logger, definition, json_parser):
 137         struct_type_class = json_parser.struct_type_class
 138         field_class = json_parser.field_class
 139         self.request = None
 140         self.logger = logger
 141         m = definition
 142         logger.debug("Parsing message definition `%s'" % m)
 143         name = m[0]
 144         self.name = name
 145         logger.debug("Message name is `%s'" % name)
 146         ignore = True
 147         self.header = None
 148         self.is_reply = json_parser.is_reply(self.name)
 149         self.is_event = json_parser.is_event(self.name)
 150         fields = []
 151         for header in get_msg_header_defs(struct_type_class, field_class,
 152                                           json_parser, logger):
 153             logger.debug("Probing header `%s'" % header.name)
 154             if header.is_part_of_def(m[1:]):
 155                 self.header = header
 156                 logger.debug("Found header `%s'" % header.name)
 157                 fields.append(field_class(field_name='header',
 158                                           field_type=self.header))
 159                 ignore = False
 160                 break
 161         if ignore and not self.is_event and not self.is_reply:
 162             raise ParseError("While parsing message `%s': could not find all "
 163                              "common header fields" % name)
 164         for field in m[1:]:
 165             if len(field) == 1 and 'crc' in field:
 166                 self.crc = field['crc']
 167                 logger.debug("Found CRC `%s'" % self.crc)
 168                 continue
 169             else:
 170                 field_type = json_parser.lookup_type_like_id(field[0])
 171                 logger.debug("Parsing message field `%s'" % field)
 172                 if len(field) == 2:
 173                     if self.header is not None and\
 174                             self.header.has_field(field[1]):
 175                         continue
 176                     p = field_class(field_name=field[1],
 177                                     field_type=field_type)
 178                 elif len(field) == 3:
 179                     if field[2] == 0:
 180                         raise ParseError(
 181                             "While parsing message `%s': variable length "
 182                             "array `%s' doesn't have reference to member "
 183                             "containing the actual length" % (
 184                                 name, field[1]))
 185                     p = field_class(
 186                         field_name=field[1],
 187                         field_type=field_type,
 188                         array_len=field[2])
 189                 elif len(field) == 4:
 190                     nelem_field = None
 191                     for f in fields:
 192                         if f.name == field[3]:
 193                             nelem_field = f
 194                     if nelem_field is None:
 195                         raise ParseError(
 196                             "While parsing message `%s': couldn't find "
 197                             "variable length array `%s' member containing "
 198                             "the actual length `%s'" % (
 199                                 name, field[1], field[3]))
 200                     p = field_class(
 201                         field_name=field[1],
 202                         field_type=field_type,
 203                         array_len=field[2],
 204                         nelem_field=nelem_field)
 205                 else:
 206                     raise Exception("Don't know how to parse message "
 207                                     "definition for message `%s': `%s'" %
 208                                     (m, m[1:]))
 209                 logger.debug("Parsed field `%s'" % p)
 210                 fields.append(p)
 211         self.fields = fields
 212         self.depends = [f.type for f in self.fields]
 213         logger.debug("Parsed message: %s" % self)
 214
 215     def __str__(self):
 216         return "Message(%s, [%s], {crc: %s}" % \
 217             (self.name,
 218              "], [".join([str(f) for f in self.fields]),
 219              self.crc)
 220
 221
 222 class StructType (Type, Struct):
 223
 224     def __init__(self, definition, json_parser, field_class, logger):
 225         t = definition
 226         logger.debug("Parsing struct definition `%s'" % t)
 227         name = t[0]
 228         fields = []
 229         for field in t[1:]:
 230             if len(field) == 1 and 'crc' in field:
 231                 self.crc = field['crc']
 232                 continue
 233             field_type = json_parser.lookup_type_like_id(field[0])
 234             logger.debug("Parsing type field `%s'" % field)
 235             if len(field) == 2:
 236                 p = field_class(field_name=field[1],
 237                                 field_type=field_type)
 238             elif len(field) == 3:
 239                 if field[2] == 0:
 240                     raise ParseError("While parsing type `%s': array `%s' has "
 241                                      "variable length" % (name, field[1]))
 242                 p = field_class(field_name=field[1],
 243                                 field_type=field_type,
 244                                 array_len=field[2])
 245             elif len(field) == 4:
 246                 nelem_field = None
 247                 for f in fields:
 248                     if f.name == field[3]:
 249                         nelem_field = f
 250                 if nelem_field is None:
 251                     raise ParseError(
 252                         "While parsing message `%s': couldn't find "
 253                         "variable length array `%s' member containing "
 254                         "the actual length `%s'" % (
 255                             name, field[1], field[3]))
 256                 p = field_class(field_name=field[1],
 257                                 field_type=field_type,
 258                                 array_len=field[2],
 259                                 nelem_field=nelem_field)
 260             else:
 261                 raise ParseError(
 262                     "Don't know how to parse field `%s' of type definition "
 263                     "for type `%s'" % (field, t))
 264             fields.append(p)
 265         Type.__init__(self, name)
 266         Struct.__init__(self, name, fields)
 267
 268     def __str__(self):
 269         return "StructType(%s, %s)" % (Type.__str__(self),
 270                                        Struct.__str__(self))
 271
 272     def has_field(self, name):
 273         return name in self.field_names
 274
 275     def is_part_of_def(self, definition):
 276         for idx in range(len(self.fields)):
 277             field = definition[idx]
 278             p = self.fields[idx]
 279             if field[1] != p.name:
 280                 return False
 281             if field[0] != p.type.name:
 282                 raise ParseError(
 283                     "Unexpected field type `%s' (should be `%s'), "
 284                     "while parsing msg/def/field `%s/%s/%s'" %
 285                     (field[0], p.type, p.name, definition, field))
 286         return True
 287
 288
 289 class JsonParser(object):
 290     def __init__(self, logger, files, simple_type_class=SimpleType,
 291                  enum_class=Enum, union_class=Union,
 292                  struct_type_class=StructType, field_class=Field,
 293                  message_class=Message):
 294         self.services = {}
 295         self.messages = {}
 296         self.enums = {}
 297         self.unions = {}
 298         self.types = {
 299             x: simple_type_class(x) for x in [
 300                 'i8', 'i16', 'i32', 'i64',
 301                 'u8', 'u16', 'u32', 'u64',
 302                 'f64'
 303             ]
 304         }
 305
 306         self.replies = set()
 307         self.events = set()
 308         self.simple_type_class = simple_type_class
 309         self.enum_class = enum_class
 310         self.union_class = union_class
 311         self.struct_type_class = struct_type_class
 312         self.field_class = field_class
 313         self.message_class = message_class
 314
 315         self.exceptions = []
 316         self.json_files = []
 317         self.types_by_json = {}
 318         self.enums_by_json = {}
 319         self.unions_by_json = {}
 320         self.messages_by_json = {}
 321         self.logger = logger
 322         for f in files:
 323             self.parse_json_file(f)
 324         self.finalize_parsing()
 325
 326     def parse_json_file(self, path):
 327         self.logger.info("Parsing json api file: `%s'" % path)
 328         self.json_files.append(path)
 329         self.types_by_json[path] = []
 330         self.enums_by_json[path] = []
 331         self.unions_by_json[path] = []
 332         self.messages_by_json[path] = {}
 333         with open(path) as f:
 334             j = json.load(f)
 335             for k in j['services']:
 336                 if k in self.services:
 337                     raise ParseError("Duplicate service `%s'" % k)
 338                 self.services[k] = j['services'][k]
 339                 self.replies.add(self.services[k]["reply"])
 340                 if "events" in self.services[k]:
 341                     for x in self.services[k]["events"]:
 342                         self.events.add(x)
 343             for e in j['enums']:
 344                 name = e[0]
 345                 value_pairs = e[1:-1]
 346                 enumtype = self.types[e[-1]["enumtype"]]
 347                 enum = self.enum_class(name, value_pairs, enumtype)
 348                 self.enums[enum.name] = enum
 349                 self.logger.debug("Parsed enum: %s" % enum)
 350                 self.enums_by_json[path].append(enum)
 351             exceptions = []
 352             progress = 0
 353             last_progress = 0
 354             while True:
 355                 for u in j['unions']:
 356                     name = u[0]
 357                     if name in self.unions:
 358                         progress = progress + 1
 359                         continue
 360                     try:
 361                         type_pairs = [[self.lookup_type_like_id(t), n]
 362                                       for t, n in u[1:-1]]
 363                         crc = u[-1]["crc"]
 364                         union = self.union_class(name, type_pairs, crc)
 365                         progress = progress + 1
 366                     except ParseError as e:
 367                         exceptions.append(e)
 368                         continue
 369                     self.unions[union.name] = union
 370                     self.logger.debug("Parsed union: %s" % union)
 371                     self.unions_by_json[path].append(union)
 372                 for t in j['types']:
 373                     if t[0] in self.types:
 374                         progress = progress + 1
 375                         continue
 376                     try:
 377                         type_ = self.struct_type_class(t, self,
 378                                                        self.field_class,
 379                                                        self.logger)
 380                         if type_.name in self.types:
 381                             raise ParseError(
 382                                 "Duplicate type `%s'" % type_.name)
 383                         progress = progress + 1
 384                     except ParseError as e:
 385                         exceptions.append(e)
 386                         continue
 387                     self.types[type_.name] = type_
 388                     self.types_by_json[path].append(type_)
 389                     self.logger.debug("Parsed type: %s" % type_)
 390                 if not exceptions:
 391                     # finished parsing
 392                     break
 393                 if progress <= last_progress:
 394                     # cannot make forward progress
 395                     self.exceptions.extend(exceptions)
 396                 exceptions = []
 397                 last_progress = progress
 398                 progress = 0
 399             prev_length = len(self.messages)
 400             processed = []
 401             while True:
 402                 exceptions = []
 403                 for m in j['messages']:
 404                     if m in processed:
 405                         continue
 406                     try:
 407                         msg = self.message_class(self.logger, m, self)
 408                         if msg.name in self.messages:
 409                             raise ParseError(
 410                                 "Duplicate message `%s'" % msg.name)
 411                     except ParseError as e:
 412                         exceptions.append(e)
 413                         continue
 414                     self.messages[msg.name] = msg
 415                     self.messages_by_json[path][msg.name] = msg
 416                     processed.append(m)
 417                 if prev_length == len(self.messages):
 418                     # cannot make forward progress ...
 419                     self.exceptions.extend(exceptions)
 420                     break
 421                 prev_length = len(self.messages)
 422
 423     def lookup_type_like_id(self, name):
 424         mundane_name = remove_magic(name)
 425         if name in self.types:
 426             return self.types[name]
 427         elif name in self.enums:
 428             return self.enums[name]
 429         elif name in self.unions:
 430             return self.unions[name]
 431         elif mundane_name in self.types:
 432             return self.types[mundane_name]
 433         elif mundane_name in self.enums:
 434             return self.enums[mundane_name]
 435         elif mundane_name in self.unions:
 436             return self.unions[mundane_name]
 437         raise ParseError(
 438             "Could not find type, enum or union by magic name `%s' nor by "
 439             "mundane name `%s'" % (name, mundane_name))
 440
 441     def is_reply(self, message):
 442         return message in self.replies
 443
 444     def is_event(self, message):
 445         return message in self.events
 446
 447     def get_reply(self, message):
 448         return self.messages[self.services[message]['reply']]
 449
 450     def finalize_parsing(self):
 451         if len(self.messages) == 0:
 452             for e in self.exceptions:
 453                 self.logger.warning(e)
 454         for jn, j in self.messages_by_json.items():
 455             remove = []
 456             for n, m in j.items():
 457                 try:
 458                     if not m.is_reply and not m.is_event:
 459                         try:
 460                             m.reply = self.get_reply(n)
 461                             if "stream" in self.services[m.name]:
 462                                 m.reply_is_stream = \
 463                                     self.services[m.name]["stream"]
 464                             else:
 465                                 m.reply_is_stream = False
 466                             m.reply.request = m
 467                         except:
 468                             raise ParseError(
 469                                 "Cannot find reply to message `%s'" % n)
 470                 except ParseError as e:
 471                     self.exceptions.append(e)
 472                     remove.append(n)
 473
 474             self.messages_by_json[jn] = {
 475                 k: v for k, v in j.items() if k not in remove}