API: Add support for type aliases
[vpp.git] / extras / japi / java / jvpp / gen / jvppgen / jvpp_model.py
1 #!/usr/bin/env python2
2 #
3 # Copyright (c) 2018 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 #
16 import json
17 import pprint
18 from collections import OrderedDict
19
20 BASE_PACKAGE = "io.fd.vpp.jvpp"
21
22
23 class ParseException(Exception):
24     pass
25
26
27 class Type(object):
28     def __init__(self, name, java_name, java_name_fqn, jni_signature, jni_type, jni_accessor,
29                  host_to_net_function, net_to_host_function):
30         """
31         Initializes Type class.
32
33         :param name: name of type as defined in .api file, e.g. u8, u32[] or mac_entry
34         :param java_name: corresponding java name, e.g. byte, int[] or MacEntry
35         :param java_name_fqn: fully qualified java name, e.g. io.fd.vpp.jvpp.core.types.MacEntry
36         :param jni_signature: JNI Type signature, e.g. B, [I or Lio.fd.vpp.jvpp.core.types.MacEntry;
37                               See https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#type_signatures
38         :param jni_type: JNI reference type, e.g. jbyte jintArray, jobject
39                          See https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#reference_types
40         :param jni_accessor: Java type do by used in Get<type>Field, Set<type>Field and other functions.
41                              See https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#accessing_fields_of_objects
42         :param host_to_net_function: name of function host to net byte order swap function
43         :param net_to_host_function: name of function net to host byte order swap function
44         """
45         self.name = name
46         self.java_name = java_name
47
48         # Java generation specific properties, TODO(VPP-1186): move to Java specific subclass
49         self.java_name_fqn = java_name_fqn
50
51         # JNI generation specific properties, TODO(VPP-1186): move to JNI specific subclass
52         self.jni_signature = jni_signature
53
54         # Native type, see:
55         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#primitive_types
56         # and
57         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#reference_types
58         self.jni_type = jni_type
59
60         # Java type do by used in Get<type>Field, Set<type>Field and other functions, see:
61         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#accessing_fields_of_objects
62         self.jni_accessor = jni_accessor
63
64         self.host_to_net_function = host_to_net_function
65         self.net_to_host_function = net_to_host_function
66         self.is_swap_needed = host_to_net_function and net_to_host_function
67
68
69 class SimpleType(Type):
70     def __init__(self, name, java_name, jni_signature, jni_type, jni_accessor,
71                  host_to_net_function=None, net_to_host_function=None):
72         super(SimpleType, self).__init__(
73             name=name,
74             java_name=java_name,
75             java_name_fqn=java_name,
76             jni_signature=jni_signature,
77             jni_type=jni_type,
78             jni_accessor=jni_accessor,
79             host_to_net_function=host_to_net_function,
80             net_to_host_function=net_to_host_function
81         )
82         self.vpp_name = name
83
84     def get_host_to_net_function(self, host_ref_name, net_ref_name):
85         return "%s = %s(%s)" % (net_ref_name, self.host_to_net_function, host_ref_name)
86
87     def __str__(self):
88         return "SimpleType{name:%s, java_name:%s}" % (self.name, self.java_name)
89
90
91 # TODO(VPP-1187): add array host to net functions to reduce number of members and simplify JNI generation
92 class Array(Type):
93     def __init__(self, base_type):
94         super(Array, self).__init__(
95             name=base_type.name + _ARRAY_SUFFIX,
96             java_name=base_type.java_name + _ARRAY_SUFFIX,
97             java_name_fqn=base_type.java_name_fqn + _ARRAY_SUFFIX,
98             jni_signature="[%s" % base_type.jni_signature,
99             jni_type="%sArray" % base_type.jni_type,
100             jni_accessor="Object",
101             host_to_net_function=base_type.host_to_net_function,
102             net_to_host_function=base_type.net_to_host_function
103         )
104         self.base_type = base_type
105
106     def get_host_to_net_function(self, host_ref_name, net_ref_name):
107         return self.base_type.get_host_to_net_function(host_ref_name, net_ref_name)
108
109     def __str__(self):
110         return "Array{name:%s, java_name:%s}" % (self.name, self.java_name)
111
112
113 class Enum(Type):
114     def __init__(self, name, value, constants, definition, plugin_name):
115         _java_name = _underscore_to_camelcase_upper(name)
116
117         super(Enum, self).__init__(
118             name=name,
119             java_name=_java_name,
120             java_name_fqn="io.fd.vpp.jvpp.%s.types.%s" % (plugin_name, _java_name),
121             jni_signature="Lio/fd/vpp/jvpp/%s/types/%s;" % (plugin_name, _java_name),
122             jni_type="jobject",
123             jni_accessor="Object",
124             host_to_net_function="_host_to_net_%s" % name,
125             net_to_host_function="_net_to_host_%s" % name
126         )
127
128         self.value = value
129         self.constants = constants
130         self.doc = _message_to_javadoc(definition)
131         self.java_name_lower = _underscore_to_camelcase_lower(name)
132         self.vpp_name = "%s%s%s" % (_VPP_TYPE_PREFIX, name, _VPP_TYPE_SUFFIX)
133         # Fully qualified class name used by FindClass function, see:
134         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#FindClass
135         self.jni_name = "io/fd/vpp/jvpp/%s/types/%s" % (plugin_name, _java_name)
136
137     def get_host_to_net_function(self, host_ref_name, net_ref_name):
138         return "_host_to_net_%s(env, %s, &(%s))" % (self.name, host_ref_name, net_ref_name)
139
140
141 class Class(Type):
142     def __init__(self, name, crc, fields, definition, plugin_name):
143         _java_name = _underscore_to_camelcase_upper(name)
144
145         super(Class, self).__init__(
146             name=name,
147             java_name=_java_name,
148             java_name_fqn="io.fd.vpp.jvpp.%s.types.%s" % (plugin_name, _java_name),
149             jni_signature="Lio/fd/vpp/jvpp/%s/types/%s;" % (plugin_name, _java_name),
150             jni_type="jobject",
151             jni_accessor="Object",
152             host_to_net_function="_host_to_net_%s" % name,
153             net_to_host_function="_net_to_host_%s" % name
154         )
155
156         self.crc = crc
157         self.fields = fields
158         self.doc = _message_to_javadoc(definition)
159         self.java_name_lower = _underscore_to_camelcase_lower(name)
160         self.vpp_name = "%s%s%s" % (_VPP_TYPE_PREFIX, name, _VPP_TYPE_SUFFIX)
161         # Fully qualified class name used by FindClass function, see:
162         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#FindClass
163         self.jni_name = "io/fd/vpp/jvpp/%s/types/%s" % (plugin_name, _java_name)
164
165     def get_host_to_net_function(self, host_ref_name, net_ref_name):
166         return "_host_to_net_%s(env, %s, &(%s))" % (self.name, host_ref_name, net_ref_name)
167
168
169 class Union(Type):
170     def __init__(self, name, crc, fields, definition, plugin_name):
171         _java_name = _underscore_to_camelcase_upper(name)
172
173         super(Union, self).__init__(
174             name=name,
175             java_name=_java_name,
176             java_name_fqn="io.fd.vpp.jvpp.%s.types.%s" % (plugin_name, _java_name),
177             jni_signature="Lio/fd/vpp/jvpp/%s/types/%s;" % (plugin_name, _java_name),
178             jni_type="jobject",
179             jni_accessor="Object",
180             host_to_net_function="_host_to_net_%s" % name,
181             net_to_host_function="_net_to_host_%s" % name
182         )
183
184         self.crc = crc
185         self.fields = fields
186         self.doc = _message_to_javadoc(definition)
187         self.java_name_lower = _underscore_to_camelcase_lower(name)
188         self.vpp_name = "%s%s%s" % (_VPP_TYPE_PREFIX, name, _VPP_TYPE_SUFFIX)
189         # Fully qualified class name used by FindClass function, see:
190         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#FindClass
191         self.jni_name = "io/fd/vpp/jvpp/%s/types/%s" % (plugin_name, _java_name)
192
193     def get_host_to_net_function(self, host_ref_name, net_ref_name):
194         return "_host_to_net_%s(env, %s, &(%s))" % (self.name, host_ref_name, net_ref_name)
195
196
197 class Field(object):
198     def __init__(self, name, field_type, array_len=None, array_len_field=None):
199         self.name = name
200         self.java_name = _underscore_to_camelcase_lower(name)
201         self.java_name_upper = _underscore_to_camelcase_upper(name)
202         self.type = field_type
203         self.array_len = array_len
204         self.array_len_field = array_len_field
205
206     def __str__(self):
207         return "Field{name:%s, java_name:%s, type:%s}" % (self.name, self.java_name, self.type)
208
209
210 class Message(object):
211     def __init__(self, name, crc, fields, definition):
212         self.name = name
213         self.java_name_upper = _underscore_to_camelcase_upper(name)
214         self.java_name_lower = _underscore_to_camelcase_lower(name)
215         self.crc = crc[2:]
216         self.fields = fields
217         self.has_fields = fields != []
218         self.doc = _message_to_javadoc(definition)
219
220     def __str__(self):
221         return "Message{name:%s, java_name:%s, crc:%s, fields:%s}" % (
222             self.name, self.java_name_upper, self.crc, self.fields)
223
224
225 class Event(Message):
226     def __init__(self, name, crc, fields, definition):
227         super(Event, self).__init__(name, crc, fields, definition)
228
229
230 class Request(Message):
231     def __init__(self, name, reply, crc, fields, definition):
232         super(Request, self).__init__(name, crc, fields, definition)
233         self.reply = reply
234         self.reply_java = _underscore_to_camelcase_upper(reply)
235
236     def __str__(self):
237         return "Request{name:%s, reply:%s, crc:%s, fields:%s}" % (self.name, self.reply, self.crc, self.fields)
238
239
240 class Reply(Message):
241     def __init__(self, name, request, crc, fields, definition):
242         super(Reply, self).__init__(name, crc, fields, definition)
243         self.request = request
244         self.request_java = _underscore_to_camelcase_upper(request)
245
246     def __str__(self):
247         return "Reply{name:%s, request:%s, crc:%s, fields:%s}" % (self.name, self.request, self.crc, self.fields)
248
249
250 class Dump(Message):
251     def __init__(self, name, details, crc, fields, definition):
252         super(Dump, self).__init__(name, crc, fields, definition)
253         self.details = details
254         self.reply_java = _underscore_to_camelcase_upper(details)
255
256     def __str__(self):
257         return "Dump{name:%s, details:%s, crc:%s, fields:%s}" % (self.name, self.details, self.crc, self.fields)
258
259
260 class Details(Message):
261     def __init__(self, name, dump, crc, fields, definition):
262         super(Details, self).__init__(name, crc, fields, definition)
263         self.dump = dump
264         self.request_java = _underscore_to_camelcase_upper(dump)
265
266     def __str__(self):
267         return "Details{name:%s, dump:%s, crc:%s, fields:%s}" % (self.name, self.dump, self.crc, self.fields)
268
269
270 def is_retval(field):
271     return field.name == u'retval'
272
273
274 def is_array(field):
275     return field.array_len is not None
276
277
278 def is_request(msg):
279     return hasattr(msg, 'reply')
280
281
282 def is_reply(msg):
283     return hasattr(msg, 'request')
284
285
286 def is_dump(msg):
287     return hasattr(msg, 'details')
288
289
290 def is_details(msg):
291     return hasattr(msg, 'dump')
292
293
294 def is_event(msg):
295     return isinstance(msg, Event)
296
297
298 def is_control_ping(msg):
299     return msg.name == u'control_ping'
300
301
302 def is_control_ping_reply(msg):
303     return msg.name == u'control_ping_reply'
304
305
306 class JVppModel(object):
307     def __init__(self, logger, json_api_files, plugin_name):
308         self.logger = logger
309         # TODO(VPP-1188): provide json_file_by_definition map to improve javadoc
310         self.json_api_files = json_api_files
311         self.plugin_package = BASE_PACKAGE + "." + plugin_name
312         self.plugin_name = plugin_name
313         self.plugin_java_name = _underscore_to_camelcase_upper(plugin_name)
314         self._load_json_files(json_api_files)
315         self._parse_services()
316         self._parse_messages()
317         self._validate_messages()
318
319     def _load_json_files(self, json_api_files):
320         types = {}
321         self._messages = []
322         self._services = {}
323         self._aliases = {}
324         for file_name in json_api_files:
325             with open(file_name) as f:
326                 j = json.load(f)
327                 types.update({d[0]: {'type': 'enum', 'data': d} for d in j['enums']})
328                 types.update({d[0]: {'type': 'type', 'data': d} for d in j['types']})
329                 types.update({d[0]: {'type': 'union', 'data': d} for d in j['unions']})
330                 self._messages.extend(j['messages'])
331                 self._services.update(j['services'])
332                 self._aliases.update(j['aliases'])
333
334         self._parse_types(types)
335
336     def _parse_types(self, types):
337         self._parse_simple_types()
338         i = 0
339         while True:
340             unresolved = {}
341             for name, value in types.items():
342                 if name in self._types_by_name:
343                     continue
344
345                 type = value['type']
346                 data = value['data'][1:]
347                 try:
348                     if type == 'enum':
349                         type = self._parse_enum(name, data)
350                     elif type == 'union':
351                         type = self._parse_union(name, data)
352                     elif type == 'type':
353                         type = self._parse_type(name, data)
354                     else:
355                         self.logger.warning("Unsupported type %s. Ignoring...", type)
356                         continue
357
358                     self._types_by_name[name] = type
359                     self._types_by_name[name + _ARRAY_SUFFIX] = Array(type)
360                 except ParseException as e:
361                     self.logger.debug("Failed to parse %s type in iteration %s: %s.", name, i, e)
362                     unresolved[name] = value
363             if len(unresolved) == 0:
364                 break
365             if i > 3:
366                 raise ParseException('Unresolved type definitions {}'
367                                      .format(unresolved))
368             types = unresolved
369             i += 1
370
371         self.types = self._types_by_name.values()
372
373     def _parse_simple_types(self):
374         # Mapping according to:
375         # http://docs.oracle.com/javase/7/do+'[]'cs/technotes/guides/jni/spec/types.html
376         # and
377         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#Get_type_Field_routines
378         #
379         # Unsigned types are converted to signed java types that have the same size.
380         # It is the API user responsibility to interpret them correctly.
381
382         self._types_by_name = OrderedDict({
383             'u8': SimpleType('u8', 'byte', 'B', 'jbyte', 'Byte'),
384             'i8': SimpleType('i8', 'byte', 'B', 'jbyte', 'Byte'),
385             'u16': SimpleType('u16', 'short', 'S', 'jshort', 'Short',
386                               host_to_net_function='clib_host_to_net_u16',
387                               net_to_host_function='clib_net_to_host_u16'),
388             'i16': SimpleType('i16', 'short', 'S', 'jshort', 'Short',
389                               host_to_net_function='clib_host_to_net_i16',
390                               net_to_host_function='clib_net_to_host_i16'),
391             'u32': SimpleType('u32', 'int', 'I', 'jint', 'Int',
392                               host_to_net_function='clib_host_to_net_u32',
393                               net_to_host_function='clib_net_to_host_u32'),
394             'i32': SimpleType('i32', 'int', 'I', 'jint', 'Int',
395                               host_to_net_function='clib_host_to_net_i32',
396                               net_to_host_function='clib_net_to_host_i32'),
397             'u64': SimpleType('u64', 'long', 'J', 'jlong', 'Long',
398                               host_to_net_function='clib_host_to_net_u64',
399                               net_to_host_function='clib_net_to_host_u64'),
400             'i64': SimpleType('i64', 'long', 'J', 'jlong', 'Long',
401                               host_to_net_function='clib_host_to_net_i64',
402                               net_to_host_function='clib_net_to_host_i64'),
403             'f64': SimpleType('f64', 'double', 'D', 'jdouble', 'Double')
404         })
405
406         for n, t in self._types_by_name.items():
407             self._types_by_name[n + _ARRAY_SUFFIX] = Array(t)
408
409     def _parse_enum(self, name, definition):
410         self.logger.debug("Parsing enum %s: %s", name, definition)
411         constants = []
412         type_name = None
413         for item in definition:
414             if type(item) is dict and 'enumtype' in item:
415                 type_name = item['enumtype']
416                 continue
417             constants.append({'name': item[0], 'value': item[1]})
418         if not type_name:
419             raise ParseException("'enumtype' was not defined for %s" % definition)
420         return Enum(name, Field('value', self._types_by_name[type_name]), constants, definition, self.plugin_name)
421
422     def _parse_union(self, name, definition):
423         self.logger.debug("Parsing union %s: %s", name, definition)
424         crc, fields = self._parse_fields(definition)
425         return Union(name, crc, fields, definition, self.plugin_name)
426
427     def _parse_type(self, name, definition):
428         self.logger.debug("Parsing type %s: %s", name, definition)
429         crc, fields = self._parse_fields(definition)
430         return Class(name, crc, fields, definition, self.plugin_name)
431
432     def _parse_services(self):
433         self._dumps_by_details = {}
434         self._requests_by_reply = {}
435         for name, service in self._services.iteritems():
436             if _is_stream(service):
437                 self._dumps_by_details[service['reply']] = name
438             else:
439                 self._requests_by_reply[service['reply']] = name
440
441     def _parse_messages(self):
442         # Preserve ordering from JSON file to make debugging easier.
443         self._messages_by_name = OrderedDict()
444         for msg in self._messages:
445             try:
446                 name = msg[0]
447                 definition = msg[1:]
448                 self._messages_by_name[name] = self._parse_message(name, definition)
449             except ParseException as e:
450                 self.logger.warning("Failed to parse message %s: %s. Skipping message.", name, e)
451
452     def _parse_message(self, name, definition):
453         self.logger.debug("Parsing message %s: %s", name, definition)
454         crc, fields = self._parse_fields(definition)
455         if name in self._services:
456             service = self._services[name]
457             reply = service['reply']
458             if _is_stream(service):
459                 return Dump(name, reply, crc, filter(_is_request_field, fields), definition)
460             if reply:
461                 return Request(name, reply, crc, filter(_is_request_field, fields), definition)
462             else:
463                 return Event(name, crc, filter(_is_request_field, fields), definition)
464         elif name in self._requests_by_reply:
465             return Reply(name, self._requests_by_reply[name], crc, filter(_is_reply_field, fields), definition)
466         elif name in self._dumps_by_details:
467             return Details(name, self._dumps_by_details[name], crc, filter(_is_reply_field, fields), definition)
468         else:
469             # TODO: some messages like combined_counters are not visible in the services.
470             # Throw exception instead (requires fixing vppagigen).
471             return Event(name, crc, filter(_is_request_field, fields), definition)
472
473     def _parse_fields(self, definition):
474         crc = None
475         fields = []
476         for item in definition:
477             if type(item) == dict and 'crc' in item:
478                 crc = item['crc']
479             else:
480                 fields.append(self._parse_field(item, fields))
481         if not crc:
482             raise ParseException("CRC was not defined for %s" % definition)
483         return crc, fields
484
485     def _parse_field(self, field, fields):
486         type_name = _extract_type_name(field[0])
487         if type_name in self._aliases and type_name not in self._types_by_name:
488             aliased_type = self._types_by_name.get(self._aliases.get(type_name).get("type"))
489             self._types_by_name[type_name] = aliased_type
490
491         if type_name in self._types_by_name:
492             if len(field) > 2:
493                 # Array field
494                 array_len_field = None
495                 if len(field) == 4:
496                     for f in fields:
497                         if f.name == field[3]:
498                             array_len_field = f
499                     if not array_len_field:
500                         raise ParseException("Could not find field %s declared as length of array %s",
501                                              field[3], field[1])
502                 return Field(field[1], self._types_by_name[type_name + _ARRAY_SUFFIX], field[2], array_len_field)
503             else:
504                 return Field(field[1], self._types_by_name[type_name])
505         else:
506             raise ParseException("Unknown field type %s" % field)
507
508     def _validate_messages(self):
509         """
510         In case if message A is known to be reply for message B, and message B was not correctly parsed,
511         remove message A from the set of all messages.
512         """
513         to_be_removed = []
514         messages = self._messages_by_name
515         for name, msg in messages.iteritems():
516             if (is_request(msg) and msg.reply not in messages) \
517                     or (is_reply(msg) and msg.request not in messages) \
518                     or (is_dump(msg) and msg.details not in messages) \
519                     or (is_details(msg) and msg.dump not in messages):
520                 to_be_removed.append(name)
521
522         for name in to_be_removed:
523             del messages[name]
524
525         self.messages = self._messages_by_name.values()
526
527 _ARRAY_SUFFIX = '[]'
528
529
530 def _underscore_to_camelcase_upper(name):
531     return name.title().replace("_", "")
532
533
534 def _underscore_to_camelcase_lower(name):
535     name = name.title().replace("_", "")
536     return name[0].lower() + name[1:]
537
538
539 def _message_to_javadoc(message_definition):
540     """ Converts JSON message definition to javadoc """
541     formatted_message = pprint.pformat(message_definition, indent=4, width=120, depth=None)
542     return " * " + formatted_message.replace("\n", "\n * ")
543
544
545 def _is_stream(service):
546     """
547     Checks if service represents stream, e.g.:
548     "ip_address_dump": {
549         "reply": "ip_address_details",
550         "stream": true
551     }
552     :param service: JSON definition of service
553     :return: value assigned to "stream" or None
554     """
555     return "stream" in service
556
557
558 def _extract_type_name(name):
559     if name.startswith(_VPP_TYPE_PREFIX) and name.endswith(_VPP_TYPE_SUFFIX):
560         return name[len(_VPP_TYPE_PREFIX): - len(_VPP_TYPE_SUFFIX)]
561     return name
562
563 _VPP_TYPE_PREFIX = "vl_api_"
564
565 _VPP_TYPE_SUFFIX = "_t"
566
567
568 def _is_request_field(field):
569     # Skip fields that are hidden to the jvpp user (handled by JNI layer)
570     return field.name not in {'_vl_msg_id', 'client_index', 'context'}
571
572
573 def _is_reply_field(field):
574     # Skip fields that are hidden to the jvpp user:
575     # _vl_msg_id is handled at JNI layer,
576     # Unlike in the request case, context is visible to allow matching replies with requests at Java layer.
577     return field.name not in {'_vl_msg_id'}