16099689880886ce582db19a453e04ccdcba4a4a
[vpp.git] / extras / japi / java / jvpp / gen / jvppgen / jvpp_model.py
1 #!/usr/bin/env python2
2 #
3 # Copyright (c) 2018 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 #
16 import json
17 import pprint
18 from collections import OrderedDict
19
20 import binascii
21
22 BASE_PACKAGE = "io.fd.vpp.jvpp"
23
24
25 class ParseException(Exception):
26     pass
27
28
29 class Type(object):
30     def __init__(self, name, java_name, java_name_fqn, jni_signature, jni_type, jni_accessor,
31                  host_to_net_function, net_to_host_function):
32         """
33         Initializes Type class.
34
35         :param name: name of type as defined in .api file, e.g. u8, u32[] or mac_entry
36         :param java_name: corresponding java name, e.g. byte, int[] or MacEntry
37         :param java_name_fqn: fully qualified java name, e.g. io.fd.vpp.jvpp.core.types.MacEntry
38         :param jni_signature: JNI Type signature, e.g. B, [I or Lio.fd.vpp.jvpp.core.types.MacEntry;
39                               See https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#type_signatures
40         :param jni_type: JNI reference type, e.g. jbyte jintArray, jobject
41                          See https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#reference_types
42         :param jni_accessor: Java type do by used in Get<type>Field, Set<type>Field and other functions.
43                              See https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#accessing_fields_of_objects
44         :param host_to_net_function: name of function host to net byte order swap function
45         :param net_to_host_function: name of function net to host byte order swap function
46         """
47         self.name = name
48         self.java_name = java_name
49
50         # Java generation specific properties, TODO(VPP-1186): move to Java specific subclass
51         self.java_name_fqn = java_name_fqn
52
53         # JNI generation specific properties, TODO(VPP-1186): move to JNI specific subclass
54         self.jni_signature = jni_signature
55
56         # Native type, see:
57         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#primitive_types
58         # and
59         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#reference_types
60         self.jni_type = jni_type
61
62         # Java type do by used in Get<type>Field, Set<type>Field and other functions, see:
63         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#accessing_fields_of_objects
64         self.jni_accessor = jni_accessor
65
66         self.host_to_net_function = host_to_net_function
67         self.net_to_host_function = net_to_host_function
68         self.is_swap_needed = host_to_net_function and net_to_host_function
69
70
71 class SimpleType(Type):
72     def __init__(self, name, java_name, jni_signature, jni_type, jni_accessor,
73                  host_to_net_function=None, net_to_host_function=None):
74         super(SimpleType, self).__init__(
75             name=name,
76             java_name=java_name,
77             java_name_fqn=java_name,
78             jni_signature=jni_signature,
79             jni_type=jni_type,
80             jni_accessor=jni_accessor,
81             host_to_net_function=host_to_net_function,
82             net_to_host_function=net_to_host_function
83         )
84         self.vpp_name = name
85
86     def get_host_to_net_function(self, host_ref_name, net_ref_name):
87         return "%s = %s(%s)" % (net_ref_name, self.host_to_net_function, host_ref_name)
88
89     def __str__(self):
90         return "SimpleType{name:%s, java_name:%s}" % (self.name, self.java_name)
91
92
93 # TODO(VPP-1187): add array host to net functions to reduce number of members and simplify JNI generation
94 class Array(Type):
95     def __init__(self, base_type):
96         super(Array, self).__init__(
97             name=base_type.name + _ARRAY_SUFFIX,
98             java_name=base_type.java_name + _ARRAY_SUFFIX,
99             java_name_fqn=base_type.java_name_fqn + _ARRAY_SUFFIX,
100             jni_signature="[%s" % base_type.jni_signature,
101             jni_type="%sArray" % base_type.jni_type,
102             jni_accessor="Object",
103             host_to_net_function=base_type.host_to_net_function,
104             net_to_host_function=base_type.net_to_host_function
105         )
106         self.base_type = base_type
107
108     def get_host_to_net_function(self, host_ref_name, net_ref_name):
109         return self.base_type.get_host_to_net_function(host_ref_name, net_ref_name)
110
111     def __str__(self):
112         return "Array{name:%s, java_name:%s}" % (self.name, self.java_name)
113
114
115 class Enum(Type):
116     def __init__(self, name, value, constants, definition, plugin_name):
117         _java_name = _underscore_to_camelcase_upper(name)
118
119         super(Enum, self).__init__(
120             name=name,
121             java_name=_java_name,
122             java_name_fqn="io.fd.vpp.jvpp.%s.types.%s" % (plugin_name, _java_name),
123             jni_signature="Lio/fd/vpp/jvpp/%s/types/%s;" % (plugin_name, _java_name),
124             jni_type="jobject",
125             jni_accessor="Object",
126             host_to_net_function="_host_to_net_%s" % name,
127             net_to_host_function="_net_to_host_%s" % name
128         )
129
130         self.value = value
131         self.constants = constants
132         self.doc = _message_to_javadoc(definition)
133         self.java_name_lower = _underscore_to_camelcase_lower(name)
134         self.vpp_name = "%s%s%s" % (_VPP_TYPE_PREFIX, name, _VPP_TYPE_SUFFIX)
135         # Fully qualified class name used by FindClass function, see:
136         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#FindClass
137         self.jni_name = "io/fd/vpp/jvpp/%s/types/%s" % (plugin_name, _java_name)
138
139     def get_host_to_net_function(self, host_ref_name, net_ref_name):
140         return "_host_to_net_%s(env, %s, &(%s))" % (self.name, host_ref_name, net_ref_name)
141
142
143 class Class(Type):
144     def __init__(self, name, crc, fields, definition, plugin_name):
145         _java_name = _underscore_to_camelcase_upper(name)
146
147         super(Class, self).__init__(
148             name=name,
149             java_name=_java_name,
150             java_name_fqn="io.fd.vpp.jvpp.%s.types.%s" % (plugin_name, _java_name),
151             jni_signature="Lio/fd/vpp/jvpp/%s/types/%s;" % (plugin_name, _java_name),
152             jni_type="jobject",
153             jni_accessor="Object",
154             host_to_net_function="_host_to_net_%s" % name,
155             net_to_host_function="_net_to_host_%s" % name
156         )
157
158         self.crc = crc
159         self.fields = fields
160         self.doc = _message_to_javadoc(definition)
161         self.java_name_lower = _underscore_to_camelcase_lower(name)
162         self.vpp_name = "%s%s%s" % (_VPP_TYPE_PREFIX, name, _VPP_TYPE_SUFFIX)
163         # Fully qualified class name used by FindClass function, see:
164         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#FindClass
165         self.jni_name = "io/fd/vpp/jvpp/%s/types/%s" % (plugin_name, _java_name)
166
167     def get_host_to_net_function(self, host_ref_name, net_ref_name):
168         return "_host_to_net_%s(env, %s, &(%s))" % (self.name, host_ref_name, net_ref_name)
169
170
171 class Union(Type):
172     def __init__(self, name, crc, fields, definition, plugin_name):
173         _java_name = _underscore_to_camelcase_upper(name)
174
175         super(Union, self).__init__(
176             name=name,
177             java_name=_java_name,
178             java_name_fqn="io.fd.vpp.jvpp.%s.types.%s" % (plugin_name, _java_name),
179             jni_signature="Lio/fd/vpp/jvpp/%s/types/%s;" % (plugin_name, _java_name),
180             jni_type="jobject",
181             jni_accessor="Object",
182             host_to_net_function="_host_to_net_%s" % name,
183             net_to_host_function="_net_to_host_%s" % name
184         )
185
186         self.crc = crc
187         self.fields = fields
188         self.doc = _message_to_javadoc(definition)
189         self.java_name_lower = _underscore_to_camelcase_lower(name)
190         self.vpp_name = "%s%s%s" % (_VPP_TYPE_PREFIX, name, _VPP_TYPE_SUFFIX)
191         # Fully qualified class name used by FindClass function, see:
192         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#FindClass
193         self.jni_name = "io/fd/vpp/jvpp/%s/types/%s" % (plugin_name, _java_name)
194
195     def get_host_to_net_function(self, host_ref_name, net_ref_name):
196         return "_host_to_net_%s(env, %s, &(%s))" % (self.name, host_ref_name, net_ref_name)
197
198
199 class Field(object):
200     def __init__(self, name, field_type, array_len=None, array_len_field=None):
201         self.name = name
202         self.java_name = _underscore_to_camelcase_lower(name)
203         self.java_name_upper = _underscore_to_camelcase_upper(name)
204         self.type = field_type
205         self.array_len = array_len
206         self.array_len_field = array_len_field
207
208     def __str__(self):
209         return "Field{name:%s, java_name:%s, type:%s}" % (self.name, self.java_name, self.type)
210
211
212 class Message(object):
213     def __init__(self, name, crc, fields, definition):
214         self.name = name
215         self.java_name_upper = _underscore_to_camelcase_upper(name)
216         self.java_name_lower = _underscore_to_camelcase_lower(name)
217         self.crc = crc[2:]
218         self.fields = fields
219         self.has_fields = fields != []
220         self.doc = _message_to_javadoc(definition)
221
222     def __str__(self):
223         return "Message{name:%s, java_name:%s, crc:%s, fields:%s}" % (
224             self.name, self.java_name_upper, self.crc, self.fields)
225
226
227 class Event(Message):
228     def __init__(self, name, crc, fields, definition):
229         super(Event, self).__init__(name, crc, fields, definition)
230
231
232 class Request(Message):
233     def __init__(self, name, reply, crc, fields, definition):
234         super(Request, self).__init__(name, crc, fields, definition)
235         self.reply = reply
236         self.reply_java = _underscore_to_camelcase_upper(reply)
237
238     def __str__(self):
239         return "Request{name:%s, reply:%s, crc:%s, fields:%s}" % (self.name, self.reply, self.crc, self.fields)
240
241
242 class Reply(Message):
243     def __init__(self, name, request, crc, fields, definition):
244         super(Reply, self).__init__(name, crc, fields, definition)
245         self.request = request
246         self.request_java = _underscore_to_camelcase_upper(request)
247
248     def __str__(self):
249         return "Reply{name:%s, request:%s, crc:%s, fields:%s}" % (self.name, self.request, self.crc, self.fields)
250
251
252 class Dump(Message):
253     def __init__(self, name, details, crc, fields, definition):
254         super(Dump, self).__init__(name, crc, fields, definition)
255         self.details = details
256         self.reply_java = _underscore_to_camelcase_upper(details)
257
258     def __str__(self):
259         return "Dump{name:%s, details:%s, crc:%s, fields:%s}" % (self.name, self.details, self.crc, self.fields)
260
261
262 class Details(Message):
263     def __init__(self, name, dump, crc, fields, definition):
264         super(Details, self).__init__(name, crc, fields, definition)
265         self.dump = dump
266         self.request_java = _underscore_to_camelcase_upper(dump)
267
268     def __str__(self):
269         return "Details{name:%s, dump:%s, crc:%s, fields:%s}" % (self.name, self.dump, self.crc, self.fields)
270
271
272 def is_retval(field):
273     return field.name == u'retval'
274
275
276 def is_array(field):
277     return field.array_len is not None
278
279
280 def is_request(msg):
281     return hasattr(msg, 'reply')
282
283
284 def is_reply(msg):
285     return hasattr(msg, 'request')
286
287
288 def is_dump(msg):
289     return hasattr(msg, 'details')
290
291
292 def is_details(msg):
293     return hasattr(msg, 'dump')
294
295
296 def is_event(msg):
297     return isinstance(msg, Event)
298
299
300 def is_control_ping(msg):
301     return msg.name == u'control_ping'
302
303
304 def is_control_ping_reply(msg):
305     return msg.name == u'control_ping_reply'
306
307
308 def crc(block):
309     s = str(block).encode()
310     return binascii.crc32(s) & 0xffffffff
311
312
313 class JVppModel(object):
314     def __init__(self, logger, json_api_files, plugin_name):
315         self.logger = logger
316         # TODO(VPP-1188): provide json_file_by_definition map to improve javadoc
317         self.json_api_files = json_api_files
318         self.plugin_package = BASE_PACKAGE + "." + plugin_name
319         self.plugin_name = plugin_name
320         self.plugin_java_name = _underscore_to_camelcase_upper(plugin_name)
321         self._load_json_files(json_api_files)
322         self._parse_services()
323         self._parse_messages()
324         self._validate_messages()
325
326     def _load_json_files(self, json_api_files):
327         types = {}
328         self._messages = []
329         self._services = {}
330         self._aliases = {}
331         for file_name in json_api_files:
332             with open(file_name) as f:
333                 j = json.load(f)
334                 types.update({d[0]: {'type': 'enum', 'data': d} for d in j['enums']})
335                 types.update({d[0]: {'type': 'type', 'data': d} for d in j['types']})
336                 types.update({d[0]: {'type': 'union', 'data': d} for d in j['unions']})
337                 self._messages.extend(j['messages'])
338                 self._services.update(j['services'])
339                 self._aliases.update(j['aliases'])
340
341         self._parse_types(types)
342
343     def _parse_aliases(self, types):
344         for alias_name in self._aliases:
345             alias = self._aliases[alias_name]
346             alias_type = {"type": "type"}
347             java_name_lower = _underscore_to_camelcase_lower(alias_name)
348             vpp_type = alias["type"]
349             crc_value = '0x%08x' % crc(alias_name)
350             if "length" in alias:
351                 length = alias["length"]
352                 alias_type["data"] = [
353                     alias_name,
354                     [
355                         vpp_type,
356                         java_name_lower,
357                         length
358                     ],
359                     {
360                         "crc": crc_value
361                     }
362                 ]
363             else:
364                 alias_type["data"] = [
365                     alias_name,
366                     [
367                         vpp_type,
368                         java_name_lower
369                     ],
370                     {
371                         "crc": crc_value
372                     }
373                 ]
374
375             types[alias_name] = alias_type
376
377     def _parse_types(self, types):
378         self._parse_simple_types()
379         self._parse_aliases(types)
380         i = 0
381         while True:
382             unresolved = {}
383             for name, value in types.items():
384                 if name in self._types_by_name:
385                     continue
386
387                 type = value['type']
388                 data = value['data'][1:]
389                 try:
390                     if type == 'enum':
391                         type = self._parse_enum(name, data)
392                     elif type == 'union':
393                         type = self._parse_union(name, data)
394                     elif type == 'type':
395                         type = self._parse_type(name, data)
396                     else:
397                         self.logger.warning("Unsupported type %s. Ignoring...", type)
398                         continue
399
400                     self._types_by_name[name] = type
401                     self._types_by_name[name + _ARRAY_SUFFIX] = Array(type)
402                 except ParseException as e:
403                     self.logger.debug("Failed to parse %s type in iteration %s: %s.", name, i, e)
404                     unresolved[name] = value
405             if len(unresolved) == 0:
406                 break
407             if i > 3:
408                 raise ParseException('Unresolved type definitions {}'
409                                      .format(unresolved))
410             types = unresolved
411             i += 1
412
413         self.types = self._types_by_name.values()
414
415     def _parse_simple_types(self):
416         # Mapping according to:
417         # http://docs.oracle.com/javase/7/do+'[]'cs/technotes/guides/jni/spec/types.html
418         # and
419         # https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#Get_type_Field_routines
420         #
421         # Unsigned types are converted to signed java types that have the same size.
422         # It is the API user responsibility to interpret them correctly.
423
424         self._types_by_name = OrderedDict({
425             'u8': SimpleType('u8', 'byte', 'B', 'jbyte', 'Byte'),
426             'i8': SimpleType('i8', 'byte', 'B', 'jbyte', 'Byte'),
427             'u16': SimpleType('u16', 'short', 'S', 'jshort', 'Short',
428                               host_to_net_function='clib_host_to_net_u16',
429                               net_to_host_function='clib_net_to_host_u16'),
430             'i16': SimpleType('i16', 'short', 'S', 'jshort', 'Short',
431                               host_to_net_function='clib_host_to_net_i16',
432                               net_to_host_function='clib_net_to_host_i16'),
433             'u32': SimpleType('u32', 'int', 'I', 'jint', 'Int',
434                               host_to_net_function='clib_host_to_net_u32',
435                               net_to_host_function='clib_net_to_host_u32'),
436             'i32': SimpleType('i32', 'int', 'I', 'jint', 'Int',
437                               host_to_net_function='clib_host_to_net_i32',
438                               net_to_host_function='clib_net_to_host_i32'),
439             'u64': SimpleType('u64', 'long', 'J', 'jlong', 'Long',
440                               host_to_net_function='clib_host_to_net_u64',
441                               net_to_host_function='clib_net_to_host_u64'),
442             'i64': SimpleType('i64', 'long', 'J', 'jlong', 'Long',
443                               host_to_net_function='clib_host_to_net_i64',
444                               net_to_host_function='clib_net_to_host_i64'),
445             'f64': SimpleType('f64', 'double', 'D', 'jdouble', 'Double')
446         })
447
448         for n, t in self._types_by_name.items():
449             self._types_by_name[n + _ARRAY_SUFFIX] = Array(t)
450
451     def _parse_enum(self, name, definition):
452         self.logger.debug("Parsing enum %s: %s", name, definition)
453         constants = []
454         type_name = None
455         for item in definition:
456             if type(item) is dict and 'enumtype' in item:
457                 type_name = item['enumtype']
458                 continue
459             constants.append({'name': item[0], 'value': item[1]})
460         if not type_name:
461             raise ParseException("'enumtype' was not defined for %s" % definition)
462         return Enum(name, Field('value', self._types_by_name[type_name]), constants, definition, self.plugin_name)
463
464     def _parse_union(self, name, definition):
465         self.logger.debug("Parsing union %s: %s", name, definition)
466         crc, fields = self._parse_fields(definition)
467         return Union(name, crc, fields, definition, self.plugin_name)
468
469     def _parse_type(self, name, definition):
470         self.logger.debug("Parsing type %s: %s", name, definition)
471         crc, fields = self._parse_fields(definition)
472         return Class(name, crc, fields, definition, self.plugin_name)
473
474     def _parse_services(self):
475         self._dumps_by_details = {}
476         self._requests_by_reply = {}
477         for name, service in self._services.iteritems():
478             if _is_stream(service):
479                 self._dumps_by_details[service['reply']] = name
480             else:
481                 self._requests_by_reply[service['reply']] = name
482
483     def _parse_messages(self):
484         # Preserve ordering from JSON file to make debugging easier.
485         self._messages_by_name = OrderedDict()
486         for msg in self._messages:
487             try:
488                 name = msg[0]
489                 definition = msg[1:]
490                 self._messages_by_name[name] = self._parse_message(name, definition)
491             except ParseException as e:
492                 self.logger.warning("Failed to parse message %s: %s. Skipping message.", name, e)
493
494     def _parse_message(self, name, definition):
495         self.logger.debug("Parsing message %s: %s", name, definition)
496         crc, fields = self._parse_fields(definition)
497         if name in self._services:
498             service = self._services[name]
499             reply = service['reply']
500             if _is_stream(service):
501                 return Dump(name, reply, crc, filter(_is_request_field, fields), definition)
502             if reply:
503                 return Request(name, reply, crc, filter(_is_request_field, fields), definition)
504             else:
505                 return Event(name, crc, filter(_is_request_field, fields), definition)
506         elif name in self._requests_by_reply:
507             return Reply(name, self._requests_by_reply[name], crc, filter(_is_reply_field, fields), definition)
508         elif name in self._dumps_by_details:
509             return Details(name, self._dumps_by_details[name], crc, filter(_is_reply_field, fields), definition)
510         else:
511             # TODO: some messages like combined_counters are not visible in the services.
512             # Throw exception instead (requires fixing vppagigen).
513             return Event(name, crc, filter(_is_request_field, fields), definition)
514
515     def _parse_fields(self, definition):
516         crc = None
517         fields = []
518         for item in definition:
519             if type(item) == dict and 'crc' in item:
520                 crc = item['crc']
521             else:
522                 fields.append(self._parse_field(item, fields))
523         if not crc:
524             raise ParseException("CRC was not defined for %s" % definition)
525         return crc, fields
526
527     def _parse_field(self, field, fields):
528         type_name = _extract_type_name(field[0])
529
530         if type_name in self._types_by_name:
531             if len(field) > 2:
532                 # Array field
533                 array_len_field = None
534                 if len(field) == 4:
535                     for f in fields:
536                         if f.name == field[3]:
537                             array_len_field = f
538                     if not array_len_field:
539                         raise ParseException("Could not find field %s declared as length of array %s",
540                                              field[3], field[1])
541                 return Field(field[1], self._types_by_name[type_name + _ARRAY_SUFFIX], field[2], array_len_field)
542             else:
543                 return Field(field[1], self._types_by_name[type_name])
544         else:
545             raise ParseException("Unknown field type %s" % field)
546
547     def _validate_messages(self):
548         """
549         In case if message A is known to be reply for message B, and message B was not correctly parsed,
550         remove message A from the set of all messages.
551         """
552         to_be_removed = []
553         messages = self._messages_by_name
554         for name, msg in messages.iteritems():
555             if (is_request(msg) and msg.reply not in messages) \
556                     or (is_reply(msg) and msg.request not in messages) \
557                     or (is_dump(msg) and msg.details not in messages) \
558                     or (is_details(msg) and msg.dump not in messages):
559                 to_be_removed.append(name)
560
561         for name in to_be_removed:
562             del messages[name]
563
564         self.messages = self._messages_by_name.values()
565
566
567 _ARRAY_SUFFIX = '[]'
568
569
570 def _underscore_to_camelcase_upper(name):
571     return name.title().replace("_", "")
572
573
574 def _underscore_to_camelcase_lower(name):
575     name = name.title().replace("_", "")
576     return name[0].lower() + name[1:]
577
578
579 def _message_to_javadoc(message_definition):
580     """ Converts JSON message definition to javadoc """
581     formatted_message = pprint.pformat(message_definition, indent=4, width=120, depth=None)
582     return " * " + formatted_message.replace("\n", "\n * ")
583
584
585 def _is_stream(service):
586     """
587     Checks if service represents stream, e.g.:
588     "ip_address_dump": {
589         "reply": "ip_address_details",
590         "stream": true
591     }
592     :param service: JSON definition of service
593     :return: value assigned to "stream" or None
594     """
595     return "stream" in service
596
597
598 def _extract_type_name(name):
599     if name.startswith(_VPP_TYPE_PREFIX) and name.endswith(_VPP_TYPE_SUFFIX):
600         return name[len(_VPP_TYPE_PREFIX): - len(_VPP_TYPE_SUFFIX)]
601     return name
602
603 _VPP_TYPE_PREFIX = "vl_api_"
604
605 _VPP_TYPE_SUFFIX = "_t"
606
607
608 def _is_request_field(field):
609     # Skip fields that are hidden to the jvpp user (handled by JNI layer)
610     return field.name not in {'_vl_msg_id', 'client_index', 'context'}
611
612
613 def _is_reply_field(field):
614     # Skip fields that are hidden to the jvpp user:
615     # _vl_msg_id is handled at JNI layer,
616     # Unlike in the request case, context is visible to allow matching replies with requests at Java layer.
617     return field.name not in {'_vl_msg_id'}