4 # Copyright (c) 2019 Cisco and/or its affiliates.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at:
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
19 # Convert from VPP API trace to JSON.
26 from ipaddress import *
27 from collections import namedtuple
28 from vpp_papi import MACAddress, VPPApiJSONFiles
33 def serialize_likely_small_unsigned_integer(x):
36 # Low bit set means it fits into 1 byte.
38 return struct.pack("B", 1 + 2 * r)
40 # Low 2 bits 1 0 means it fits into 2 bytes.
43 return struct.pack("<H", 4 * r + 2)
47 return struct.pack("<I", 8 * r + 4)
49 return struct.pack("<BQ", 0, x)
52 def unserialize_likely_small_unsigned_integer(data, offset):
53 y = struct.unpack_from("B", data, offset)[0]
58 p = struct.unpack_from("B", data, offset + 1)[0]
59 r += (y // 4) + (p << 6)
63 (p1, p2, p3) = struct.unpack_from("BBB", data, offset+1)
64 r += ((y // 8) + (p1 << (5 + 8 * 0))
65 + (p2 << (5 + 8 * 1)) + (p3 << (5 + 8 * 2)))
67 return struct.unpack_from(">Q", data, offset+1)[0], 8
70 def serialize_cstring(s):
71 bstring = s.encode('utf8')
73 b = serialize_likely_small_unsigned_integer(l)
74 b += struct.pack('{}s'.format(l), bstring)
78 def unserialize_cstring(data, offset):
79 l, size = unserialize_likely_small_unsigned_integer(data, offset)
80 name = struct.unpack_from('{}s'.format(l), data, offset+size)[0]
81 return name.decode('utf8'), size + len(name)
84 def unserialize_msgtbl(data, offset):
88 nmsg = struct.unpack_from(">I", data, offset)[0]
91 (msgid, size) = unserialize_likely_small_unsigned_integer(
94 (name, size) = unserialize_cstring(data, offset + o)
96 msgtable_by_id[msgid] = name
97 msgtable_by_name[name] = msgid
100 return msgtable_by_id, msgtable_by_name, o
103 def serialize_msgtbl(messages):
106 data = bytearray(100000)
108 data = struct.pack(">I", nmsg)
110 for k, v in messages.items():
111 name = k + '_' + v.crc[2:]
112 data += serialize_likely_small_unsigned_integer(v._vl_msg_id)
113 data += serialize_cstring(name)
117 def apitrace2json(messages, filename):
119 with open(filename, 'rb') as file:
120 bytes_read = file.read()
122 (nitems, msgtbl_size, wrapped) = struct.unpack_from(">IIB",
124 logging.debug('nitems: {} message table size: {} wrapped: {}'
125 .format(nitems, msgtbl_size, wrapped))
127 sys.stdout.write('Wrapped/incomplete trace, results may vary')
130 msgtbl_by_id, msgtbl_by_name, size = unserialize_msgtbl(bytes_read,
136 size = struct.unpack_from(">I", bytes_read, offset)[0]
140 msgid = struct.unpack_from(">H", bytes_read, offset)[0]
141 name = msgtbl_by_id[msgid]
142 n = name[:name.rfind("_")]
144 if n + '_' + msgobj.crc[2:] != name:
145 sys.exit("CRC Mismatch between JSON API definition "
146 "and trace. {}".format(name))
148 x, s = msgobj.unpack(bytes_read[offset:offset+size])
149 msgname = type(x).__name__
151 # Replace named tuple illegal _0
154 result.append({'name': msgname, 'args': y})
161 def json2apitrace(messages, filename):
162 """Input JSON file and API message definition. Output API trace
166 with open(filename, 'r') as file:
167 msgs = json.load(file, object_hook=vpp_decode)
171 msgobj = messages[name]
172 m['args']['_vl_msg_id'] = messages[name]._vl_msg_id
173 b = msgobj.pack(m['args'])
175 result += struct.pack('>I', len(b))
177 return len(msgs), result
180 class VPPEncoder(json.JSONEncoder):
181 def default(self, o):
183 return "base64:" + base64.b64encode(o).decode('utf-8')
184 # Let the base class default method raise the TypeError
185 return json.JSONEncoder.default(self, o)
187 def encode(self, obj):
188 def hint_tuples(item):
189 if isinstance(item, tuple):
190 return hint_tuples(item._asdict())
191 if isinstance(item, list):
192 return [hint_tuples(e) for e in item]
193 if isinstance(item, dict):
194 return {key: hint_tuples(value) for key, value in item.items()}
198 return super(VPPEncoder, self).encode(hint_tuples(obj))
202 for k, v in obj.items():
203 if type(v) is str and v.startswith('base64:'):
204 s = v.lstrip('base64:')
205 obj[k] = base64.b64decode(v[7:])
209 def vpp_encoder(obj):
210 if isinstance(obj, IPv6Network):
212 if isinstance(obj, IPv4Network):
214 if isinstance(obj, IPv6Address):
216 if isinstance(obj, IPv4Address):
218 if isinstance(obj, MACAddress):
220 if type(obj) is bytes:
221 return "base64:" + base64.b64encode(obj).decode('ascii')
222 raise TypeError('Unknown object {} {}\n'.format(type(obj), obj))
236 def topython(messages, services):
238 pp = pprint.PrettyPrinter()
241 #!/usr/bin/env python3
242 from vpp_papi import VPP, VppEnum
243 vpp = VPP(use_socket=True)
244 vpp.connect(name='vppapitrace')
248 if m['name'] not in services:
249 s += '# ignoring reply message: {}\n'.format(m['name'])
251 if m['name'] in message_filter:
252 s += '# ignoring message {}\n'.format(m['name'])
254 for k in argument_filter:
259 a = pp.pformat(m['args'])
260 s += 'rv = vpp.api.{}(**{})\n'.format(m['name'], a)
261 s += 'print("RV:", rv)\n'
262 s += 'vpp.disconnect()\n'
266 def todump_items(k, v, level):
267 klen = len(k) if k else 0
268 spaces = ' ' * level + ' ' * (klen + 3)
269 wrapper = textwrap.TextWrapper(initial_indent="", subsequent_indent=spaces, width=60)
273 s += ' ' * level + '{}:\n'.format(k)
274 for k2, v2 in v.items():
275 s += todump_items(k2, v2, level + 1)
280 s += '{}'.format(todump_items(k, v2, level))
284 w = wrapper.fill(bytes.hex(v))
285 s += ' ' * level + '{}: {}\n'.format(k, w)
289 s += ' ' * level + '{}: {}\n'.format(k, v)
293 def todump(messages, services):
295 pp = pprint.PrettyPrinter()
299 if m['name'] not in services:
300 s += '# ignoring reply message: {}\n'.format(m['name'])
302 #if m['name'] in message_filter:
303 # s += '# ignoring message {}\n'.format(m['name'])
305 for k in argument_filter:
310 a = pp.pformat(m['args'])
311 s += '{}:\n'.format(m['name'])
312 s += todump_items(None, m['args'], 0)
316 def init_api(apidir):
317 # Read API definitions
318 apifiles = VPPApiJSONFiles.find_api_files(api_dir=apidir)
321 for file in apifiles:
322 with open(file) as apidef_file:
323 m, s = VPPApiJSONFiles.process_json_file(apidef_file)
326 return messages, services
329 def replaymsgs(vpp, msgs):
332 if name not in vpp.services:
334 if name == 'control_ping':
337 m['args'].pop('client_index')
340 if m['args']['context'] == 0:
341 m['args']['context'] = 1
342 f = vpp.get_function(name)
344 print('RV {}'.format(rv))
348 """Replay into running VPP instance"""
350 from vpp_papi import VPP
355 filename, file_extension = os.path.splitext(args.input)
356 input_type = JSON if file_extension == '.json' else APITRACE
358 vpp = VPP(use_socket=args.socket)
359 rv = vpp.connect(name='vppapireplay', chroot_prefix=args.shmprefix)
361 sys.exit('Cannot connect to VPP')
363 if input_type == JSON:
364 with open(args.input, 'r') as file:
365 msgs = json.load(file, object_hook=vpp_decode)
367 msgs = apitrace2json(messages, args.input)
369 replaymsgs(vpp, msgs)
382 filename, file_extension = os.path.splitext(args.input)
383 input_type = JSON if file_extension == '.json' else APITRACE
384 filename, file_extension = os.path.splitext(args.output)
389 if file_extension == '.json' or filename == '-':
391 elif file_extension == '.py':
394 output_type = APITRACE
396 if input_type == output_type:
397 sys.exit("error: Nothing to convert between")
399 if input_type != JSON and output_type == APITRACE:
400 sys.exit("error: Input file must be JSON file: {}".format(args.input))
402 messages, services = init_api(args.apidir)
404 if input_type == JSON and output_type == APITRACE:
406 for k, v in messages.items():
410 n, result = json2apitrace(messages, args.input)
411 msgtbl = serialize_msgtbl(messages)
413 print('API messages: {}'.format(n))
414 header = struct.pack(">IIB", n, len(msgtbl), 0)
416 with open(args.output, 'wb') as outfile:
417 outfile.write(header)
418 outfile.write(msgtbl)
419 outfile.write(result)
423 if input_type == APITRACE:
424 result = apitrace2json(messages, args.input)
425 if output_type == PYTHON:
426 s = json.dumps(result, cls=VPPEncoder, default=vpp_encoder)
427 x = json.loads(s, object_hook=vpp_decode)
428 s = topython(x, services)
429 elif output_type == DUMP:
430 s = json.dumps(result, cls=VPPEncoder, default=vpp_encoder)
431 x = json.loads(s, object_hook=vpp_decode)
432 s = todump(x, services)
434 s = json.dumps(result, cls=VPPEncoder,
435 default=vpp_encoder, indent=4 * ' ')
436 elif output_type == PYTHON:
437 with open(args.input, 'r') as file:
438 x = json.load(file, object_hook=vpp_decode)
439 s = topython(x, services)
441 sys.exit('Input file must be API trace file: {}'.format(args.input))
443 if args.output == '-':
444 sys.stdout.write(s + '\n')
446 print('Generating {} from API trace: {}'
447 .format(args.output, args.input))
448 with open(args.output, 'w') as outfile:
455 parser = argparse.ArgumentParser()
456 parser.add_argument('--debug', action='store_true',
457 help='enable debug mode')
458 parser.add_argument('--apidir',
459 help='Location of JSON API definitions')
461 parser.set_defaults(func=general)
462 subparsers = parser.add_subparsers(title='subcommands',
463 description='valid subcommands',
464 help='additional help')
466 parser_convert = subparsers.add_parser('convert',
467 help='Convert API trace to JSON or Python and back')
468 parser_convert.add_argument('input',
469 help='Input file (API trace | JSON)')
470 parser_convert.add_argument('--todump', action='store_true', help='Output text format')
471 parser_convert.add_argument('output',
472 help='Output file (Python | JSON | API trace)')
473 parser_convert.set_defaults(func=generate)
476 parser_replay = subparsers.add_parser('replay',
477 help='Replay messages to running VPP instance')
478 parser_replay.add_argument('input', help='Input file (API trace | JSON)')
479 parser_replay.add_argument('--socket', action='store_true',
480 help='use default socket to connect to VPP')
481 parser_replay.add_argument('--shmprefix',
482 help='connect to VPP on shared memory prefix')
483 parser_replay.set_defaults(func=replay)
485 args = parser.parse_args()
487 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)