--- /dev/null
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module for converting in-memory data into raw JSON output.
+
+CSIT and VPP PAPI are using custom data types
+that are not directly serializable into JSON.
+
+Thus, before writing the raw outpt onto disk,
+the data is recursively converted to equivalent serializable types,
+in extreme cases replaced by string representation.
+
+Validation is outside the scope of this module,
+as it should use the JSON data read from disk.
+"""
+
+import json
+import os
+
+from collections.abc import Iterable, Mapping, Set
+from enum import IntFlag
+
+
+def _pre_serialize_recursive(data):
+ """Recursively sort and convert to a more serializable form.
+
+ VPP PAPI code can give data with its own MACAddres type,
+ or various other enum and flag types.
+ The default json.JSONEncoder method raises TypeError on that.
+ First point of this function is to apply str() or repr()
+ to leaf values that need it.
+
+ Also, PAPI responses are namedtuples, which confuses
+ the json.JSONEncoder method (so it does not recurse).
+ Dictization (see PapiExecutor) helps somewhat, but it turns namedtuple
+ into a UserDict, which also confuses json.JSONEncoder.
+ Therefore, we recursively convert any Mapping into an ordinary dict.
+
+ We also convert iterables to list (sorted if the iterable was a set),
+ and prevent numbers from getting converted to strings.
+
+ As we are doing such low level operations,
+ we also convert mapping keys to strings
+ and sort the mapping items by keys alphabetically,
+ except "data" field moved to the end.
+
+ :param data: Object to make serializable, dictized when applicable.
+ :type data: object
+ :returns: Serializable equivalent of the argument.
+ :rtype: object
+ :raises ValueError: If the argument does not support string conversion.
+ """
+ # Recursion ends at scalar values, first handle irregular ones.
+ if isinstance(data, IntFlag):
+ return repr(data)
+ if isinstance(data, bytes):
+ return data.hex()
+ # The regular ones are good to go.
+ if isinstance(data, (str, int, float, bool)):
+ return data
+ # Recurse over, convert and sort mappings.
+ if isinstance(data, Mapping):
+ # Convert and sort alphabetically.
+ ret = {
+ str(key): _pre_serialize_recursive(data[key])
+ for key in sorted(data.keys())
+ }
+ # If exists, move "data" field to the end.
+ if u"data" in ret:
+ data_value = ret.pop(u"data")
+ ret[u"data"] = data_value
+ # If exists, move "type" field at the start.
+ if u"type" in ret:
+ type_value = ret.pop(u"type")
+ ret_old = ret
+ ret = dict(type=type_value)
+ ret.update(ret_old)
+ return ret
+ # Recurse over and convert iterables.
+ if isinstance(data, Iterable):
+ list_data = [_pre_serialize_recursive(item) for item in data]
+ # Additionally, sets are exported as sorted.
+ if isinstance(data, Set):
+ list_data = sorted(list_data)
+ return list_data
+ # Unknown structure, attempt str().
+ return str(data)
+
+
+def _pre_serialize_root(data):
+ """Recursively convert to a more serializable form, tweak order.
+
+ See _pre_serialize_recursive for most of changes this does.
+
+ The logic here (outside the recursive function) only affects
+ field ordering in the root mapping,
+ to make it more human friendly.
+ We are moving "version" to the top,
+ followed by start time and end time.
+ and various long fields (such as "log") to the bottom.
+
+ Some edits are done in-place, do not trust the argument value after calling.
+
+ :param data: Root data to make serializable, dictized when applicable.
+ :type data: dict
+ :returns: Order-tweaked version of the argument.
+ :rtype: dict
+ :raises KeyError: If the data does not contain required fields.
+ :raises TypeError: If the argument is not a dict.
+ :raises ValueError: If the argument does not support string conversion.
+ """
+ if not isinstance(data, dict):
+ raise RuntimeError(f"Root data object needs to be a dict: {data!r}")
+ data = _pre_serialize_recursive(data)
+ log = data.pop(u"log")
+ new_data = dict(version=data.pop(u"version"))
+ new_data[u"start_time"] = data.pop(u"start_time")
+ new_data[u"end_time"] = data.pop(u"end_time")
+ new_data.update(data)
+ new_data[u"log"] = log
+ return new_data
+
+
+def write_raw_output(raw_file_path, raw_data):
+ """Prepare data for serialization and dump into a file.
+
+ Ancestor directories are created if needed.
+
+ :param to_raw_path: Local filesystem path, including the file name.
+ :type to_raw_path: str
+ """
+ raw_data = _pre_serialize_root(raw_data)
+ os.makedirs(os.path.dirname(raw_file_path), exist_ok=True)
+ with open(raw_file_path, u"xt", encoding="utf-8") as file_out:
+ json.dump(raw_data, file_out, indent=1)
Code Review / csit.git / blobdiff