resources/tools/presentation/convert_xml_json.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Convert output_info.xml files into JSON structures.
  15
  16 Version: 0.1.0
  17 Date:    22nd June 2021
  18
  19 The json structure is defined in https://gerrit.fd.io/r/c/csit/+/28992
  20 """
  21
  22 import os
  23 import re
  24 import json
  25 import logging
  26 import gzip
  27
  28 from os.path import join
  29 from shutil import rmtree
  30 from copy import deepcopy
  31
  32 from pal_utils import get_files
  33
  34
  35 class JSONData:
  36     """A Class storing and manipulating data from tests.
  37     """
  38
  39     def __init__(self, template=None):
  40         """Initialization.
  41
  42         :param template: JSON formatted template used to store data. It can
  43             include default values.
  44         :type template: dict
  45         """
  46
  47         self._template = deepcopy(template)
  48         self._data = self._template if self._template else dict()
  49
  50     def __str__(self):
  51         """Return a string with human readable data.
  52
  53         :returns: Readable description.
  54         :rtype: str
  55         """
  56         return str(self._data)
  57
  58     def __repr__(self):
  59         """Return a string executable as Python constructor call.
  60
  61         :returns: Executable constructor call.
  62         :rtype: str
  63         """
  64         return f"JSONData(template={self._template!r})"
  65
  66     @property
  67     def data(self):
  68         """Getter
  69
  70         :return: Data stored in the object.
  71         :rtype: dict
  72         """
  73         return self._data
  74
  75     def update(self, kwargs):
  76         """Update the data with new data from the dictionary.
  77
  78         :param kwargs: Key value pairs to be added to the data.
  79         :type kwargs: dict
  80         """
  81         self._data.update(kwargs)
  82
  83     def set_key(self, key, val):
  84         """Setter.
  85
  86         :param key: The key to be updated / added.
  87         :param val: The key value.
  88         :type key: str
  89         :type val: object
  90         """
  91         self._data[key] = deepcopy(val)
  92
  93     def add_to_list(self, key, val):
  94         """Add an item to the list identified by key.
  95
  96         :param key: The key identifying the list.
  97         :param val: The val to be appended to the list. If val is a list,
  98             extend is used.
  99         """
 100         if self._data.get(key, None) is None:
 101             self._data[key] = list()
 102         if isinstance(val, list):
 103             self._data[key].extend(val)
 104         else:
 105             self._data[key].append(val)
 106
 107     def dump(self, file_out, indent=None):
 108         """Write JSON data to a file.
 109
 110         :param file_out: Path to the output JSON file.
 111         :param indent: Indentation of items in JSON string. It is directly
 112             passed to json.dump method.
 113         :type file_out: str
 114         :type indent: str
 115         """
 116         try:
 117             with open(file_out, u"w") as file_handler:
 118                 json.dump(self._data, file_handler, indent=indent)
 119         except OSError as err:
 120             logging.warning(f"{repr(err)} Skipping")
 121
 122     def load(self, file_in):
 123         """Load JSON data from a file.
 124
 125         :param file_in: Path to the input JSON file.
 126         :type file_in: str
 127         :raises: ValueError if the data being deserialized is not a valid
 128             JSON document.
 129         :raises: IOError if the file is not found or corrupted.
 130         """
 131         with open(file_in, u"r") as file_handler:
 132             self._data = json.load(file_handler)
 133
 134
 135 def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
 136     """Export data from a test to a json structure.
 137
 138     :param tid: Test ID.
 139     :param in_data: Test data.
 140     :param out: Path to output json file.
 141     :param template: JSON template with optional default values.
 142     :param metadata: Data which are not stored in XML structure.
 143     :type tid: str
 144     :type in_data: dict
 145     :type out: str
 146     :type template: dict
 147     :type metadata: dict
 148     """
 149
 150     data = JSONData(template=template)
 151
 152     data.update(metadata)
 153     data.set_key(u"test_id", tid)
 154     t_type = in_data.get(u"type", u"")
 155     t_type = u"NDRPDR" if t_type == u"CPS" else t_type  # It is NDRPDR
 156     data.set_key(u"test_type", t_type)
 157     tags = in_data.get(u"tags", list())
 158     data.set_key(u"tags", tags)
 159     data.set_key(u"documentation", in_data.get(u"documentation", u""))
 160     data.set_key(u"message", in_data.get(u"msg", u""))
 161     data.set_key(u"start_time", in_data.get(u"starttime", u""))
 162     data.set_key(u"end_time", in_data.get(u"endtime", u""))
 163     data.set_key(u"status", in_data.get(u"status", u"FAILED"))
 164     sut_type = u""
 165     if u"vpp" in tid:
 166         sut_type = u"vpp"
 167     elif u"dpdk" in tid:
 168         sut_type = u"dpdk"
 169     data.set_key(u"sut_type", sut_type)
 170
 171     # Process configuration history:
 172     in_papi = deepcopy(in_data.get(u"conf_history", None))
 173     if in_papi:
 174         regex_dut = re.compile(r'\*\*DUT(\d):\*\*')
 175         node_id = u"dut1"
 176         for line in in_papi.split(u"\n"):
 177             if not line:
 178                 continue
 179             groups = re.search(regex_dut, line)
 180             if groups:
 181                 node_id = f"dut{groups.group(1)}"
 182             else:
 183                 data.add_to_list(
 184                     u"log",
 185                     {
 186                         u"source_type": u"node",
 187                         u"source_id": node_id,
 188                         u"msg_type": u"papi",
 189                         u"log_level": u"INFO",
 190                         u"timestamp": in_data.get(u"starttime", u""),
 191                         u"msg": line,
 192                         u"data": list()
 193                     }
 194                 )
 195
 196     # Process show runtime:
 197     in_sh_run = deepcopy(in_data.get(u"show-run", None))
 198     if in_sh_run:
 199         # Transform to openMetrics format
 200         for key, val in in_sh_run.items():
 201             log_item = {
 202                 u"source_type": u"node",
 203                 u"source_id": key,
 204                 u"msg_type": u"metric",
 205                 u"log_level": u"INFO",
 206                 u"timestamp": in_data.get(u"starttime", u""),
 207                 u"msg": u"show_runtime",
 208                 u"data": list()
 209             }
 210             for item in val.get(u"runtime", list()):
 211                 for metric, m_data in item.items():
 212                     if metric == u"name":
 213                         continue
 214                     for idx, m_item in enumerate(m_data):
 215                         log_item[u"data"].append(
 216                             {
 217                                 u"name": metric,
 218                                 u"value": m_item,
 219                                 u"labels": {
 220                                     u"host": val.get(u"host", u""),
 221                                     u"socket": val.get(u"socket", u""),
 222                                     u"graph_node": item.get(u"name", u""),
 223                                     u"thread_id": str(idx)
 224                                 }
 225                             }
 226                         )
 227             data.add_to_list(u"log", log_item)
 228
 229     # Process results:
 230     results = dict()
 231     if t_type == u"DEVICETEST":
 232         pass  # Nothing to add.
 233     elif t_type == u"NDRPDR":
 234         results = {
 235             u"throughput": {
 236                 u"unit":
 237                     u"cps" if u"TCP_CPS" in tags or u"UDP_CPS" in tags
 238                     else u"pps",
 239                 u"ndr": {
 240                     u"value": {
 241                         u"lower": in_data.get(u"throughput", dict()).
 242                                   get(u"NDR", dict()).get(u"LOWER", u"NaN"),
 243                         u"upper": in_data.get(u"throughput", dict()).
 244                                   get(u"NDR", dict()).get(u"UPPER", u"NaN")
 245                     },
 246                     u"value_gbps": {
 247                         u"lower": in_data.get(u"gbps", dict()).
 248                                   get(u"NDR", dict()).get(u"LOWER", u"NaN"),
 249                         u"upper": in_data.get(u"gbps", dict()).
 250                                   get(u"NDR", dict()).get(u"UPPER", u"NaN")
 251                     }
 252                 },
 253                 u"pdr": {
 254                     u"value": {
 255                         u"lower": in_data.get(u"throughput", dict()).
 256                                   get(u"PDR", dict()).get(u"LOWER", u"NaN"),
 257                         u"upper": in_data.get(u"throughput", dict()).
 258                                   get(u"PDR", dict()).get(u"UPPER", u"NaN")
 259                     },
 260                     u"value_gbps": {
 261                         u"lower": in_data.get(u"gbps", dict()).
 262                                   get(u"PDR", dict()).get(u"LOWER", u"NaN"),
 263                         u"upper": in_data.get(u"gbps", dict()).
 264                                   get(u"PDR", dict()).get(u"UPPER", u"NaN")
 265                     }
 266                 }
 267             },
 268             u"latency": {
 269                 u"forward": {
 270                     u"pdr_90": in_data.get(u"latency", dict()).
 271                                get(u"PDR90", dict()).get(u"direction1", u"NaN"),
 272                     u"pdr_50": in_data.get(u"latency", dict()).
 273                                get(u"PDR50", dict()).get(u"direction1", u"NaN"),
 274                     u"pdr_10": in_data.get(u"latency", dict()).
 275                                get(u"PDR10", dict()).get(u"direction1", u"NaN"),
 276                     u"pdr_0": in_data.get(u"latency", dict()).
 277                               get(u"LAT0", dict()).get(u"direction1", u"NaN")
 278                 },
 279                 u"reverse": {
 280                     u"pdr_90": in_data.get(u"latency", dict()).
 281                                get(u"PDR90", dict()).get(u"direction2", u"NaN"),
 282                     u"pdr_50": in_data.get(u"latency", dict()).
 283                                get(u"PDR50", dict()).get(u"direction2", u"NaN"),
 284                     u"pdr_10": in_data.get(u"latency", dict()).
 285                                get(u"PDR10", dict()).get(u"direction2", u"NaN"),
 286                     u"pdr_0": in_data.get(u"latency", dict()).
 287                               get(u"LAT0", dict()).get(u"direction2", u"NaN")
 288                 }
 289             }
 290         }
 291     elif t_type == "MRR":
 292         results = {
 293             u"unit": u"pps",  # Old data use only pps
 294             u"samples": in_data.get(u"result", dict()).get(u"samples", list()),
 295             u"avg": in_data.get(u"result", dict()).get(u"receive-rate", u"NaN"),
 296             u"stdev": in_data.get(u"result", dict()).
 297                       get(u"receive-stdev", u"NaN")
 298         }
 299     elif t_type == "SOAK":
 300         results = {
 301             u"critical_rate": {
 302                 u"lower": in_data.get(u"throughput", dict()).
 303                           get(u"LOWER", u"NaN"),
 304                 u"upper": in_data.get(u"throughput", dict()).
 305                           get(u"UPPER", u"NaN"),
 306             }
 307         }
 308     elif t_type == "HOSTSTACK":
 309         results = in_data.get(u"result", dict())
 310     # elif t_type == "TCP":  # Not used ???
 311     #     results = in_data.get(u"result", u"NaN")
 312     elif t_type == "RECONF":
 313         results = {
 314             u"loss": in_data.get(u"result", dict()).get(u"loss", u"NaN"),
 315             u"time": in_data.get(u"result", dict()).get(u"time", u"NaN")
 316         }
 317     else:
 318         pass
 319     data.set_key(u"results", results)
 320
 321     data.dump(out, indent=u"    ")
 322
 323
 324 def convert_xml_to_json(spec, data):
 325     """Convert downloaded XML files into JSON.
 326
 327     Procedure:
 328     - create one json file for each test,
 329     - gzip all json files one by one,
 330     - delete json files.
 331
 332     :param spec: Specification read from the specification files.
 333     :param data: Input data parsed from output.xml files.
 334     :type spec: Specification
 335     :type data: InputData
 336     """
 337
 338     logging.info(u"Converting downloaded XML files to JSON ...")
 339
 340     template_name = spec.output.get(u"use-template", None)
 341     structure = spec.output.get(u"structure", u"tree")
 342     if template_name:
 343         with open(template_name, u"r") as file_handler:
 344             template = json.load(file_handler)
 345     else:
 346         template = None
 347
 348     build_dir = spec.environment[u"paths"][u"DIR[BUILD,JSON]"]
 349     try:
 350         rmtree(build_dir)
 351     except FileNotFoundError:
 352         pass  # It does not exist
 353
 354     os.mkdir(build_dir)
 355
 356     for job, builds in data.data.items():
 357         logging.info(f"  Processing job {job}")
 358         if structure == "tree":
 359             os.makedirs(join(build_dir, job), exist_ok=True)
 360         for build_nr, build in builds.items():
 361             logging.info(f"  Processing build {build_nr}")
 362             if structure == "tree":
 363                 os.makedirs(join(build_dir, job, build_nr), exist_ok=True)
 364             for test_id, test_data in build[u"tests"].items():
 365                 groups = re.search(re.compile(r'-(\d+[tT](\d+[cC]))-'), test_id)
 366                 if groups:
 367                     test_id = test_id.replace(groups.group(1), groups.group(2))
 368                 logging.info(f"  Processing test {test_id}")
 369                 if structure == "tree":
 370                     dirs = test_id.split(u".")[:-1]
 371                     name = test_id.split(u".")[-1]
 372                     os.makedirs(
 373                         join(build_dir, job, build_nr, *dirs), exist_ok=True
 374                     )
 375                     file_name = \
 376                         f"{join(build_dir, job, build_nr, *dirs, name)}.json"
 377                 else:
 378                     file_name = join(
 379                         build_dir,
 380                         u'.'.join((job, build_nr, test_id, u'json'))
 381                     )
 382                 suite_id = test_id.rsplit(u".", 1)[0].replace(u" ", u"_")
 383                 _export_test_from_xml_to_json(
 384                     test_id, test_data, file_name, template,
 385                     {
 386                         u"ci": u"jenkins.fd.io",
 387                         u"job": job,
 388                         u"build_number": build_nr,
 389                         u"suite_id": suite_id,
 390                         u"suite_doc": build[u"suites"].get(suite_id, dict()).
 391                                       get(u"doc", u""),
 392                         u"testbed": build[u"metadata"].get(u"testbed", u""),
 393                         u"sut_version": build[u"metadata"].get(u"version", u"")
 394                     }
 395                 )
 396
 397     # gzip the json files:
 398     for file in get_files(build_dir, u"json"):
 399         with open(file, u"rb") as src:
 400             with gzip.open(f"{file}.gz", u"wb") as dst:
 401                 dst.writelines(src)
 402             os.remove(file)
 403
 404     logging.info(u"Done.")