resources/tools/presentation/convert_xml_json.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Convert output_info.xml files into JSON structures.
  15
  16 Version: 0.1.0
  17 Date:    22nd June 2021
  18
  19 The json structure is defined in https://gerrit.fd.io/r/c/csit/+/28992
  20 """
  21
  22 import os
  23 import re
  24 import json
  25 import logging
  26 import gzip
  27
  28 from os.path import join
  29 from shutil import rmtree
  30 from copy import deepcopy
  31 from json import loads
  32
  33 from pal_utils import get_files
  34
  35
  36 class JSONData:
  37     """A Class storing and manipulating data from tests.
  38     """
  39
  40     def __init__(self, template=None):
  41         """Initialization.
  42
  43         :param template: JSON formatted template used to store data. It can
  44             include default values.
  45         :type template: dict
  46         """
  47
  48         self._template = deepcopy(template)
  49         self._data = self._template if self._template else dict()
  50
  51     def __str__(self):
  52         """Return a string with human readable data.
  53
  54         :returns: Readable description.
  55         :rtype: str
  56         """
  57         return str(self._data)
  58
  59     def __repr__(self):
  60         """Return a string executable as Python constructor call.
  61
  62         :returns: Executable constructor call.
  63         :rtype: str
  64         """
  65         return f"JSONData(template={self._template!r})"
  66
  67     @property
  68     def data(self):
  69         """Getter
  70
  71         :return: Data stored in the object.
  72         :rtype: dict
  73         """
  74         return self._data
  75
  76     def update(self, kwargs):
  77         """Update the data with new data from the dictionary.
  78
  79         :param kwargs: Key value pairs to be added to the data.
  80         :type kwargs: dict
  81         """
  82         self._data.update(kwargs)
  83
  84     def set_key(self, key, val):
  85         """Setter.
  86
  87         :param key: The key to be updated / added.
  88         :param val: The key value.
  89         :type key: str
  90         :type val: object
  91         """
  92         self._data[key] = deepcopy(val)
  93
  94     def add_to_list(self, key, val):
  95         """Add an item to the list identified by key.
  96
  97         :param key: The key identifying the list.
  98         :param val: The val to be appended to the list. If val is a list,
  99             extend is used.
 100         """
 101         if self._data.get(key, None) is None:
 102             self._data[key] = list()
 103         if isinstance(val, list):
 104             self._data[key].extend(val)
 105         else:
 106             self._data[key].append(val)
 107
 108     def dump(self, file_out, indent=None):
 109         """Write JSON data to a file.
 110
 111         :param file_out: Path to the output JSON file.
 112         :param indent: Indentation of items in JSON string. It is directly
 113             passed to json.dump method.
 114         :type file_out: str
 115         :type indent: str
 116         """
 117         try:
 118             with open(file_out, u"w") as file_handler:
 119                 json.dump(self._data, file_handler, indent=indent)
 120         except OSError as err:
 121             logging.warning(f"{repr(err)} Skipping")
 122
 123     def load(self, file_in):
 124         """Load JSON data from a file.
 125
 126         :param file_in: Path to the input JSON file.
 127         :type file_in: str
 128         :raises: ValueError if the data being deserialized is not a valid
 129             JSON document.
 130         :raises: IOError if the file is not found or corrupted.
 131         """
 132         with open(file_in, u"r") as file_handler:
 133             self._data = json.load(file_handler)
 134
 135
 136 def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
 137     """Export data from a test to a json structure.
 138
 139     :param tid: Test ID.
 140     :param in_data: Test data.
 141     :param out: Path to output json file.
 142     :param template: JSON template with optional default values.
 143     :param metadata: Data which are not stored in XML structure.
 144     :type tid: str
 145     :type in_data: dict
 146     :type out: str
 147     :type template: dict
 148     :type metadata: dict
 149     """
 150
 151     data = JSONData(template=template)
 152
 153     data.update(metadata)
 154     data.set_key(u"test_id", tid)
 155     t_type = in_data.get(u"type", u"")
 156     t_type = u"NDRPDR" if t_type == u"CPS" else t_type  # It is NDRPDR
 157     data.set_key(u"test_type", t_type)
 158     tags = in_data.get(u"tags", list())
 159     data.set_key(u"tags", tags)
 160     data.set_key(u"documentation", in_data.get(u"documentation", u""))
 161     data.set_key(u"message", in_data.get(u"msg", u""))
 162     data.set_key(u"start_time", in_data.get(u"starttime", u""))
 163     data.set_key(u"end_time", in_data.get(u"endtime", u""))
 164     data.set_key(u"status", in_data.get(u"status", u"FAILED"))
 165     sut_type = u""
 166     if u"vpp" in tid:
 167         sut_type = u"vpp"
 168     elif u"dpdk" in tid:
 169         sut_type = u"dpdk"
 170     data.set_key(u"sut_type", sut_type)
 171
 172     # Process configuration history:
 173     in_papi = deepcopy(in_data.get(u"conf_history", None))
 174     if in_papi:
 175         regex_dut = re.compile(r'\*\*DUT(\d):\*\*')
 176         node_id = u"dut1"
 177         for line in in_papi.split(u"\n"):
 178             if not line:
 179                 continue
 180             groups = re.search(regex_dut, line)
 181             if groups:
 182                 node_id = f"dut{groups.group(1)}"
 183             else:
 184                 data.add_to_list(
 185                     u"log",
 186                     {
 187                         u"source_type": u"node",
 188                         u"source_id": node_id,
 189                         u"msg_type": u"papi",
 190                         u"log_level": u"INFO",
 191                         u"timestamp": in_data.get(u"starttime", u""),
 192                         u"msg": line,
 193                         u"data": list()
 194                     }
 195                 )
 196
 197     # Process show runtime:
 198     in_sh_run = deepcopy(in_data.get(u"show-run", None))
 199     if in_sh_run:
 200         # Transform to openMetrics format
 201         for key, val in in_sh_run.items():
 202             log_item = {
 203                 u"source_type": u"node",
 204                 u"source_id": key,
 205                 u"msg_type": u"metric",
 206                 u"log_level": u"INFO",
 207                 u"timestamp": in_data.get(u"starttime", u""),
 208                 u"msg": u"show_runtime",
 209                 u"data": list()
 210             }
 211             runtime = loads(val.get(u"runtime", list()))
 212             for item in runtime:
 213                 for metric, m_data in item.items():
 214                     if metric == u"name":
 215                         continue
 216                     for idx, m_item in enumerate(m_data):
 217                         log_item[u"data"].append(
 218                             {
 219                                 u"name": metric,
 220                                 u"value": m_item,
 221                                 u"labels": {
 222                                     u"host": val.get(u"host", u""),
 223                                     u"socket": val.get(u"socket", u""),
 224                                     u"graph_node": item.get(u"name", u""),
 225                                     u"thread_id": str(idx)
 226                                 }
 227                             }
 228                         )
 229             data.add_to_list(u"log", log_item)
 230
 231     # Process results:
 232     results = dict()
 233     if t_type == u"DEVICETEST":
 234         pass  # Nothing to add.
 235     elif t_type == u"NDRPDR":
 236         results = {
 237             u"throughput": {
 238                 u"unit":
 239                     u"cps" if u"TCP_CPS" in tags or u"UDP_CPS" in tags
 240                     else u"pps",
 241                 u"ndr": {
 242                     u"value": {
 243                         u"lower": in_data.get(u"throughput", dict()).
 244                                   get(u"NDR", dict()).get(u"LOWER", u"NaN"),
 245                         u"upper": in_data.get(u"throughput", dict()).
 246                                   get(u"NDR", dict()).get(u"UPPER", u"NaN")
 247                     },
 248                     u"value_gbps": {
 249                         u"lower": in_data.get(u"gbps", dict()).
 250                                   get(u"NDR", dict()).get(u"LOWER", u"NaN"),
 251                         u"upper": in_data.get(u"gbps", dict()).
 252                                   get(u"NDR", dict()).get(u"UPPER", u"NaN")
 253                     }
 254                 },
 255                 u"pdr": {
 256                     u"value": {
 257                         u"lower": in_data.get(u"throughput", dict()).
 258                                   get(u"PDR", dict()).get(u"LOWER", u"NaN"),
 259                         u"upper": in_data.get(u"throughput", dict()).
 260                                   get(u"PDR", dict()).get(u"UPPER", u"NaN")
 261                     },
 262                     u"value_gbps": {
 263                         u"lower": in_data.get(u"gbps", dict()).
 264                                   get(u"PDR", dict()).get(u"LOWER", u"NaN"),
 265                         u"upper": in_data.get(u"gbps", dict()).
 266                                   get(u"PDR", dict()).get(u"UPPER", u"NaN")
 267                     }
 268                 }
 269             },
 270             u"latency": {
 271                 u"forward": {
 272                     u"pdr_90": in_data.get(u"latency", dict()).
 273                                get(u"PDR90", dict()).get(u"direction1", u"NaN"),
 274                     u"pdr_50": in_data.get(u"latency", dict()).
 275                                get(u"PDR50", dict()).get(u"direction1", u"NaN"),
 276                     u"pdr_10": in_data.get(u"latency", dict()).
 277                                get(u"PDR10", dict()).get(u"direction1", u"NaN"),
 278                     u"pdr_0": in_data.get(u"latency", dict()).
 279                               get(u"LAT0", dict()).get(u"direction1", u"NaN")
 280                 },
 281                 u"reverse": {
 282                     u"pdr_90": in_data.get(u"latency", dict()).
 283                                get(u"PDR90", dict()).get(u"direction2", u"NaN"),
 284                     u"pdr_50": in_data.get(u"latency", dict()).
 285                                get(u"PDR50", dict()).get(u"direction2", u"NaN"),
 286                     u"pdr_10": in_data.get(u"latency", dict()).
 287                                get(u"PDR10", dict()).get(u"direction2", u"NaN"),
 288                     u"pdr_0": in_data.get(u"latency", dict()).
 289                               get(u"LAT0", dict()).get(u"direction2", u"NaN")
 290                 }
 291             }
 292         }
 293     elif t_type == "MRR":
 294         results = {
 295             u"unit": u"pps",  # Old data use only pps
 296             u"samples": in_data.get(u"result", dict()).get(u"samples", list()),
 297             u"avg": in_data.get(u"result", dict()).get(u"receive-rate", u"NaN"),
 298             u"stdev": in_data.get(u"result", dict()).
 299                       get(u"receive-stdev", u"NaN")
 300         }
 301     elif t_type == "SOAK":
 302         results = {
 303             u"critical_rate": {
 304                 u"lower": in_data.get(u"throughput", dict()).
 305                           get(u"LOWER", u"NaN"),
 306                 u"upper": in_data.get(u"throughput", dict()).
 307                           get(u"UPPER", u"NaN"),
 308             }
 309         }
 310     elif t_type == "HOSTSTACK":
 311         results = in_data.get(u"result", dict())
 312     # elif t_type == "TCP":  # Not used ???
 313     #     results = in_data.get(u"result", u"NaN")
 314     elif t_type == "RECONF":
 315         results = {
 316             u"loss": in_data.get(u"result", dict()).get(u"loss", u"NaN"),
 317             u"time": in_data.get(u"result", dict()).get(u"time", u"NaN")
 318         }
 319     else:
 320         pass
 321     data.set_key(u"results", results)
 322
 323     data.dump(out, indent=u"    ")
 324
 325
 326 def convert_xml_to_json(spec, data):
 327     """Convert downloaded XML files into JSON.
 328
 329     Procedure:
 330     - create one json file for each test,
 331     - gzip all json files one by one,
 332     - delete json files.
 333
 334     :param spec: Specification read from the specification files.
 335     :param data: Input data parsed from output.xml files.
 336     :type spec: Specification
 337     :type data: InputData
 338     """
 339
 340     logging.info(u"Converting downloaded XML files to JSON ...")
 341
 342     template_name = spec.output.get(u"use-template", None)
 343     structure = spec.output.get(u"structure", u"tree")
 344     if template_name:
 345         with open(template_name, u"r") as file_handler:
 346             template = json.load(file_handler)
 347     else:
 348         template = None
 349
 350     build_dir = spec.environment[u"paths"][u"DIR[BUILD,JSON]"]
 351     try:
 352         rmtree(build_dir)
 353     except FileNotFoundError:
 354         pass  # It does not exist
 355
 356     os.mkdir(build_dir)
 357
 358     for job, builds in data.data.items():
 359         logging.info(f"  Processing job {job}")
 360         if structure == "tree":
 361             os.makedirs(join(build_dir, job), exist_ok=True)
 362         for build_nr, build in builds.items():
 363             logging.info(f"  Processing build {build_nr}")
 364             if structure == "tree":
 365                 os.makedirs(join(build_dir, job, build_nr), exist_ok=True)
 366             for test_id, test_data in build[u"tests"].items():
 367                 groups = re.search(re.compile(r'-(\d+[tT](\d+[cC]))-'), test_id)
 368                 if groups:
 369                     test_id = test_id.replace(groups.group(1), groups.group(2))
 370                 logging.info(f"  Processing test {test_id}")
 371                 if structure == "tree":
 372                     dirs = test_id.split(u".")[:-1]
 373                     name = test_id.split(u".")[-1]
 374                     os.makedirs(
 375                         join(build_dir, job, build_nr, *dirs), exist_ok=True
 376                     )
 377                     file_name = \
 378                         f"{join(build_dir, job, build_nr, *dirs, name)}.json"
 379                 else:
 380                     file_name = join(
 381                         build_dir,
 382                         u'.'.join((job, build_nr, test_id, u'json'))
 383                     )
 384                 suite_id = test_id.rsplit(u".", 1)[0].replace(u" ", u"_")
 385                 _export_test_from_xml_to_json(
 386                     test_id, test_data, file_name, template,
 387                     {
 388                         u"ci": u"jenkins.fd.io",
 389                         u"job": job,
 390                         u"build_number": build_nr,
 391                         u"suite_id": suite_id,
 392                         u"suite_doc": build[u"suites"].get(suite_id, dict()).
 393                                       get(u"doc", u""),
 394                         u"testbed": build[u"metadata"].get(u"testbed", u""),
 395                         u"sut_version": build[u"metadata"].get(u"version", u"")
 396                     }
 397                 )
 398
 399     # gzip the json files:
 400     for file in get_files(build_dir, u"json"):
 401         with open(file, u"rb") as src:
 402             with gzip.open(f"{file}.gz", u"wb") as dst:
 403                 dst.writelines(src)
 404             os.remove(file)
 405
 406     logging.info(u"Done.")