resources/tools/presentation/convert_xml_json.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Convert output_info.xml files into JSON structures.
  15
  16 Version: 0.1.0
  17 Date:    22nd June 2021
  18
  19 The json structure is defined in https://gerrit.fd.io/r/c/csit/+/28992
  20 """
  21
  22 import os
  23 import re
  24 import json
  25 import logging
  26 import gzip
  27
  28 from os.path import join
  29 from shutil import rmtree
  30 from copy import deepcopy
  31 from json import loads
  32
  33 from pal_utils import get_files
  34
  35
  36 class JSONData:
  37     """A Class storing and manipulating data from tests.
  38     """
  39
  40     def __init__(self, template=None):
  41         """Initialization.
  42
  43         :param template: JSON formatted template used to store data. It can
  44             include default values.
  45         :type template: dict
  46         """
  47
  48         self._template = deepcopy(template)
  49         self._data = self._template if self._template else dict()
  50
  51     def __str__(self):
  52         """Return a string with human readable data.
  53
  54         :returns: Readable description.
  55         :rtype: str
  56         """
  57         return str(self._data)
  58
  59     def __repr__(self):
  60         """Return a string executable as Python constructor call.
  61
  62         :returns: Executable constructor call.
  63         :rtype: str
  64         """
  65         return f"JSONData(template={self._template!r})"
  66
  67     @property
  68     def data(self):
  69         """Getter
  70
  71         :return: Data stored in the object.
  72         :rtype: dict
  73         """
  74         return self._data
  75
  76     def update(self, kwargs):
  77         """Update the data with new data from the dictionary.
  78
  79         :param kwargs: Key value pairs to be added to the data.
  80         :type kwargs: dict
  81         """
  82         self._data.update(kwargs)
  83
  84     def set_key(self, key, val):
  85         """Setter.
  86
  87         :param key: The key to be updated / added.
  88         :param val: The key value.
  89         :type key: str
  90         :type val: object
  91         """
  92         self._data[key] = deepcopy(val)
  93
  94     def add_to_list(self, key, val):
  95         """Add an item to the list identified by key.
  96
  97         :param key: The key identifying the list.
  98         :param val: The val to be appended to the list. If val is a list,
  99             extend is used.
 100         """
 101         if self._data.get(key, None) is None:
 102             self._data[key] = list()
 103         if isinstance(val, list):
 104             self._data[key].extend(val)
 105         else:
 106             self._data[key].append(val)
 107
 108     def dump(self, file_out, indent=None):
 109         """Write JSON data to a file.
 110
 111         :param file_out: Path to the output JSON file.
 112         :param indent: Indentation of items in JSON string. It is directly
 113             passed to json.dump method.
 114         :type file_out: str
 115         :type indent: str
 116         """
 117         try:
 118             with open(file_out, u"w") as file_handler:
 119                 json.dump(self._data, file_handler, indent=indent)
 120         except OSError as err:
 121             logging.warning(f"{repr(err)} Skipping")
 122
 123     def load(self, file_in):
 124         """Load JSON data from a file.
 125
 126         :param file_in: Path to the input JSON file.
 127         :type file_in: str
 128         :raises: ValueError if the data being deserialized is not a valid
 129             JSON document.
 130         :raises: IOError if the file is not found or corrupted.
 131         """
 132         with open(file_in, u"r") as file_handler:
 133             self._data = json.load(file_handler)
 134
 135
 136 def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
 137     """Export data from a test to a json structure.
 138
 139     :param tid: Test ID.
 140     :param in_data: Test data.
 141     :param out: Path to output json file.
 142     :param template: JSON template with optional default values.
 143     :param metadata: Data which are not stored in XML structure.
 144     :type tid: str
 145     :type in_data: dict
 146     :type out: str
 147     :type template: dict
 148     :type metadata: dict
 149     """
 150
 151     data = JSONData(template=template)
 152
 153     data.update(metadata)
 154     data.set_key(u"test_id", tid)
 155     t_type = in_data.get(u"type", u"")
 156     t_type = u"NDRPDR" if t_type == u"CPS" else t_type  # It is NDRPDR
 157     data.set_key(u"test_type", t_type)
 158     tags = in_data.get(u"tags", list())
 159     data.set_key(u"tags", tags)
 160     data.set_key(u"documentation", in_data.get(u"documentation", u""))
 161     data.set_key(u"message", in_data.get(u"msg", u""))
 162     data.set_key(u"start_time", in_data.get(u"starttime", u""))
 163     data.set_key(u"end_time", in_data.get(u"endtime", u""))
 164     data.set_key(u"status", in_data.get(u"status", u"FAILED"))
 165     sut_type = u""
 166     if u"vpp" in tid:
 167         sut_type = u"vpp"
 168     elif u"dpdk" in tid:
 169         sut_type = u"dpdk"
 170     data.set_key(u"sut_type", sut_type)
 171
 172     # Process configuration history:
 173     in_papi = deepcopy(in_data.get(u"conf_history", None))
 174     if in_papi:
 175         regex_dut = re.compile(r'\*\*DUT(\d):\*\*')
 176         node_id = u"dut1"
 177         for line in in_papi.split(u"\n"):
 178             if not line:
 179                 continue
 180             groups = re.search(regex_dut, line)
 181             if groups:
 182                 node_id = f"dut{groups.group(1)}"
 183             else:
 184                 data.add_to_list(
 185                     u"log",
 186                     {
 187                         u"source_type": u"node",
 188                         u"source_id": node_id,
 189                         u"msg_type": u"papi",
 190                         u"log_level": u"INFO",
 191                         u"timestamp": in_data.get(u"starttime", u""),
 192                         u"msg": line,
 193                         u"data": list()
 194                     }
 195                 )
 196
 197     # Process show runtime:
 198     if in_data.get(u"telemetry-show-run", None):
 199         for item in in_data[u"telemetry-show-run"].values():
 200             data.add_to_list(u"log", item.get(u"runtime", dict()))
 201     else:
 202         in_sh_run = deepcopy(in_data.get(u"show-run", None))
 203         if in_sh_run:
 204             # Transform to openMetrics format
 205             for key, val in in_sh_run.items():
 206                 log_item = {
 207                     u"source_type": u"node",
 208                     u"source_id": key,
 209                     u"msg_type": u"metric",
 210                     u"log_level": u"INFO",
 211                     u"timestamp": in_data.get(u"starttime", u""),
 212                     u"msg": u"show_runtime",
 213                     u"data": list()
 214                 }
 215                 runtime = loads(val.get(u"runtime", list()))
 216                 for item in runtime:
 217                     for metric, m_data in item.items():
 218                         if metric == u"name":
 219                             continue
 220                         for idx, m_item in enumerate(m_data):
 221                             log_item[u"data"].append(
 222                                 {
 223                                     u"name": metric,
 224                                     u"value": m_item,
 225                                     u"labels": {
 226                                         u"host": val.get(u"host", u""),
 227                                         u"socket": val.get(u"socket", u""),
 228                                         u"graph_node": item.get(u"name", u""),
 229                                         u"thread_id": str(idx)
 230                                     }
 231                                 }
 232                             )
 233                 data.add_to_list(u"log", log_item)
 234
 235     # Process results:
 236     results = dict()
 237     if t_type == u"DEVICETEST":
 238         pass  # Nothing to add.
 239     elif t_type == u"NDRPDR":
 240         results = {
 241             u"throughput": {
 242                 u"unit":
 243                     u"cps" if u"TCP_CPS" in tags or u"UDP_CPS" in tags
 244                     else u"pps",
 245                 u"ndr": {
 246                     u"value": {
 247                         u"lower": in_data.get(u"throughput", dict()).
 248                                   get(u"NDR", dict()).get(u"LOWER", u"NaN"),
 249                         u"upper": in_data.get(u"throughput", dict()).
 250                                   get(u"NDR", dict()).get(u"UPPER", u"NaN")
 251                     },
 252                     u"value_gbps": {
 253                         u"lower": in_data.get(u"gbps", dict()).
 254                                   get(u"NDR", dict()).get(u"LOWER", u"NaN"),
 255                         u"upper": in_data.get(u"gbps", dict()).
 256                                   get(u"NDR", dict()).get(u"UPPER", u"NaN")
 257                     }
 258                 },
 259                 u"pdr": {
 260                     u"value": {
 261                         u"lower": in_data.get(u"throughput", dict()).
 262                                   get(u"PDR", dict()).get(u"LOWER", u"NaN"),
 263                         u"upper": in_data.get(u"throughput", dict()).
 264                                   get(u"PDR", dict()).get(u"UPPER", u"NaN")
 265                     },
 266                     u"value_gbps": {
 267                         u"lower": in_data.get(u"gbps", dict()).
 268                                   get(u"PDR", dict()).get(u"LOWER", u"NaN"),
 269                         u"upper": in_data.get(u"gbps", dict()).
 270                                   get(u"PDR", dict()).get(u"UPPER", u"NaN")
 271                     }
 272                 }
 273             },
 274             u"latency": {
 275                 u"forward": {
 276                     u"pdr_90": in_data.get(u"latency", dict()).
 277                                get(u"PDR90", dict()).get(u"direction1", u"NaN"),
 278                     u"pdr_50": in_data.get(u"latency", dict()).
 279                                get(u"PDR50", dict()).get(u"direction1", u"NaN"),
 280                     u"pdr_10": in_data.get(u"latency", dict()).
 281                                get(u"PDR10", dict()).get(u"direction1", u"NaN"),
 282                     u"pdr_0": in_data.get(u"latency", dict()).
 283                               get(u"LAT0", dict()).get(u"direction1", u"NaN")
 284                 },
 285                 u"reverse": {
 286                     u"pdr_90": in_data.get(u"latency", dict()).
 287                                get(u"PDR90", dict()).get(u"direction2", u"NaN"),
 288                     u"pdr_50": in_data.get(u"latency", dict()).
 289                                get(u"PDR50", dict()).get(u"direction2", u"NaN"),
 290                     u"pdr_10": in_data.get(u"latency", dict()).
 291                                get(u"PDR10", dict()).get(u"direction2", u"NaN"),
 292                     u"pdr_0": in_data.get(u"latency", dict()).
 293                               get(u"LAT0", dict()).get(u"direction2", u"NaN")
 294                 }
 295             }
 296         }
 297     elif t_type == "MRR":
 298         results = {
 299             u"unit": u"pps",  # Old data use only pps
 300             u"samples": in_data.get(u"result", dict()).get(u"samples", list()),
 301             u"avg": in_data.get(u"result", dict()).get(u"receive-rate", u"NaN"),
 302             u"stdev": in_data.get(u"result", dict()).
 303                       get(u"receive-stdev", u"NaN")
 304         }
 305     elif t_type == "SOAK":
 306         results = {
 307             u"critical_rate": {
 308                 u"lower": in_data.get(u"throughput", dict()).
 309                           get(u"LOWER", u"NaN"),
 310                 u"upper": in_data.get(u"throughput", dict()).
 311                           get(u"UPPER", u"NaN"),
 312             }
 313         }
 314     elif t_type == "HOSTSTACK":
 315         results = in_data.get(u"result", dict())
 316     # elif t_type == "TCP":  # Not used ???
 317     #     results = in_data.get(u"result", u"NaN")
 318     elif t_type == "RECONF":
 319         results = {
 320             u"loss": in_data.get(u"result", dict()).get(u"loss", u"NaN"),
 321             u"time": in_data.get(u"result", dict()).get(u"time", u"NaN")
 322         }
 323     else:
 324         pass
 325     data.set_key(u"results", results)
 326
 327     data.dump(out, indent=u"    ")
 328
 329
 330 def convert_xml_to_json(spec, data):
 331     """Convert downloaded XML files into JSON.
 332
 333     Procedure:
 334     - create one json file for each test,
 335     - gzip all json files one by one,
 336     - delete json files.
 337
 338     :param spec: Specification read from the specification files.
 339     :param data: Input data parsed from output.xml files.
 340     :type spec: Specification
 341     :type data: InputData
 342     """
 343
 344     logging.info(u"Converting downloaded XML files to JSON ...")
 345
 346     template_name = spec.output.get(u"use-template", None)
 347     structure = spec.output.get(u"structure", u"tree")
 348     if template_name:
 349         with open(template_name, u"r") as file_handler:
 350             template = json.load(file_handler)
 351     else:
 352         template = None
 353
 354     build_dir = spec.environment[u"paths"][u"DIR[BUILD,JSON]"]
 355     try:
 356         rmtree(build_dir)
 357     except FileNotFoundError:
 358         pass  # It does not exist
 359
 360     os.mkdir(build_dir)
 361
 362     for job, builds in data.data.items():
 363         logging.info(f"  Processing job {job}")
 364         if structure == "tree":
 365             os.makedirs(join(build_dir, job), exist_ok=True)
 366         for build_nr, build in builds.items():
 367             logging.info(f"  Processing build {build_nr}")
 368             if structure == "tree":
 369                 os.makedirs(join(build_dir, job, build_nr), exist_ok=True)
 370             for test_id, test_data in build[u"tests"].items():
 371                 groups = re.search(re.compile(r'-(\d+[tT](\d+[cC]))-'), test_id)
 372                 if groups:
 373                     test_id = test_id.replace(groups.group(1), groups.group(2))
 374                 logging.info(f"  Processing test {test_id}")
 375                 if structure == "tree":
 376                     dirs = test_id.split(u".")[:-1]
 377                     name = test_id.split(u".")[-1]
 378                     os.makedirs(
 379                         join(build_dir, job, build_nr, *dirs), exist_ok=True
 380                     )
 381                     file_name = \
 382                         f"{join(build_dir, job, build_nr, *dirs, name)}.json"
 383                 else:
 384                     file_name = join(
 385                         build_dir,
 386                         u'.'.join((job, build_nr, test_id, u'json'))
 387                     )
 388                 suite_id = test_id.rsplit(u".", 1)[0].replace(u" ", u"_")
 389                 _export_test_from_xml_to_json(
 390                     test_id, test_data, file_name, template,
 391                     {
 392                         u"ci": u"jenkins.fd.io",
 393                         u"job": job,
 394                         u"build_number": build_nr,
 395                         u"suite_id": suite_id,
 396                         u"suite_doc": build[u"suites"].get(suite_id, dict()).
 397                                       get(u"doc", u""),
 398                         u"testbed": build[u"metadata"].get(u"testbed", u""),
 399                         u"sut_version": build[u"metadata"].get(u"version", u"")
 400                     }
 401                 )
 402
 403     # gzip the json files:
 404     for file in get_files(build_dir, u"json"):
 405         with open(file, u"rb") as src:
 406             with gzip.open(f"{file}.gz", u"wb") as dst:
 407                 dst.writelines(src)
 408             os.remove(file)
 409
 410     logging.info(u"Done.")