1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Convert output_info.xml files into JSON structures.
19 The json structure is defined in https://gerrit.fd.io/r/c/csit/+/28992
28 from os.path import join
29 from shutil import rmtree
30 from copy import deepcopy
31 from json import loads
33 from pal_utils import get_files
37 """A Class storing and manipulating data from tests.
40 def __init__(self, template=None):
43 :param template: JSON formatted template used to store data. It can
44 include default values.
48 self._template = deepcopy(template)
49 self._data = self._template if self._template else dict()
52 """Return a string with human readable data.
54 :returns: Readable description.
57 return str(self._data)
60 """Return a string executable as Python constructor call.
62 :returns: Executable constructor call.
65 return f"JSONData(template={self._template!r})"
71 :return: Data stored in the object.
76 def update(self, kwargs):
77 """Update the data with new data from the dictionary.
79 :param kwargs: Key value pairs to be added to the data.
82 self._data.update(kwargs)
84 def set_key(self, key, val):
87 :param key: The key to be updated / added.
88 :param val: The key value.
92 self._data[key] = deepcopy(val)
94 def add_to_list(self, key, val):
95 """Add an item to the list identified by key.
97 :param key: The key identifying the list.
98 :param val: The val to be appended to the list. If val is a list,
101 if self._data.get(key, None) is None:
102 self._data[key] = list()
103 if isinstance(val, list):
104 self._data[key].extend(val)
106 self._data[key].append(val)
108 def dump(self, file_out, indent=None):
109 """Write JSON data to a file.
111 :param file_out: Path to the output JSON file.
112 :param indent: Indentation of items in JSON string. It is directly
113 passed to json.dump method.
118 with open(file_out, u"w") as file_handler:
119 json.dump(self._data, file_handler, indent=indent)
120 except OSError as err:
121 logging.warning(f"{repr(err)} Skipping")
123 def load(self, file_in):
124 """Load JSON data from a file.
126 :param file_in: Path to the input JSON file.
128 :raises: ValueError if the data being deserialized is not a valid
130 :raises: IOError if the file is not found or corrupted.
132 with open(file_in, u"r") as file_handler:
133 self._data = json.load(file_handler)
136 def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
137 """Export data from a test to a json structure.
140 :param in_data: Test data.
141 :param out: Path to output json file.
142 :param template: JSON template with optional default values.
143 :param metadata: Data which are not stored in XML structure.
151 data = JSONData(template=template)
153 data.update(metadata)
154 data.set_key(u"test_id", tid)
155 t_type = in_data.get(u"type", u"")
156 t_type = u"NDRPDR" if t_type == u"CPS" else t_type # It is NDRPDR
157 data.set_key(u"test_type", t_type)
158 tags = in_data.get(u"tags", list())
159 data.set_key(u"tags", tags)
160 data.set_key(u"documentation", in_data.get(u"documentation", u""))
161 data.set_key(u"message", in_data.get(u"msg", u""))
162 data.set_key(u"start_time", in_data.get(u"starttime", u""))
163 data.set_key(u"end_time", in_data.get(u"endtime", u""))
164 data.set_key(u"status", in_data.get(u"status", u"FAILED"))
170 data.set_key(u"sut_type", sut_type)
172 # Process configuration history:
173 in_papi = deepcopy(in_data.get(u"conf_history", None))
175 regex_dut = re.compile(r'\*\*DUT(\d):\*\*')
177 for line in in_papi.split(u"\n"):
180 groups = re.search(regex_dut, line)
182 node_id = f"dut{groups.group(1)}"
187 u"source_type": u"node",
188 u"source_id": node_id,
189 u"msg_type": u"papi",
190 u"log_level": u"INFO",
191 u"timestamp": in_data.get(u"starttime", u""),
197 # Process show runtime:
198 if in_data.get(u"telemetry-show-run", None):
199 for item in in_data[u"telemetry-show-run"].values():
200 data.add_to_list(u"log", item.get(u"runtime", dict()))
202 in_sh_run = deepcopy(in_data.get(u"show-run", None))
204 # Transform to openMetrics format
205 for key, val in in_sh_run.items():
207 u"source_type": u"node",
209 u"msg_type": u"metric",
210 u"log_level": u"INFO",
211 u"timestamp": in_data.get(u"starttime", u""),
212 u"msg": u"show_runtime",
215 runtime = loads(val.get(u"runtime", list()))
217 for metric, m_data in item.items():
218 if metric == u"name":
220 for idx, m_item in enumerate(m_data):
221 log_item[u"data"].append(
226 u"host": val.get(u"host", u""),
227 u"socket": val.get(u"socket", u""),
228 u"graph_node": item.get(u"name", u""),
229 u"thread_id": str(idx)
233 data.add_to_list(u"log", log_item)
237 if t_type == u"DEVICETEST":
238 pass # Nothing to add.
239 elif t_type == u"NDRPDR":
243 u"cps" if u"TCP_CPS" in tags or u"UDP_CPS" in tags
247 u"lower": in_data.get(u"throughput", dict()).
248 get(u"NDR", dict()).get(u"LOWER", u"NaN"),
249 u"upper": in_data.get(u"throughput", dict()).
250 get(u"NDR", dict()).get(u"UPPER", u"NaN")
253 u"lower": in_data.get(u"gbps", dict()).
254 get(u"NDR", dict()).get(u"LOWER", u"NaN"),
255 u"upper": in_data.get(u"gbps", dict()).
256 get(u"NDR", dict()).get(u"UPPER", u"NaN")
261 u"lower": in_data.get(u"throughput", dict()).
262 get(u"PDR", dict()).get(u"LOWER", u"NaN"),
263 u"upper": in_data.get(u"throughput", dict()).
264 get(u"PDR", dict()).get(u"UPPER", u"NaN")
267 u"lower": in_data.get(u"gbps", dict()).
268 get(u"PDR", dict()).get(u"LOWER", u"NaN"),
269 u"upper": in_data.get(u"gbps", dict()).
270 get(u"PDR", dict()).get(u"UPPER", u"NaN")
276 u"pdr_90": in_data.get(u"latency", dict()).
277 get(u"PDR90", dict()).get(u"direction1", u"NaN"),
278 u"pdr_50": in_data.get(u"latency", dict()).
279 get(u"PDR50", dict()).get(u"direction1", u"NaN"),
280 u"pdr_10": in_data.get(u"latency", dict()).
281 get(u"PDR10", dict()).get(u"direction1", u"NaN"),
282 u"pdr_0": in_data.get(u"latency", dict()).
283 get(u"LAT0", dict()).get(u"direction1", u"NaN")
286 u"pdr_90": in_data.get(u"latency", dict()).
287 get(u"PDR90", dict()).get(u"direction2", u"NaN"),
288 u"pdr_50": in_data.get(u"latency", dict()).
289 get(u"PDR50", dict()).get(u"direction2", u"NaN"),
290 u"pdr_10": in_data.get(u"latency", dict()).
291 get(u"PDR10", dict()).get(u"direction2", u"NaN"),
292 u"pdr_0": in_data.get(u"latency", dict()).
293 get(u"LAT0", dict()).get(u"direction2", u"NaN")
297 elif t_type == "MRR":
299 u"unit": u"pps", # Old data use only pps
300 u"samples": in_data.get(u"result", dict()).get(u"samples", list()),
301 u"avg": in_data.get(u"result", dict()).get(u"receive-rate", u"NaN"),
302 u"stdev": in_data.get(u"result", dict()).
303 get(u"receive-stdev", u"NaN")
305 elif t_type == "SOAK":
308 u"lower": in_data.get(u"throughput", dict()).
309 get(u"LOWER", u"NaN"),
310 u"upper": in_data.get(u"throughput", dict()).
311 get(u"UPPER", u"NaN"),
314 elif t_type == "HOSTSTACK":
315 results = in_data.get(u"result", dict())
316 # elif t_type == "TCP": # Not used ???
317 # results = in_data.get(u"result", u"NaN")
318 elif t_type == "RECONF":
320 u"loss": in_data.get(u"result", dict()).get(u"loss", u"NaN"),
321 u"time": in_data.get(u"result", dict()).get(u"time", u"NaN")
325 data.set_key(u"results", results)
327 data.dump(out, indent=u" ")
330 def convert_xml_to_json(spec, data):
331 """Convert downloaded XML files into JSON.
334 - create one json file for each test,
335 - gzip all json files one by one,
338 :param spec: Specification read from the specification files.
339 :param data: Input data parsed from output.xml files.
340 :type spec: Specification
341 :type data: InputData
344 logging.info(u"Converting downloaded XML files to JSON ...")
346 template_name = spec.output.get(u"use-template", None)
347 structure = spec.output.get(u"structure", u"tree")
349 with open(template_name, u"r") as file_handler:
350 template = json.load(file_handler)
354 build_dir = spec.environment[u"paths"][u"DIR[BUILD,JSON]"]
357 except FileNotFoundError:
358 pass # It does not exist
362 for job, builds in data.data.items():
363 logging.info(f" Processing job {job}")
364 if structure == "tree":
365 os.makedirs(join(build_dir, job), exist_ok=True)
366 for build_nr, build in builds.items():
367 logging.info(f" Processing build {build_nr}")
368 if structure == "tree":
369 os.makedirs(join(build_dir, job, build_nr), exist_ok=True)
370 for test_id, test_data in build[u"tests"].items():
371 groups = re.search(re.compile(r'-(\d+[tT](\d+[cC]))-'), test_id)
373 test_id = test_id.replace(groups.group(1), groups.group(2))
374 logging.info(f" Processing test {test_id}")
375 if structure == "tree":
376 dirs = test_id.split(u".")[:-1]
377 name = test_id.split(u".")[-1]
379 join(build_dir, job, build_nr, *dirs), exist_ok=True
382 f"{join(build_dir, job, build_nr, *dirs, name)}.json"
386 u'.'.join((job, build_nr, test_id, u'json'))
388 suite_id = test_id.rsplit(u".", 1)[0].replace(u" ", u"_")
389 _export_test_from_xml_to_json(
390 test_id, test_data, file_name, template,
392 u"ci": u"jenkins.fd.io",
394 u"build_number": build_nr,
395 u"suite_id": suite_id,
396 u"suite_doc": build[u"suites"].get(suite_id, dict()).
398 u"testbed": build[u"metadata"].get(u"testbed", u""),
399 u"sut_version": build[u"metadata"].get(u"version", u"")
403 # gzip the json files:
404 for file in get_files(build_dir, u"json"):
405 with open(file, u"rb") as src:
406 with gzip.open(f"{file}.gz", u"wb") as dst:
410 logging.info(u"Done.")