1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from yaml import load, YAMLError
22 from pprint import pformat
24 from errors import PresentationError
26 get_last_successful_build_number, get_last_completed_build_number)
30 """Specification of Presentation and analytics layer.
32 - based on specification specified in the specification YAML file
33 - presentation and analytics layer is model driven
36 # Tags are used in specification YAML file and replaced while the file is
41 def __init__(self, cfg_file):
44 :param cfg_file: File handler for the specification YAML file.
45 :type cfg_file: BinaryIO
47 self._cfg_file = cfg_file
50 self._specification = {"environment": dict(),
51 "configuration": dict(),
61 def specification(self):
62 """Getter - specification.
64 :returns: Specification.
67 return self._specification
70 def environment(self):
71 """Getter - environment.
73 :returns: Environment specification.
76 return self._specification["environment"]
79 def configuration(self):
80 """Getter - configuration.
82 :returns: Configuration of PAL.
85 return self._specification["configuration"]
89 """Getter - static content.
91 :returns: Static content specification.
94 return self._specification["static"]
100 :returns: Mapping of the old names of test cases to the new (actual)
104 return self._specification["configuration"]["mapping"]
108 """Getter - Ignore list.
110 :returns: List of ignored test cases.
113 return self._specification["configuration"]["ignore"]
117 """Getter - Alerting.
119 :returns: Specification of alerts.
122 return self._specification["configuration"]["alerting"]
126 """Getter - specification - inputs.
132 return self._specification["input"]
136 """Getter - builds defined in specification.
138 :returns: Builds defined in the specification.
141 return self.input["builds"]
145 """Getter - specification - output formats and versions to be generated.
147 - versions: full, ...
149 :returns: Outputs to be generated.
152 return self._specification["output"]
156 """Getter - tables to be generated.
158 :returns: List of specifications of tables to be generated.
161 return self._specification["tables"]
165 """Getter - plots to be generated.
167 :returns: List of specifications of plots to be generated.
170 return self._specification["plots"]
174 """Getter - files to be generated.
176 :returns: List of specifications of files to be generated.
179 return self._specification["files"]
183 """Getter - Continuous Performance Trending and Analysis to be
186 :returns: List of specifications of Continuous Performance Trending and
187 Analysis to be generated.
190 return self._specification["cpta"]
192 def set_input_state(self, job, build_nr, state):
193 """Set the state of input
202 for build in self._specification["input"]["builds"][job]:
203 if build["build"] == build_nr:
204 build["status"] = state
207 raise PresentationError("Build '{}' is not defined for job '{}'"
208 " in specification file.".
209 format(build_nr, job))
211 raise PresentationError("Job '{}' and build '{}' is not defined in "
212 "specification file.".format(job, build_nr))
214 def set_input_file_name(self, job, build_nr, file_name):
215 """Set the state of input
224 for build in self._specification["input"]["builds"][job]:
225 if build["build"] == build_nr:
226 build["file-name"] = file_name
229 raise PresentationError("Build '{}' is not defined for job '{}'"
230 " in specification file.".
231 format(build_nr, job))
233 raise PresentationError("Job '{}' and build '{}' is not defined in "
234 "specification file.".format(job, build_nr))
236 def _get_build_number(self, job, build_type):
237 """Get the number of the job defined by its name:
238 - lastSuccessfulBuild
241 :param job: Job name.
242 :param build_type: Build type:
243 - lastSuccessfulBuild
246 :raises PresentationError: If it is not possible to get the build
248 :returns: The build number.
252 # defined as a range <start, end>
253 if build_type == "lastSuccessfulBuild":
254 # defined as a range <start, lastSuccessfulBuild>
255 ret_code, build_nr, _ = get_last_successful_build_number(
256 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
257 elif build_type == "lastCompletedBuild":
258 # defined as a range <start, lastCompletedBuild>
259 ret_code, build_nr, _ = get_last_completed_build_number(
260 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
262 raise PresentationError("Not supported build type: '{0}'".
265 raise PresentationError("Not possible to get the number of the "
268 build_nr = int(build_nr)
270 except ValueError as err:
271 raise PresentationError("Not possible to get the number of the "
272 "build number.\nReason: {0}".format(err))
274 def _get_type_index(self, item_type):
275 """Get index of item type (environment, input, output, ...) in
276 specification YAML file.
278 :param item_type: Item type: Top level items in specification YAML file,
279 e.g.: environment, input, output.
281 :returns: Index of the given item type.
286 for item in self._cfg_yaml:
287 if item["type"] == item_type:
292 def _find_tag(self, text):
293 """Find the first tag in the given text. The tag is enclosed by the
294 TAG_OPENER and TAG_CLOSER.
296 :param text: Text to be searched.
298 :returns: The tag, or None if not found.
302 start = text.index(self.TAG_OPENER)
303 end = text.index(self.TAG_CLOSER, start + 1) + 1
304 return text[start:end]
308 def _replace_tags(self, data, src_data=None):
309 """Replace tag(s) in the data by their values.
311 :param data: The data where the tags will be replaced by their values.
312 :param src_data: Data where the tags are defined. It is dictionary where
313 the key is the tag and the value is the tag value. If not given, 'data'
315 :type data: str or dict
317 :returns: Data with the tags replaced.
319 :raises: PresentationError if it is not possible to replace the tag or
320 the data is not the supported data type (str, dict).
326 if isinstance(data, str):
327 tag = self._find_tag(data)
329 data = data.replace(tag, src_data[tag[1:-1]])
331 elif isinstance(data, dict):
333 for key, value in data.items():
334 tag = self._find_tag(value)
337 data[key] = value.replace(tag, src_data[tag[1:-1]])
340 raise PresentationError("Not possible to replace the "
341 "tag '{}'".format(tag))
343 self._replace_tags(data, src_data)
345 raise PresentationError("Replace tags: Not supported data type.")
349 def _parse_env(self):
350 """Parse environment specification in the specification YAML file.
353 logging.info("Parsing specification file: environment ...")
355 idx = self._get_type_index("environment")
360 self._specification["environment"]["configuration"] = \
361 self._cfg_yaml[idx]["configuration"]
363 self._specification["environment"]["configuration"] = None
366 self._specification["environment"]["paths"] = \
367 self._replace_tags(self._cfg_yaml[idx]["paths"])
369 self._specification["environment"]["paths"] = None
372 self._specification["environment"]["urls"] = \
373 self._cfg_yaml[idx]["urls"]
375 self._specification["environment"]["urls"] = None
378 self._specification["environment"]["make-dirs"] = \
379 self._cfg_yaml[idx]["make-dirs"]
381 self._specification["environment"]["make-dirs"] = None
384 self._specification["environment"]["remove-dirs"] = \
385 self._cfg_yaml[idx]["remove-dirs"]
387 self._specification["environment"]["remove-dirs"] = None
390 self._specification["environment"]["build-dirs"] = \
391 self._cfg_yaml[idx]["build-dirs"]
393 self._specification["environment"]["build-dirs"] = None
396 self._specification["environment"]["testbeds"] = \
397 self._cfg_yaml[idx]["testbeds"]
399 self._specification["environment"]["testbeds"] = None
401 logging.info("Done.")
403 def _parse_configuration(self):
404 """Parse configuration of PAL in the specification YAML file.
407 logging.info("Parsing specification file: configuration ...")
409 idx = self._get_type_index("configuration")
411 logging.warning("No configuration information in the specification "
416 self._specification["configuration"] = self._cfg_yaml[idx]
419 raise PresentationError("No configuration defined.")
421 # Data sets: Replace ranges by lists
422 for set_name, data_set in self.configuration["data-sets"].items():
423 if not isinstance(data_set, dict):
425 for job, builds in data_set.items():
427 if isinstance(builds, dict):
428 build_end = builds.get("end", None)
430 build_end = int(build_end)
432 # defined as a range <start, build_type>
433 build_end = self._get_build_number(job, build_end)
434 builds = [x for x in range(builds["start"], build_end+1)
435 if x not in builds.get("skip", list())]
436 self.configuration["data-sets"][set_name][job] = builds
437 elif isinstance(builds, list):
438 for idx, item in enumerate(builds):
440 builds[idx] = int(item)
442 # defined as a range <build_type>
443 builds[idx] = self._get_build_number(job, item)
445 # Data sets: add sub-sets to sets (only one level):
446 for set_name, data_set in self.configuration["data-sets"].items():
447 if isinstance(data_set, list):
449 for item in data_set:
451 for key, val in self.configuration["data-sets"][item].\
455 raise PresentationError(
456 "Data set {0} is not defined in "
457 "the configuration section.".format(item))
458 self.configuration["data-sets"][set_name] = new_set
462 mapping_file_name = self._specification["configuration"].\
463 get("mapping-file", None)
464 if mapping_file_name:
465 logging.debug("Mapping file: '{0}'".format(mapping_file_name))
467 with open(mapping_file_name, 'r') as mfile:
468 mapping = load(mfile)
469 logging.debug("Loaded mapping table:\n{0}".format(mapping))
470 except (YAMLError, IOError) as err:
471 raise PresentationError(
472 msg="An error occurred while parsing the mapping file "
473 "'{0}'.".format(mapping_file_name),
475 # Make sure everything is lowercase
477 self._specification["configuration"]["mapping"] = \
478 {key.lower(): val.lower() for key, val in mapping.iteritems()}
480 self._specification["configuration"]["mapping"] = dict()
484 ignore_list_name = self._specification["configuration"].\
485 get("ignore-list", None)
487 logging.debug("Ignore list file: '{0}'".format(ignore_list_name))
489 with open(ignore_list_name, 'r') as ifile:
491 logging.debug("Loaded ignore list:\n{0}".format(ignore))
492 except (YAMLError, IOError) as err:
493 raise PresentationError(
494 msg="An error occurred while parsing the ignore list file "
495 "'{0}'.".format(ignore_list_name),
497 # Make sure everything is lowercase
499 self._specification["configuration"]["ignore"] = \
500 [item.lower() for item in ignore]
502 self._specification["configuration"]["ignore"] = list()
504 logging.info("Done.")
506 def _parse_input(self):
507 """Parse input specification in the specification YAML file.
509 :raises: PresentationError if there are no data to process.
512 logging.info("Parsing specification file: input ...")
514 idx = self._get_type_index("input")
516 raise PresentationError("No data to process.")
519 for key, value in self._cfg_yaml[idx]["general"].items():
520 self._specification["input"][key] = value
521 self._specification["input"]["builds"] = dict()
523 for job, builds in self._cfg_yaml[idx]["builds"].items():
525 if isinstance(builds, dict):
526 build_end = builds.get("end", None)
528 build_end = int(build_end)
530 # defined as a range <start, build_type>
531 build_end = self._get_build_number(job, build_end)
532 builds = [x for x in range(builds["start"], build_end+1)
533 if x not in builds.get("skip", list())]
534 self._specification["input"]["builds"][job] = list()
536 self._specification["input"]["builds"][job]. \
537 append({"build": build, "status": None})
540 logging.warning("No build is defined for the job '{}'. "
541 "Trying to continue without it.".
544 raise PresentationError("No data to process.")
546 logging.info("Done.")
548 def _parse_output(self):
549 """Parse output specification in the specification YAML file.
551 :raises: PresentationError if there is no output defined.
554 logging.info("Parsing specification file: output ...")
556 idx = self._get_type_index("output")
558 raise PresentationError("No output defined.")
561 self._specification["output"] = self._cfg_yaml[idx]
562 except (KeyError, IndexError):
563 raise PresentationError("No output defined.")
565 logging.info("Done.")
567 def _parse_static(self):
568 """Parse specification of the static content in the specification YAML
572 logging.info("Parsing specification file: static content ...")
574 idx = self._get_type_index("static")
576 logging.warning("No static content specified.")
578 for key, value in self._cfg_yaml[idx].items():
579 if isinstance(value, str):
581 self._cfg_yaml[idx][key] = self._replace_tags(
582 value, self._specification["environment"]["paths"])
586 self._specification["static"] = self._cfg_yaml[idx]
588 logging.info("Done.")
590 def _parse_elements(self):
591 """Parse elements (tables, plots) specification in the specification
595 logging.info("Parsing specification file: elements ...")
598 for element in self._cfg_yaml:
600 element["output-file"] = self._replace_tags(
601 element["output-file"],
602 self._specification["environment"]["paths"])
607 element["input-file"] = self._replace_tags(
608 element["input-file"],
609 self._specification["environment"]["paths"])
613 # add data sets to the elements:
614 if isinstance(element.get("data", None), str):
615 data_set = element["data"]
617 element["data"] = self.configuration["data-sets"][data_set]
619 raise PresentationError("Data set {0} is not defined in "
620 "the configuration section.".
623 if element["type"] == "table":
624 logging.info(" {:3d} Processing a table ...".format(count))
626 element["template"] = self._replace_tags(
628 self._specification["environment"]["paths"])
634 for item in ("reference", "compare"):
635 if element.get(item, None):
636 data_set = element[item].get("data", None)
637 if isinstance(data_set, str):
638 element[item]["data"] = \
639 self.configuration["data-sets"][data_set]
640 data_set = element[item].get("data-replacement",
642 if isinstance(data_set, str):
643 element[item]["data-replacement"] = \
644 self.configuration["data-sets"][data_set]
646 if element.get("history", None):
647 for i in range(len(element["history"])):
648 data_set = element["history"][i].get("data", None)
649 if isinstance(data_set, str):
650 element["history"][i]["data"] = \
651 self.configuration["data-sets"][data_set]
652 data_set = element["history"][i].get(
653 "data-replacement", None)
654 if isinstance(data_set, str):
655 element["history"][i]["data-replacement"] = \
656 self.configuration["data-sets"][data_set]
659 raise PresentationError("Wrong data set used in {0}.".
660 format(element.get("title", "")))
662 self._specification["tables"].append(element)
665 elif element["type"] == "plot":
666 logging.info(" {:3d} Processing a plot ...".format(count))
668 # Add layout to the plots:
669 layout = element["layout"].get("layout", None)
670 if layout is not None:
671 element["layout"].pop("layout")
673 for key, val in (self.configuration["plot-layouts"]
675 element["layout"][key] = val
677 raise PresentationError("Layout {0} is not defined in "
678 "the configuration section.".
680 self._specification["plots"].append(element)
683 elif element["type"] == "file":
684 logging.info(" {:3d} Processing a file ...".format(count))
686 element["dir-tables"] = self._replace_tags(
687 element["dir-tables"],
688 self._specification["environment"]["paths"])
691 self._specification["files"].append(element)
694 elif element["type"] == "cpta":
695 logging.info(" {:3d} Processing Continuous Performance "
696 "Trending and Analysis ...".format(count))
698 for plot in element["plots"]:
699 # Add layout to the plots:
700 layout = plot.get("layout", None)
701 if layout is not None:
704 self.configuration["plot-layouts"][layout]
706 raise PresentationError(
707 "Layout {0} is not defined in the "
708 "configuration section.".format(layout))
710 if isinstance(plot.get("data", None), str):
711 data_set = plot["data"]
714 self.configuration["data-sets"][data_set]
716 raise PresentationError(
717 "Data set {0} is not defined in "
718 "the configuration section.".
720 self._specification["cpta"] = element
723 logging.info("Done.")
725 def read_specification(self):
726 """Parse specification in the specification YAML file.
728 :raises: PresentationError if an error occurred while parsing the
732 self._cfg_yaml = load(self._cfg_file)
733 except YAMLError as err:
734 raise PresentationError(msg="An error occurred while parsing the "
735 "specification file.",
739 self._parse_configuration()
743 self._parse_elements()
745 logging.debug("Specification: \n{}".
746 format(pformat(self._specification)))