1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from yaml import load, YAMLError
22 from pprint import pformat
24 from errors import PresentationError
25 from utils import get_last_successful_build_number
26 from utils import get_last_completed_build_number
29 class Specification(object):
30 """Specification of Presentation and analytics layer.
32 - based on specification specified in the specification YAML file
33 - presentation and analytics layer is model driven
36 # Tags are used in specification YAML file and replaced while the file is
41 def __init__(self, cfg_file):
44 :param cfg_file: File handler for the specification YAML file.
45 :type cfg_file: BinaryIO
47 self._cfg_file = cfg_file
50 self._specification = {"environment": dict(),
51 "configuration": dict(),
61 def specification(self):
62 """Getter - specification.
64 :returns: Specification.
67 return self._specification
70 def environment(self):
71 """Getter - environment.
73 :returns: Environment specification.
76 return self._specification["environment"]
79 def configuration(self):
80 """Getter - configuration.
82 :returns: Configuration of PAL.
85 return self._specification["configuration"]
89 """Getter - static content.
91 :returns: Static content specification.
94 return self._specification["static"]
100 :returns: Mapping of the old names of test cases to the new (actual)
104 return self._specification["configuration"]["mapping"]
108 """Getter - Ignore list.
110 :returns: List of ignored test cases.
113 return self._specification["configuration"]["ignore"]
117 """Getter - Alerting.
119 :returns: Specification of alerts.
122 return self._specification["configuration"]["alerting"]
126 """Getter - specification - inputs.
132 return self._specification["input"]
136 """Getter - builds defined in specification.
138 :returns: Builds defined in the specification.
141 return self.input["builds"]
145 """Getter - specification - output formats and versions to be generated.
147 - versions: full, ...
149 :returns: Outputs to be generated.
152 return self._specification["output"]
156 """Getter - tables to be generated.
158 :returns: List of specifications of tables to be generated.
161 return self._specification["tables"]
165 """Getter - plots to be generated.
167 :returns: List of specifications of plots to be generated.
170 return self._specification["plots"]
174 """Getter - files to be generated.
176 :returns: List of specifications of files to be generated.
179 return self._specification["files"]
183 """Getter - Continuous Performance Trending and Analysis to be
186 :returns: List of specifications of Continuous Performance Trending and
187 Analysis to be generated.
190 return self._specification["cpta"]
192 def set_input_state(self, job, build_nr, state):
193 """Set the state of input
202 for build in self._specification["input"]["builds"][job]:
203 if build["build"] == build_nr:
204 build["status"] = state
207 raise PresentationError("Build '{}' is not defined for job '{}'"
208 " in specification file.".
209 format(build_nr, job))
211 raise PresentationError("Job '{}' and build '{}' is not defined in "
212 "specification file.".format(job, build_nr))
214 def set_input_file_name(self, job, build_nr, file_name):
215 """Set the state of input
224 for build in self._specification["input"]["builds"][job]:
225 if build["build"] == build_nr:
226 build["file-name"] = file_name
229 raise PresentationError("Build '{}' is not defined for job '{}'"
230 " in specification file.".
231 format(build_nr, job))
233 raise PresentationError("Job '{}' and build '{}' is not defined in "
234 "specification file.".format(job, build_nr))
236 def _get_build_number(self, job, build_type):
237 """Get the number of the job defined by its name:
238 - lastSuccessfulBuild
241 :param job: Job name.
242 :param build_type: Build type:
243 - lastSuccessfulBuild
246 :raises PresentationError: If it is not possible to get the build
248 :returns: The build number.
252 # defined as a range <start, end>
253 if build_type == "lastSuccessfulBuild":
254 # defined as a range <start, lastSuccessfulBuild>
255 ret_code, build_nr, _ = get_last_successful_build_number(
256 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
257 elif build_type == "lastCompletedBuild":
258 # defined as a range <start, lastCompletedBuild>
259 ret_code, build_nr, _ = get_last_completed_build_number(
260 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
262 raise PresentationError("Not supported build type: '{0}'".
265 raise PresentationError("Not possible to get the number of the "
268 build_nr = int(build_nr)
270 except ValueError as err:
271 raise PresentationError("Not possible to get the number of the "
272 "build number.\nReason: {0}".format(err))
274 def _get_type_index(self, item_type):
275 """Get index of item type (environment, input, output, ...) in
276 specification YAML file.
278 :param item_type: Item type: Top level items in specification YAML file,
279 e.g.: environment, input, output.
281 :returns: Index of the given item type.
286 for item in self._cfg_yaml:
287 if item["type"] == item_type:
292 def _find_tag(self, text):
293 """Find the first tag in the given text. The tag is enclosed by the
294 TAG_OPENER and TAG_CLOSER.
296 :param text: Text to be searched.
298 :returns: The tag, or None if not found.
302 start = text.index(self.TAG_OPENER)
303 end = text.index(self.TAG_CLOSER, start + 1) + 1
304 return text[start:end]
308 def _replace_tags(self, data, src_data=None):
309 """Replace tag(s) in the data by their values.
311 :param data: The data where the tags will be replaced by their values.
312 :param src_data: Data where the tags are defined. It is dictionary where
313 the key is the tag and the value is the tag value. If not given, 'data'
315 :type data: str or dict
317 :returns: Data with the tags replaced.
319 :raises: PresentationError if it is not possible to replace the tag or
320 the data is not the supported data type (str, dict).
326 if isinstance(data, str):
327 tag = self._find_tag(data)
329 data = data.replace(tag, src_data[tag[1:-1]])
331 elif isinstance(data, dict):
333 for key, value in data.items():
334 tag = self._find_tag(value)
337 data[key] = value.replace(tag, src_data[tag[1:-1]])
340 raise PresentationError("Not possible to replace the "
341 "tag '{}'".format(tag))
343 self._replace_tags(data, src_data)
345 raise PresentationError("Replace tags: Not supported data type.")
349 def _parse_env(self):
350 """Parse environment specification in the specification YAML file.
353 logging.info("Parsing specification file: environment ...")
355 idx = self._get_type_index("environment")
360 self._specification["environment"]["configuration"] = \
361 self._cfg_yaml[idx]["configuration"]
363 self._specification["environment"]["configuration"] = None
366 self._specification["environment"]["paths"] = \
367 self._replace_tags(self._cfg_yaml[idx]["paths"])
369 self._specification["environment"]["paths"] = None
372 self._specification["environment"]["urls"] = \
373 self._cfg_yaml[idx]["urls"]
375 self._specification["environment"]["urls"] = None
378 self._specification["environment"]["make-dirs"] = \
379 self._cfg_yaml[idx]["make-dirs"]
381 self._specification["environment"]["make-dirs"] = None
384 self._specification["environment"]["remove-dirs"] = \
385 self._cfg_yaml[idx]["remove-dirs"]
387 self._specification["environment"]["remove-dirs"] = None
390 self._specification["environment"]["build-dirs"] = \
391 self._cfg_yaml[idx]["build-dirs"]
393 self._specification["environment"]["build-dirs"] = None
396 self._specification["environment"]["testbeds"] = \
397 self._cfg_yaml[idx]["testbeds"]
399 self._specification["environment"]["testbeds"] = None
401 logging.info("Done.")
403 def _parse_configuration(self):
404 """Parse configuration of PAL in the specification YAML file.
407 logging.info("Parsing specification file: configuration ...")
409 idx = self._get_type_index("configuration")
411 logging.warning("No configuration information in the specification "
416 self._specification["configuration"] = self._cfg_yaml[idx]
419 raise PresentationError("No configuration defined.")
421 # Data sets: Replace ranges by lists
422 for set_name, data_set in self.configuration["data-sets"].items():
423 if not isinstance(data_set, dict):
425 for job, builds in data_set.items():
427 if isinstance(builds, dict):
428 build_end = builds.get("end", None)
430 build_end = int(build_end)
432 # defined as a range <start, build_type>
433 build_end = self._get_build_number(job, build_end)
434 builds = [x for x in range(builds["start"], build_end+1)
435 if x not in builds.get("skip", list())]
436 self.configuration["data-sets"][set_name][job] = builds
437 elif isinstance(builds, list):
438 for idx, item in enumerate(builds):
440 builds[idx] = int(item)
442 # defined as a range <build_type>
443 builds[idx] = self._get_build_number(job, item)
445 # Data sets: add sub-sets to sets (only one level):
446 for set_name, data_set in self.configuration["data-sets"].items():
447 if isinstance(data_set, list):
449 for item in data_set:
451 for key, val in self.configuration["data-sets"][item].\
455 raise PresentationError(
456 "Data set {0} is not defined in "
457 "the configuration section.".format(item))
458 self.configuration["data-sets"][set_name] = new_set
462 mapping_file_name = self._specification["configuration"].\
463 get("mapping-file", None)
464 if mapping_file_name:
465 logging.debug("Mapping file: '{0}'".format(mapping_file_name))
467 with open(mapping_file_name, 'r') as mfile:
468 mapping = load(mfile)
469 logging.debug("Loaded mapping table:\n{0}".format(mapping))
470 except (YAMLError, IOError) as err:
471 raise PresentationError(
472 msg="An error occurred while parsing the mapping file "
473 "'{0}'.".format(mapping_file_name),
475 # Make sure everything is lowercase
477 self._specification["configuration"]["mapping"] = \
478 {key.lower(): val.lower() for key, val in mapping.iteritems()}
480 self._specification["configuration"]["mapping"] = dict()
484 ignore_list_name = self._specification["configuration"].\
485 get("ignore-list", None)
487 logging.debug("Ignore list file: '{0}'".format(ignore_list_name))
489 with open(ignore_list_name, 'r') as ifile:
491 logging.debug("Loaded ignore list:\n{0}".format(ignore))
492 except (YAMLError, IOError) as err:
493 raise PresentationError(
494 msg="An error occurred while parsing the ignore list file "
495 "'{0}'.".format(ignore_list_name),
497 # Make sure everything is lowercase
499 self._specification["configuration"]["ignore"] = \
500 [item.lower() for item in ignore]
502 self._specification["configuration"]["ignore"] = list()
504 logging.info("Done.")
506 def _parse_input(self):
507 """Parse input specification in the specification YAML file.
509 :raises: PresentationError if there are no data to process.
512 logging.info("Parsing specification file: input ...")
514 idx = self._get_type_index("input")
516 raise PresentationError("No data to process.")
519 for key, value in self._cfg_yaml[idx]["general"].items():
520 self._specification["input"][key] = value
521 self._specification["input"]["builds"] = dict()
523 for job, builds in self._cfg_yaml[idx]["builds"].items():
525 if isinstance(builds, dict):
526 build_end = builds.get("end", None)
528 build_end = int(build_end)
530 # defined as a range <start, build_type>
531 build_end = self._get_build_number(job, build_end)
532 builds = [x for x in range(builds["start"], build_end+1)
533 if x not in builds.get("skip", list())]
534 self._specification["input"]["builds"][job] = list()
536 self._specification["input"]["builds"][job]. \
537 append({"build": build, "status": None})
540 logging.warning("No build is defined for the job '{}'. "
541 "Trying to continue without it.".
544 raise PresentationError("No data to process.")
546 logging.info("Done.")
548 def _parse_output(self):
549 """Parse output specification in the specification YAML file.
551 :raises: PresentationError if there is no output defined.
554 logging.info("Parsing specification file: output ...")
556 idx = self._get_type_index("output")
558 raise PresentationError("No output defined.")
561 self._specification["output"] = self._cfg_yaml[idx]
562 except (KeyError, IndexError):
563 raise PresentationError("No output defined.")
565 logging.info("Done.")
567 def _parse_static(self):
568 """Parse specification of the static content in the specification YAML
572 logging.info("Parsing specification file: static content ...")
574 idx = self._get_type_index("static")
576 logging.warning("No static content specified.")
578 for key, value in self._cfg_yaml[idx].items():
579 if isinstance(value, str):
581 self._cfg_yaml[idx][key] = self._replace_tags(
582 value, self._specification["environment"]["paths"])
586 self._specification["static"] = self._cfg_yaml[idx]
588 logging.info("Done.")
590 def _parse_elements(self):
591 """Parse elements (tables, plots) specification in the specification
595 logging.info("Parsing specification file: elements ...")
598 for element in self._cfg_yaml:
600 element["output-file"] = self._replace_tags(
601 element["output-file"],
602 self._specification["environment"]["paths"])
607 element["input-file"] = self._replace_tags(
608 element["input-file"],
609 self._specification["environment"]["paths"])
613 # add data sets to the elements:
614 if isinstance(element.get("data", None), str):
615 data_set = element["data"]
617 element["data"] = self.configuration["data-sets"][data_set]
619 raise PresentationError("Data set {0} is not defined in "
620 "the configuration section.".
623 if element["type"] == "table":
624 logging.info(" {:3d} Processing a table ...".format(count))
626 element["template"] = self._replace_tags(
628 self._specification["environment"]["paths"])
634 for item in ("reference", "compare"):
635 if element.get(item, None):
636 data_set = element[item].get("data", None)
637 if isinstance(data_set, str):
638 element[item]["data"] = \
639 self.configuration["data-sets"][data_set]
641 if element.get("history", None):
642 for i in range(len(element["history"])):
643 data_set = element["history"][i].get("data", None)
644 if isinstance(data_set, str):
645 element["history"][i]["data"] = \
646 self.configuration["data-sets"][data_set]
649 raise PresentationError("Wrong data set used in {0}.".
650 format(element.get("title", "")))
652 self._specification["tables"].append(element)
655 elif element["type"] == "plot":
656 logging.info(" {:3d} Processing a plot ...".format(count))
658 # Add layout to the plots:
659 layout = element["layout"].get("layout", None)
660 if layout is not None:
661 element["layout"].pop("layout")
663 for key, val in (self.configuration["plot-layouts"]
665 element["layout"][key] = val
667 raise PresentationError("Layout {0} is not defined in "
668 "the configuration section.".
670 self._specification["plots"].append(element)
673 elif element["type"] == "file":
674 logging.info(" {:3d} Processing a file ...".format(count))
676 element["dir-tables"] = self._replace_tags(
677 element["dir-tables"],
678 self._specification["environment"]["paths"])
681 self._specification["files"].append(element)
684 elif element["type"] == "cpta":
685 logging.info(" {:3d} Processing Continuous Performance "
686 "Trending and Analysis ...".format(count))
688 for plot in element["plots"]:
689 # Add layout to the plots:
690 layout = plot.get("layout", None)
691 if layout is not None:
694 self.configuration["plot-layouts"][layout]
696 raise PresentationError(
697 "Layout {0} is not defined in the "
698 "configuration section.".format(layout))
700 if isinstance(plot.get("data", None), str):
701 data_set = plot["data"]
704 self.configuration["data-sets"][data_set]
706 raise PresentationError(
707 "Data set {0} is not defined in "
708 "the configuration section.".
710 self._specification["cpta"] = element
713 logging.info("Done.")
715 def read_specification(self):
716 """Parse specification in the specification YAML file.
718 :raises: PresentationError if an error occurred while parsing the
722 self._cfg_yaml = load(self._cfg_file)
723 except YAMLError as err:
724 raise PresentationError(msg="An error occurred while parsing the "
725 "specification file.",
729 self._parse_configuration()
733 self._parse_elements()
735 logging.debug("Specification: \n{}".
736 format(pformat(self._specification)))