1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from pprint import pformat
23 from yaml import load, FullLoader, YAMLError
25 from pal_errors import PresentationError
26 from pal_utils import (
27 get_last_successful_build_nr, get_last_completed_build_number
32 """Specification of Presentation and analytics layer.
34 - based on specification specified in the specification YAML file
35 - presentation and analytics layer is model driven
38 # Tags are used in specification YAML file and replaced while the file is
43 def __init__(self, cfg_file):
46 :param cfg_file: File handler for the specification YAML file.
47 :type cfg_file: BinaryIO
49 self._cfg_file = cfg_file
52 self._specification = {
53 u"environment": dict(),
54 u"configuration": dict(),
65 def specification(self):
66 """Getter - specification.
68 :returns: Specification.
71 return self._specification
74 def environment(self):
75 """Getter - environment.
77 :returns: Environment specification.
80 return self._specification[u"environment"]
83 def configuration(self):
84 """Getter - configuration.
86 :returns: Configuration of PAL.
89 return self._specification[u"configuration"]
93 """Getter - static content.
95 :returns: Static content specification.
98 return self._specification[u"static"]
104 :returns: Mapping of the old names of test cases to the new (actual)
108 return self._specification[u"configuration"][u"mapping"]
112 """Getter - Ignore list.
114 :returns: List of ignored test cases.
117 return self._specification[u"configuration"][u"ignore"]
121 """Getter - Alerting.
123 :returns: Specification of alerts.
126 return self._specification[u"configuration"][u"alerting"]
130 """Getter - specification - inputs.
136 return self._specification[u"input"]
139 def input(self, new_value):
140 """Setter - specification - inputs.
142 :param new_value: New value to be set.
143 :type new_value: dict
145 self._specification[u"input"] = new_value
149 """Getter - builds defined in specification.
151 :returns: Builds defined in the specification.
154 return self.input[u"builds"]
157 def builds(self, new_value):
158 """Setter - builds defined in specification.
160 :param new_value: New value to be set.
161 :type new_value: dict
163 self.input[u"builds"] = new_value
165 def add_build(self, job, build):
166 """Add a build to the specification.
168 :param job: The job which run the build.
169 :param build: The build to be added.
173 if self._specification[u"input"][u"builds"].get(job, None) is None:
174 self._specification[u"input"][u"builds"][job] = list()
175 self._specification[u"input"][u"builds"][job].append(build)
179 """Getter - specification - output formats and versions to be generated.
181 - versions: full, ...
183 :returns: Outputs to be generated.
186 return self._specification[u"output"]
190 """Getter - tables to be generated.
192 :returns: List of specifications of tables to be generated.
195 return self._specification[u"tables"]
199 """Getter - plots to be generated.
201 :returns: List of specifications of plots to be generated.
204 return self._specification[u"plots"]
208 """Getter - files to be generated.
210 :returns: List of specifications of files to be generated.
213 return self._specification[u"files"]
217 """Getter - Continuous Performance Trending and Analysis to be
220 :returns: List of specifications of Continuous Performance Trending and
221 Analysis to be generated.
224 return self._specification[u"cpta"]
226 def set_input_state(self, job, build_nr, state):
227 """Set the state of input
229 :param job: Job name.
230 :param build_nr: Build number.
231 :param state: The new input state.
235 :raises: PresentationError if wrong job and/or build is provided.
239 for build in self._specification[u"input"][u"builds"][job]:
240 if build[u"build"] == build_nr:
241 build[u"status"] = state
244 raise PresentationError(
245 f"Build {build_nr} is not defined for job {job} in "
246 f"specification file."
249 raise PresentationError(
250 f"Job {job} and build {build_nr} is not defined in "
251 f"specification file."
254 def set_input_file_name(self, job, build_nr, file_name):
255 """Set the state of input
257 :param job: Job name.
258 :param build_nr: Build number.
259 :param file_name: The new file name.
263 :raises: PresentationError if wrong job and/or build is provided.
267 for build in self._specification[u"input"][u"builds"][job]:
268 if build[u"build"] == build_nr:
269 build[u"file-name"] = file_name
272 raise PresentationError(
273 f"Build {build_nr} is not defined for job {job} in "
274 f"specification file."
277 raise PresentationError(
278 f"Job {job} and build {build_nr} is not defined in "
279 f"specification file."
282 def _get_build_number(self, job, build_type):
283 """Get the number of the job defined by its name:
284 - lastSuccessfulBuild
287 :param job: Job name.
288 :param build_type: Build type:
289 - lastSuccessfulBuild
292 :raises PresentationError: If it is not possible to get the build
294 :returns: The build number.
298 # defined as a range <start, end>
299 if build_type == u"lastSuccessfulBuild":
300 # defined as a range <start, lastSuccessfulBuild>
301 ret_code, build_nr, _ = get_last_successful_build_nr(
302 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
303 elif build_type == u"lastCompletedBuild":
304 # defined as a range <start, lastCompletedBuild>
305 ret_code, build_nr, _ = get_last_completed_build_number(
306 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
308 raise PresentationError(f"Not supported build type: {build_type}")
310 raise PresentationError(u"Not possible to get the number of the "
313 build_nr = int(build_nr)
315 except ValueError as err:
316 raise PresentationError(
317 f"Not possible to get the number of the build number. Reason:\n"
321 def _get_type_index(self, item_type):
322 """Get index of item type (environment, input, output, ...) in
323 specification YAML file.
325 :param item_type: Item type: Top level items in specification YAML file,
326 e.g.: environment, input, output.
328 :returns: Index of the given item type.
333 for item in self._cfg_yaml:
334 if item[u"type"] == item_type:
339 def _find_tag(self, text):
340 """Find the first tag in the given text. The tag is enclosed by the
341 TAG_OPENER and TAG_CLOSER.
343 :param text: Text to be searched.
345 :returns: The tag, or None if not found.
349 start = text.index(self.TAG_OPENER)
350 end = text.index(self.TAG_CLOSER, start + 1) + 1
351 return text[start:end]
355 def _replace_tags(self, data, src_data=None):
356 """Replace tag(s) in the data by their values.
358 :param data: The data where the tags will be replaced by their values.
359 :param src_data: Data where the tags are defined. It is dictionary where
360 the key is the tag and the value is the tag value. If not given,
361 'data' is used instead.
362 :type data: str, list or dict
364 :returns: Data with the tags replaced.
365 :rtype: str, list or dict
366 :raises: PresentationError if it is not possible to replace the tag or
367 the data is not the supported data type (str, list or dict).
373 if isinstance(data, str):
374 tag = self._find_tag(data)
376 data = data.replace(tag, src_data[tag[1:-1]])
379 if isinstance(data, list):
382 new_list.append(self._replace_tags(item, src_data))
385 if isinstance(data, dict):
387 for key, value in data.items():
388 tag = self._find_tag(value)
391 data[key] = value.replace(tag, src_data[tag[1:-1]])
394 raise PresentationError(
395 f"Not possible to replace the tag {tag}"
398 self._replace_tags(data, src_data)
401 raise PresentationError(u"Replace tags: Not supported data type.")
403 def _parse_env(self):
404 """Parse environment specification in the specification YAML file.
407 logging.info(u"Parsing specification file: environment ...")
409 idx = self._get_type_index(u"environment")
414 self._specification[u"environment"][u"configuration"] = \
415 self._cfg_yaml[idx][u"configuration"]
417 self._specification[u"environment"][u"configuration"] = None
420 self._specification[u"environment"][u"paths"] = \
421 self._replace_tags(self._cfg_yaml[idx][u"paths"])
423 self._specification[u"environment"][u"paths"] = None
426 self._specification[u"environment"][u"urls"] = \
427 self._cfg_yaml[idx][u"urls"]
429 self._specification[u"environment"][u"urls"] = None
432 self._specification[u"environment"][u"make-dirs"] = \
433 self._cfg_yaml[idx][u"make-dirs"]
435 self._specification[u"environment"][u"make-dirs"] = None
438 self._specification[u"environment"][u"remove-dirs"] = \
439 self._cfg_yaml[idx][u"remove-dirs"]
441 self._specification[u"environment"][u"remove-dirs"] = None
444 self._specification[u"environment"][u"build-dirs"] = \
445 self._cfg_yaml[idx][u"build-dirs"]
447 self._specification[u"environment"][u"build-dirs"] = None
450 self._specification[u"environment"][u"testbeds"] = \
451 self._cfg_yaml[idx][u"testbeds"]
453 self._specification[u"environment"][u"testbeds"] = None
455 logging.info(u"Done.")
457 def _load_mapping_table(self):
458 """Load a mapping table if it is specified. If not, use empty list.
461 mapping_file_name = self._specification[u"configuration"].\
462 get(u"mapping-file", None)
463 if mapping_file_name:
465 with open(mapping_file_name, u'r') as mfile:
466 mapping = load(mfile, Loader=FullLoader)
467 # Make sure everything is lowercase
468 self._specification[u"configuration"][u"mapping"] = \
469 {key.lower(): val.lower() for key, val in
471 logging.debug(f"Loaded mapping table:\n{mapping}")
472 except (YAMLError, IOError) as err:
473 raise PresentationError(
474 msg=f"An error occurred while parsing the mapping file "
475 f"{mapping_file_name}",
479 self._specification[u"configuration"][u"mapping"] = dict()
481 def _load_ignore_list(self):
482 """Load an ignore list if it is specified. If not, use empty list.
485 ignore_list_name = self._specification[u"configuration"].\
486 get(u"ignore-list", None)
489 with open(ignore_list_name, u'r') as ifile:
490 ignore = load(ifile, Loader=FullLoader)
491 # Make sure everything is lowercase
492 self._specification[u"configuration"][u"ignore"] = \
493 [item.lower() for item in ignore]
494 logging.debug(f"Loaded ignore list:\n{ignore}")
495 except (YAMLError, IOError) as err:
496 raise PresentationError(
497 msg=f"An error occurred while parsing the ignore list file "
498 f"{ignore_list_name}.",
502 self._specification[u"configuration"][u"ignore"] = list()
504 def _parse_configuration(self):
505 """Parse configuration of PAL in the specification YAML file.
508 logging.info(u"Parsing specification file: configuration ...")
510 idx = self._get_type_index("configuration")
513 u"No configuration information in the specification file."
518 self._specification[u"configuration"] = self._cfg_yaml[idx]
520 raise PresentationError(u"No configuration defined.")
522 # Data sets: Replace ranges by lists
523 for set_name, data_set in self.configuration[u"data-sets"].items():
524 if not isinstance(data_set, dict):
526 for job, builds in data_set.items():
529 if isinstance(builds, dict):
530 build_end = builds.get(u"end", None)
531 max_builds = builds.get(u"max-builds", None)
532 reverse = builds.get(u"reverse", False)
534 build_end = int(build_end)
536 # defined as a range <start, build_type>
537 build_end = self._get_build_number(job, build_end)
538 builds = list(range(builds[u"start"], build_end + 1))
539 if max_builds and max_builds < len(builds):
540 builds = builds[-max_builds:]
543 self.configuration[u"data-sets"][set_name][job] = builds
544 elif isinstance(builds, list):
545 for idx, item in enumerate(builds):
547 builds[idx] = int(item)
549 # defined as a range <build_type>
550 builds[idx] = self._get_build_number(job, item)
552 # Data sets: add sub-sets to sets (only one level):
553 for set_name, data_set in self.configuration[u"data-sets"].items():
554 if isinstance(data_set, list):
556 for item in data_set:
558 for key, val in self.configuration[u"data-sets"][item].\
562 raise PresentationError(
563 f"Data set {item} is not defined in "
564 f"the configuration section."
566 self.configuration[u"data-sets"][set_name] = new_set
569 self._load_mapping_table()
572 self._load_ignore_list()
574 logging.info(u"Done.")
576 def _parse_input(self):
577 """Parse input specification in the specification YAML file.
579 :raises: PresentationError if there are no data to process.
582 logging.info(u"Parsing specification file: input ...")
584 idx = self._get_type_index(u"input")
586 raise PresentationError(u"No data to process.")
589 for key, value in self._cfg_yaml[idx][u"general"].items():
590 self._specification[u"input"][key] = value
591 self._specification[u"input"][u"builds"] = dict()
593 for job, builds in self._cfg_yaml[idx][u"builds"].items():
595 if isinstance(builds, dict):
596 build_end = builds.get(u"end", None)
597 max_builds = builds.get(u"max-builds", None)
598 reverse = bool(builds.get(u"reverse", False))
600 build_end = int(build_end)
602 # defined as a range <start, build_type>
603 if build_end in (u"lastCompletedBuild",
604 u"lastSuccessfulBuild"):
606 build_end = self._get_build_number(job, build_end)
607 builds = [x for x in range(builds[u"start"],
609 if x not in builds.get(u"skip", list())]
612 if max_builds and max_builds < len(builds):
613 builds = builds[:max_builds]
614 self._specification[u"input"][u"builds"][job] = list()
616 self._specification[u"input"][u"builds"][job]. \
617 append({u"build": build, u"status": None})
621 f"No build is defined for the job {job}. Trying to "
622 f"continue without it."
626 raise PresentationError(u"No data to process.")
628 logging.info(u"Done.")
630 def _parse_output(self):
631 """Parse output specification in the specification YAML file.
633 :raises: PresentationError if there is no output defined.
636 logging.info(u"Parsing specification file: output ...")
638 idx = self._get_type_index(u"output")
640 raise PresentationError(u"No output defined.")
643 self._specification[u"output"] = self._cfg_yaml[idx]
644 except (KeyError, IndexError):
645 raise PresentationError(u"No output defined.")
647 logging.info(u"Done.")
649 def _parse_static(self):
650 """Parse specification of the static content in the specification YAML
654 logging.info(u"Parsing specification file: static content ...")
656 idx = self._get_type_index(u"static")
658 logging.warning(u"No static content specified.")
660 for key, value in self._cfg_yaml[idx].items():
661 if isinstance(value, str):
663 self._cfg_yaml[idx][key] = self._replace_tags(
664 value, self._specification[u"environment"][u"paths"])
668 self._specification[u"static"] = self._cfg_yaml[idx]
670 logging.info(u"Done.")
672 def _parse_elements_tables(self, table):
673 """Parse tables from the specification YAML file.
675 :param table: Table to be parsed from the specification file.
677 :raises PresentationError: If wrong data set is used.
681 table[u"template"] = self._replace_tags(
683 self._specification[u"environment"][u"paths"])
689 for item in (u"reference", u"compare"):
690 if table.get(item, None):
691 data_set = table[item].get(u"data", None)
692 if isinstance(data_set, str):
693 table[item][u"data"] = \
694 self.configuration[u"data-sets"][data_set]
695 data_set = table[item].get(u"data-replacement", None)
696 if isinstance(data_set, str):
697 table[item][u"data-replacement"] = \
698 self.configuration[u"data-sets"][data_set]
700 if table.get(u"history", None):
701 for i in range(len(table[u"history"])):
702 data_set = table[u"history"][i].get(u"data", None)
703 if isinstance(data_set, str):
704 table[u"history"][i][u"data"] = \
705 self.configuration[u"data-sets"][data_set]
706 data_set = table[u"history"][i].get(
707 u"data-replacement", None)
708 if isinstance(data_set, str):
709 table[u"history"][i][u"data-replacement"] = \
710 self.configuration[u"data-sets"][data_set]
712 if table.get(u"columns", None):
713 for i in range(len(table[u"columns"])):
714 data_set = table[u"columns"][i].get(u"data-set", None)
715 if isinstance(data_set, str):
716 table[u"columns"][i][u"data-set"] = \
717 self.configuration[u"data-sets"][data_set]
718 data_set = table[u"columns"][i].get(
719 u"data-replacement", None)
720 if isinstance(data_set, str):
721 table[u"columns"][i][u"data-replacement"] = \
722 self.configuration[u"data-sets"][data_set]
725 raise PresentationError(
726 f"Wrong data set used in {table.get(u'title', u'')}."
729 self._specification[u"tables"].append(table)
731 def _parse_elements_plots(self, plot):
732 """Parse plots from the specification YAML file.
734 :param plot: Plot to be parsed from the specification file.
736 :raises PresentationError: If plot layout is not defined.
739 # Add layout to the plots:
740 layout = plot[u"layout"].get(u"layout", None)
741 if layout is not None:
742 plot[u"layout"].pop(u"layout")
744 for key, val in (self.configuration[u"plot-layouts"]
746 plot[u"layout"][key] = val
748 raise PresentationError(
749 f"Layout {layout} is not defined in the "
750 f"configuration section."
752 self._specification[u"plots"].append(plot)
754 def _parse_elements_files(self, file):
755 """Parse files from the specification YAML file.
757 :param file: File to be parsed from the specification file.
762 file[u"dir-tables"] = self._replace_tags(
764 self._specification[u"environment"][u"paths"])
767 self._specification[u"files"].append(file)
769 def _parse_elements_cpta(self, cpta):
770 """Parse cpta from the specification YAML file.
772 :param cpta: cpta to be parsed from the specification file.
774 :raises PresentationError: If wrong data set is used or if plot layout
778 for plot in cpta[u"plots"]:
779 # Add layout to the plots:
780 layout = plot.get(u"layout", None)
781 if layout is not None:
784 self.configuration[u"plot-layouts"][layout]
786 raise PresentationError(
787 f"Layout {layout} is not defined in the "
788 f"configuration section."
791 if isinstance(plot.get(u"data", None), str):
792 data_set = plot[u"data"]
795 self.configuration[u"data-sets"][data_set]
797 raise PresentationError(
798 f"Data set {data_set} is not defined in "
799 f"the configuration section."
801 self._specification[u"cpta"] = cpta
803 def _parse_elements(self):
804 """Parse elements (tables, plots, ..) specification in the specification
808 logging.info(u"Parsing specification file: elements ...")
811 for element in self._cfg_yaml:
815 element[u"output-file"] = self._replace_tags(
816 element[u"output-file"],
817 self._specification[u"environment"][u"paths"])
822 element[u"input-file"] = self._replace_tags(
823 element[u"input-file"],
824 self._specification[u"environment"][u"paths"])
829 element[u"output-file-links"] = self._replace_tags(
830 element[u"output-file-links"],
831 self._specification[u"environment"][u"paths"])
835 # Add data sets to the elements:
836 if isinstance(element.get(u"data", None), str):
837 data_set = element[u"data"]
840 self.configuration[u"data-sets"][data_set]
842 raise PresentationError(
843 f"Data set {data_set} is not defined in the "
844 f"configuration section."
846 elif isinstance(element.get(u"data", None), list):
848 for item in element[u"data"]:
851 self.configuration[u"data-sets"][item]
854 raise PresentationError(
855 f"Data set {item} is not defined in the "
856 f"configuration section."
858 element[u"data"] = new_list
861 if element[u"type"] == u"table":
863 logging.info(f" {count:3d} Processing a table ...")
864 self._parse_elements_tables(element)
867 elif element[u"type"] == u"plot":
869 logging.info(f" {count:3d} Processing a plot ...")
870 self._parse_elements_plots(element)
873 elif element[u"type"] == u"file":
875 logging.info(f" {count:3d} Processing a file ...")
876 self._parse_elements_files(element)
879 elif element[u"type"] == u"cpta":
882 f" {count:3d} Processing Continuous Performance Trending "
885 self._parse_elements_cpta(element)
888 logging.info(u"Done.")
890 def read_specification(self):
891 """Parse specification in the specification YAML file.
893 :raises: PresentationError if an error occurred while parsing the
897 self._cfg_yaml = load(self._cfg_file, Loader=FullLoader)
898 except YAMLError as err:
899 raise PresentationError(msg=u"An error occurred while parsing the "
900 u"specification file.",
904 self._parse_configuration()
908 self._parse_elements()
910 logging.debug(f"Specification: \n{pformat(self._specification)}")