1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from pprint import pformat
23 from yaml import load, FullLoader, YAMLError
25 from pal_errors import PresentationError
26 from pal_utils import (
27 get_last_successful_build_nr, get_last_completed_build_number
32 """Specification of Presentation and analytics layer.
34 - based on specification specified in the specification YAML file
35 - presentation and analytics layer is model driven
38 # Tags are used in specification YAML file and replaced while the file is
43 def __init__(self, cfg_file):
46 :param cfg_file: File handler for the specification YAML file.
47 :type cfg_file: BinaryIO
49 self._cfg_file = cfg_file
52 self._specification = {
53 u"environment": dict(),
54 u"configuration": dict(),
65 def specification(self):
66 """Getter - specification.
68 :returns: Specification.
71 return self._specification
74 def environment(self):
75 """Getter - environment.
77 :returns: Environment specification.
80 return self._specification[u"environment"]
83 def configuration(self):
84 """Getter - configuration.
86 :returns: Configuration of PAL.
89 return self._specification[u"configuration"]
93 """Getter - static content.
95 :returns: Static content specification.
98 return self._specification[u"static"]
104 :returns: Mapping of the old names of test cases to the new (actual)
108 return self._specification[u"configuration"][u"mapping"]
112 """Getter - Ignore list.
114 :returns: List of ignored test cases.
117 return self._specification[u"configuration"][u"ignore"]
121 """Getter - Alerting.
123 :returns: Specification of alerts.
126 return self._specification[u"configuration"][u"alerting"]
130 """Getter - specification - inputs.
136 return self._specification[u"input"]
139 def input(self, new_value):
140 """Setter - specification - inputs.
142 :param new_value: New value to be set.
143 :type new_value: dict
145 self._specification[u"input"] = new_value
149 """Getter - builds defined in specification.
151 :returns: Builds defined in the specification.
154 return self.input[u"builds"]
157 def builds(self, new_value):
158 """Setter - builds defined in specification.
160 :param new_value: New value to be set.
161 :type new_value: dict
163 self.input[u"builds"] = new_value
165 def add_build(self, job, build):
166 """Add a build to the specification.
168 :param job: The job which run the build.
169 :param build: The build to be added.
173 if self._specification[u"input"][u"builds"].get(job, None) is None:
174 self._specification[u"input"][u"builds"][job] = list()
175 self._specification[u"input"][u"builds"][job].append(build)
179 """Getter - specification - output formats and versions to be generated.
181 - versions: full, ...
183 :returns: Outputs to be generated.
186 return self._specification[u"output"]
190 """Getter - tables to be generated.
192 :returns: List of specifications of tables to be generated.
195 return self._specification[u"tables"]
199 """Getter - plots to be generated.
201 :returns: List of specifications of plots to be generated.
204 return self._specification[u"plots"]
208 """Getter - files to be generated.
210 :returns: List of specifications of files to be generated.
213 return self._specification[u"files"]
217 """Getter - Continuous Performance Trending and Analysis to be
220 :returns: List of specifications of Continuous Performance Trending and
221 Analysis to be generated.
224 return self._specification[u"cpta"]
226 def set_input_state(self, job, build_nr, state):
227 """Set the state of input
229 :param job: Job name.
230 :param build_nr: Build number.
231 :param state: The new input state.
235 :raises: PresentationError if wrong job and/or build is provided.
239 for build in self._specification[u"input"][u"builds"][job]:
240 if build[u"build"] == build_nr:
241 build[u"status"] = state
244 raise PresentationError(
245 f"Build {build_nr} is not defined for job {job} in "
246 f"specification file."
249 raise PresentationError(
250 f"Job {job} and build {build_nr} is not defined in "
251 f"specification file."
254 def set_input_file_name(self, job, build_nr, file_name):
255 """Set the state of input
257 :param job: Job name.
258 :param build_nr: Build number.
259 :param file_name: The new file name.
263 :raises: PresentationError if wrong job and/or build is provided.
267 for build in self._specification[u"input"][u"builds"][job]:
268 if build[u"build"] == build_nr:
269 build[u"file-name"] = file_name
272 raise PresentationError(
273 f"Build {build_nr} is not defined for job {job} in "
274 f"specification file."
277 raise PresentationError(
278 f"Job {job} and build {build_nr} is not defined in "
279 f"specification file."
282 def _get_build_number(self, job, build_type):
283 """Get the number of the job defined by its name:
284 - lastSuccessfulBuild
287 :param job: Job name.
288 :param build_type: Build type:
289 - lastSuccessfulBuild
292 :raises PresentationError: If it is not possible to get the build
294 :returns: The build number.
298 # defined as a range <start, end>
299 if build_type == u"lastSuccessfulBuild":
300 # defined as a range <start, lastSuccessfulBuild>
301 ret_code, build_nr, _ = get_last_successful_build_nr(
302 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
303 elif build_type == u"lastCompletedBuild":
304 # defined as a range <start, lastCompletedBuild>
305 ret_code, build_nr, _ = get_last_completed_build_number(
306 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
308 raise PresentationError(f"Not supported build type: {build_type}")
310 raise PresentationError(
311 f"Not possible to get the build number of {job}."
314 build_nr = int(build_nr)
316 except ValueError as err:
317 raise PresentationError(
318 f"Not possible to get the build number of {job}. Reason:\n"
322 def _get_type_index(self, item_type):
323 """Get index of item type (environment, input, output, ...) in
324 specification YAML file.
326 :param item_type: Item type: Top level items in specification YAML file,
327 e.g.: environment, input, output.
329 :returns: Index of the given item type.
334 for item in self._cfg_yaml:
335 if item[u"type"] == item_type:
340 def _find_tag(self, text):
341 """Find the first tag in the given text. The tag is enclosed by the
342 TAG_OPENER and TAG_CLOSER.
344 :param text: Text to be searched.
346 :returns: The tag, or None if not found.
350 start = text.index(self.TAG_OPENER)
351 end = text.index(self.TAG_CLOSER, start + 1) + 1
352 return text[start:end]
356 def _replace_tags(self, data, src_data=None):
357 """Replace tag(s) in the data by their values.
359 :param data: The data where the tags will be replaced by their values.
360 :param src_data: Data where the tags are defined. It is dictionary where
361 the key is the tag and the value is the tag value. If not given,
362 'data' is used instead.
363 :type data: str, list or dict
365 :returns: Data with the tags replaced.
366 :rtype: str, list or dict
367 :raises: PresentationError if it is not possible to replace the tag or
368 the data is not the supported data type (str, list or dict).
374 if isinstance(data, str):
375 tag = self._find_tag(data)
377 data = data.replace(tag, src_data[tag[1:-1]])
380 if isinstance(data, list):
383 new_list.append(self._replace_tags(item, src_data))
386 if isinstance(data, dict):
388 for key, value in data.items():
389 tag = self._find_tag(value)
392 data[key] = value.replace(tag, src_data[tag[1:-1]])
395 raise PresentationError(
396 f"Not possible to replace the tag {tag}"
399 self._replace_tags(data, src_data)
402 raise PresentationError(u"Replace tags: Not supported data type.")
404 def _parse_env(self):
405 """Parse environment specification in the specification YAML file.
408 logging.info(u"Parsing specification file: environment ...")
410 idx = self._get_type_index(u"environment")
415 self._specification[u"environment"][u"configuration"] = \
416 self._cfg_yaml[idx][u"configuration"]
418 self._specification[u"environment"][u"configuration"] = None
421 self._specification[u"environment"][u"paths"] = \
422 self._replace_tags(self._cfg_yaml[idx][u"paths"])
424 self._specification[u"environment"][u"paths"] = None
427 self._specification[u"environment"][u"urls"] = \
428 self._cfg_yaml[idx][u"urls"]
430 self._specification[u"environment"][u"urls"] = None
433 self._specification[u"environment"][u"make-dirs"] = \
434 self._cfg_yaml[idx][u"make-dirs"]
436 self._specification[u"environment"][u"make-dirs"] = None
439 self._specification[u"environment"][u"remove-dirs"] = \
440 self._cfg_yaml[idx][u"remove-dirs"]
442 self._specification[u"environment"][u"remove-dirs"] = None
445 self._specification[u"environment"][u"build-dirs"] = \
446 self._cfg_yaml[idx][u"build-dirs"]
448 self._specification[u"environment"][u"build-dirs"] = None
451 self._specification[u"environment"][u"testbeds"] = \
452 self._cfg_yaml[idx][u"testbeds"]
454 self._specification[u"environment"][u"testbeds"] = None
456 logging.info(u"Done.")
458 def _load_mapping_table(self):
459 """Load a mapping table if it is specified. If not, use empty list.
462 mapping_file_name = self._specification[u"configuration"].\
463 get(u"mapping-file", None)
464 if mapping_file_name:
466 with open(mapping_file_name, u'r') as mfile:
467 mapping = load(mfile, Loader=FullLoader)
468 # Make sure everything is lowercase
469 self._specification[u"configuration"][u"mapping"] = \
470 {key.lower(): val.lower() for key, val in
472 logging.debug(f"Loaded mapping table:\n{mapping}")
473 except (YAMLError, IOError) as err:
474 raise PresentationError(
475 msg=f"An error occurred while parsing the mapping file "
476 f"{mapping_file_name}",
480 self._specification[u"configuration"][u"mapping"] = dict()
482 def _load_ignore_list(self):
483 """Load an ignore list if it is specified. If not, use empty list.
486 ignore_list_name = self._specification[u"configuration"].\
487 get(u"ignore-list", None)
490 with open(ignore_list_name, u'r') as ifile:
491 ignore = load(ifile, Loader=FullLoader)
492 # Make sure everything is lowercase
493 self._specification[u"configuration"][u"ignore"] = \
494 [item.lower() for item in ignore]
495 logging.debug(f"Loaded ignore list:\n{ignore}")
496 except (YAMLError, IOError) as err:
497 raise PresentationError(
498 msg=f"An error occurred while parsing the ignore list file "
499 f"{ignore_list_name}.",
503 self._specification[u"configuration"][u"ignore"] = list()
505 def _parse_configuration(self):
506 """Parse configuration of PAL in the specification YAML file.
509 logging.info(u"Parsing specification file: configuration ...")
511 idx = self._get_type_index("configuration")
514 u"No configuration information in the specification file."
519 self._specification[u"configuration"] = self._cfg_yaml[idx]
521 raise PresentationError(u"No configuration defined.")
523 # Data sets: Replace ranges by lists
524 for set_name, data_set in self.configuration[u"data-sets"].items():
525 if not isinstance(data_set, dict):
527 for job, builds in data_set.items():
530 if isinstance(builds, dict):
531 build_end = builds.get(u"end", None)
532 max_builds = builds.get(u"max-builds", None)
533 reverse = builds.get(u"reverse", False)
535 build_end = int(build_end)
537 # defined as a range <start, build_type>
538 build_end = self._get_build_number(job, build_end)
539 builds = list(range(builds[u"start"], build_end + 1))
540 if max_builds and max_builds < len(builds):
541 builds = builds[-max_builds:]
544 self.configuration[u"data-sets"][set_name][job] = builds
545 elif isinstance(builds, list):
546 for idx, item in enumerate(builds):
548 builds[idx] = int(item)
550 # defined as a range <build_type>
551 builds[idx] = self._get_build_number(job, item)
553 # Data sets: add sub-sets to sets (only one level):
554 for set_name, data_set in self.configuration[u"data-sets"].items():
555 if isinstance(data_set, list):
557 for item in data_set:
559 for key, val in self.configuration[u"data-sets"][item].\
563 raise PresentationError(
564 f"Data set {item} is not defined in "
565 f"the configuration section."
567 self.configuration[u"data-sets"][set_name] = new_set
570 self._load_mapping_table()
573 self._load_ignore_list()
575 logging.info(u"Done.")
577 def _parse_input(self):
578 """Parse input specification in the specification YAML file.
580 :raises: PresentationError if there are no data to process.
583 logging.info(u"Parsing specification file: input ...")
585 idx = self._get_type_index(u"input")
587 raise PresentationError(u"No data to process.")
590 for key, value in self._cfg_yaml[idx][u"general"].items():
591 self._specification[u"input"][key] = value
592 self._specification[u"input"][u"builds"] = dict()
594 for job, builds in self._cfg_yaml[idx][u"builds"].items():
596 if isinstance(builds, dict):
597 build_end = builds.get(u"end", None)
598 max_builds = builds.get(u"max-builds", None)
599 reverse = bool(builds.get(u"reverse", False))
601 build_end = int(build_end)
603 # defined as a range <start, build_type>
604 if build_end in (u"lastCompletedBuild",
605 u"lastSuccessfulBuild"):
607 build_end = self._get_build_number(job, build_end)
608 builds = [x for x in range(builds[u"start"],
610 if x not in builds.get(u"skip", list())]
613 if max_builds and max_builds < len(builds):
614 builds = builds[:max_builds]
615 self._specification[u"input"][u"builds"][job] = list()
617 self._specification[u"input"][u"builds"][job]. \
618 append({u"build": build, u"status": None})
622 f"No build is defined for the job {job}. Trying to "
623 f"continue without it."
627 raise PresentationError(u"No data to process.")
629 logging.info(u"Done.")
631 def _parse_output(self):
632 """Parse output specification in the specification YAML file.
634 :raises: PresentationError if there is no output defined.
637 logging.info(u"Parsing specification file: output ...")
639 idx = self._get_type_index(u"output")
641 raise PresentationError(u"No output defined.")
644 self._specification[u"output"] = self._cfg_yaml[idx]
645 except (KeyError, IndexError):
646 raise PresentationError(u"No output defined.")
648 logging.info(u"Done.")
650 def _parse_static(self):
651 """Parse specification of the static content in the specification YAML
655 logging.info(u"Parsing specification file: static content ...")
657 idx = self._get_type_index(u"static")
659 logging.warning(u"No static content specified.")
661 for key, value in self._cfg_yaml[idx].items():
662 if isinstance(value, str):
664 self._cfg_yaml[idx][key] = self._replace_tags(
665 value, self._specification[u"environment"][u"paths"])
669 self._specification[u"static"] = self._cfg_yaml[idx]
671 logging.info(u"Done.")
673 def _parse_elements_tables(self, table):
674 """Parse tables from the specification YAML file.
676 :param table: Table to be parsed from the specification file.
678 :raises PresentationError: If wrong data set is used.
682 table[u"template"] = self._replace_tags(
684 self._specification[u"environment"][u"paths"])
690 for item in (u"reference", u"compare"):
691 if table.get(item, None):
692 data_set = table[item].get(u"data", None)
693 if isinstance(data_set, str):
694 table[item][u"data"] = \
695 self.configuration[u"data-sets"][data_set]
696 data_set = table[item].get(u"data-replacement", None)
697 if isinstance(data_set, str):
698 table[item][u"data-replacement"] = \
699 self.configuration[u"data-sets"][data_set]
701 if table.get(u"history", None):
702 for i in range(len(table[u"history"])):
703 data_set = table[u"history"][i].get(u"data", None)
704 if isinstance(data_set, str):
705 table[u"history"][i][u"data"] = \
706 self.configuration[u"data-sets"][data_set]
707 data_set = table[u"history"][i].get(
708 u"data-replacement", None)
709 if isinstance(data_set, str):
710 table[u"history"][i][u"data-replacement"] = \
711 self.configuration[u"data-sets"][data_set]
713 if table.get(u"columns", None):
714 for i in range(len(table[u"columns"])):
715 data_set = table[u"columns"][i].get(u"data-set", None)
716 if isinstance(data_set, str):
717 table[u"columns"][i][u"data-set"] = \
718 self.configuration[u"data-sets"][data_set]
719 data_set = table[u"columns"][i].get(
720 u"data-replacement", None)
721 if isinstance(data_set, str):
722 table[u"columns"][i][u"data-replacement"] = \
723 self.configuration[u"data-sets"][data_set]
726 raise PresentationError(
727 f"Wrong data set used in {table.get(u'title', u'')}."
730 self._specification[u"tables"].append(table)
732 def _parse_elements_plots(self, plot):
733 """Parse plots from the specification YAML file.
735 :param plot: Plot to be parsed from the specification file.
737 :raises PresentationError: If plot layout is not defined.
740 # Add layout to the plots:
741 layout = plot[u"layout"].get(u"layout", None)
742 if layout is not None:
743 plot[u"layout"].pop(u"layout")
745 for key, val in (self.configuration[u"plot-layouts"]
747 plot[u"layout"][key] = val
749 raise PresentationError(
750 f"Layout {layout} is not defined in the "
751 f"configuration section."
753 self._specification[u"plots"].append(plot)
755 def _parse_elements_files(self, file):
756 """Parse files from the specification YAML file.
758 :param file: File to be parsed from the specification file.
763 file[u"dir-tables"] = self._replace_tags(
765 self._specification[u"environment"][u"paths"])
768 self._specification[u"files"].append(file)
770 def _parse_elements_cpta(self, cpta):
771 """Parse cpta from the specification YAML file.
773 :param cpta: cpta to be parsed from the specification file.
775 :raises PresentationError: If wrong data set is used or if plot layout
779 for plot in cpta[u"plots"]:
780 # Add layout to the plots:
781 layout = plot.get(u"layout", None)
782 if layout is not None:
785 self.configuration[u"plot-layouts"][layout]
787 raise PresentationError(
788 f"Layout {layout} is not defined in the "
789 f"configuration section."
792 if isinstance(plot.get(u"data", None), str):
793 data_set = plot[u"data"]
796 self.configuration[u"data-sets"][data_set]
798 raise PresentationError(
799 f"Data set {data_set} is not defined in "
800 f"the configuration section."
802 self._specification[u"cpta"] = cpta
804 def _parse_elements(self):
805 """Parse elements (tables, plots, ..) specification in the specification
809 logging.info(u"Parsing specification file: elements ...")
812 for element in self._cfg_yaml:
816 element[u"output-file"] = self._replace_tags(
817 element[u"output-file"],
818 self._specification[u"environment"][u"paths"])
823 element[u"input-file"] = self._replace_tags(
824 element[u"input-file"],
825 self._specification[u"environment"][u"paths"])
830 element[u"output-file-links"] = self._replace_tags(
831 element[u"output-file-links"],
832 self._specification[u"environment"][u"paths"])
836 # Add data sets to the elements:
837 if isinstance(element.get(u"data", None), str):
838 data_set = element[u"data"]
841 self.configuration[u"data-sets"][data_set]
843 raise PresentationError(
844 f"Data set {data_set} is not defined in the "
845 f"configuration section."
847 elif isinstance(element.get(u"data", None), list):
849 for item in element[u"data"]:
852 self.configuration[u"data-sets"][item]
855 raise PresentationError(
856 f"Data set {item} is not defined in the "
857 f"configuration section."
859 element[u"data"] = new_list
862 if element[u"type"] == u"table":
864 logging.info(f" {count:3d} Processing a table ...")
865 self._parse_elements_tables(element)
868 elif element[u"type"] == u"plot":
870 logging.info(f" {count:3d} Processing a plot ...")
871 self._parse_elements_plots(element)
874 elif element[u"type"] == u"file":
876 logging.info(f" {count:3d} Processing a file ...")
877 self._parse_elements_files(element)
880 elif element[u"type"] == u"cpta":
883 f" {count:3d} Processing Continuous Performance Trending "
886 self._parse_elements_cpta(element)
889 logging.info(u"Done.")
891 def read_specification(self):
892 """Parse specification in the specification YAML file.
894 :raises: PresentationError if an error occurred while parsing the
898 self._cfg_yaml = load(self._cfg_file, Loader=FullLoader)
899 except YAMLError as err:
900 raise PresentationError(msg=u"An error occurred while parsing the "
901 u"specification file.",
905 self._parse_configuration()
909 self._parse_elements()
911 logging.debug(f"Specification: \n{pformat(self._specification)}")