1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
20 from os.path import join, exists
23 from pprint import pformat
25 from yaml import load, FullLoader, YAMLError
27 from pal_errors import PresentationError
28 from pal_utils import (
29 get_last_successful_build_nr, get_last_completed_build_number
34 """Specification of Presentation and analytics layer.
36 - based on specification specified in the specification YAML files
37 - presentation and analytics layer is model driven
40 # Tags are used in specification YAML file and replaced while the file is
45 def __init__(self, cfg_dir):
48 :param cfg_dir: Directory with the specification files.
51 self._cfg_dir = cfg_dir
54 self._specification = {
55 u"environment": dict(),
68 def specification(self):
69 """Getter - specification.
71 :returns: Specification.
74 return self._specification
77 def environment(self):
78 """Getter - environment.
80 :returns: Environment specification.
83 return self._specification[u"environment"]
87 """Getter - data_sets.
92 return self._specification[u"data_sets"]
101 return self._specification[u"layouts"]
105 """Getter - static content.
107 :returns: Static content specification.
110 return self._specification[u"static"]
116 :returns: Mapping of the old names of test cases to the new (actual)
120 return self.environment[u"mapping"]
124 """Getter - Ignore list.
126 :returns: List of ignored test cases.
129 return self.environment[u"ignore"]
133 """Getter - Alerting.
137 :returns: Specification of alerts.
140 return self.environment[u"alerting"]
144 """Getter - specification - inputs.
150 return self._specification[u"input"]
153 def input(self, new_value):
154 """Setter - specification - inputs.
156 :param new_value: New value to be set.
157 :type new_value: dict
159 self._specification[u"input"] = new_value
161 def add_build(self, job, build):
162 """Add a build to the list of builds if it does not exist there.
164 :param job: The job which run the build.
165 :param build: The build to be added.
169 if self.input.get(job, None) is None:
170 self.input[job] = list()
171 for existing_build in self.input[job]:
172 if existing_build[u"build"] == build[u"build"]:
175 self.input[job].append(build)
179 """Getter - specification - output formats and versions to be generated.
181 - versions: full, ...
183 :returns: Outputs to be generated.
186 return self._specification[u"output"]
190 """Getter - tables to be generated.
192 :returns: List of specifications of tables to be generated.
195 return self._specification.get(u"tables", list())
199 """Getter - plots to be generated.
201 :returns: List of specifications of plots to be generated.
204 return self._specification.get(u"plots", list())
208 """Getter - files to be generated.
210 :returns: List of specifications of files to be generated.
213 return self._specification.get(u"files", list())
217 """Getter - Continuous Performance Trending and Analysis to be
220 :returns: List of specifications of Continuous Performance Trending and
221 Analysis to be generated.
224 return self._specification[u"cpta"]
226 def set_input_state(self, job, build_nr, state):
227 """Set the state of the input.
229 :param job: Job name.
230 :param build_nr: Build number.
231 :param state: The new input state.
235 :raises: PresentationError if wrong job and/or build is provided.
239 for build in self.input[job]:
240 if build[u"build"] == build_nr:
241 build[u"status"] = state
244 raise PresentationError(
245 f"Build {build_nr} is not defined for job {job} in "
246 f"specification file."
249 raise PresentationError(
250 f"Job {job} and build {build_nr} is not defined in "
251 f"specification file."
254 def set_input_file_name(self, job, build_nr, file_name):
255 """Set the file name for the input.
257 :param job: Job name.
258 :param build_nr: Build number.
259 :param file_name: The new file name.
263 :raises: PresentationError if wrong job and/or build is provided.
267 for build in self.input[job]:
268 if build[u"build"] == build_nr:
269 build[u"file-name"] = file_name
272 raise PresentationError(
273 f"Build {build_nr} is not defined for job {job} in "
274 f"specification file."
277 raise PresentationError(
278 f"Job {job} and build {build_nr} is not defined in "
279 f"specification file."
282 def _get_build_number(self, job, build_type):
283 """Get the number of the job defined by its name:
284 - lastSuccessfulBuild
287 :param job: Job name.
288 :param build_type: Build type:
289 - lastSuccessfulBuild
292 :raises PresentationError: If it is not possible to get the build
294 :returns: The build number.
298 # defined as a range <start, end>
299 if build_type == u"lastSuccessfulBuild":
300 # defined as a range <start, lastSuccessfulBuild>
301 ret_code, build_nr, _ = get_last_successful_build_nr(
302 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
303 elif build_type == u"lastCompletedBuild":
304 # defined as a range <start, lastCompletedBuild>
305 ret_code, build_nr, _ = get_last_completed_build_number(
306 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
308 raise PresentationError(f"Not supported build type: {build_type}")
310 raise PresentationError(
311 f"Not possible to get the build number of {job}."
314 build_nr = int(build_nr)
316 except ValueError as err:
317 raise PresentationError(
318 f"Not possible to get the build number of {job}. Reason:\n"
322 def _get_type_index(self, item_type):
323 """Get index of item type (environment, input, output, ...) in
324 specification YAML file.
326 :param item_type: Item type: Top level items in specification YAML file,
327 e.g.: environment, input, output.
329 :returns: Index of the given item type.
334 for item in self._cfg_yaml:
335 if item[u"type"] == item_type:
340 def _find_tag(self, text):
341 """Find the first tag in the given text. The tag is enclosed by the
342 TAG_OPENER and TAG_CLOSER.
344 :param text: Text to be searched.
346 :returns: The tag, or None if not found.
350 start = text.index(self.TAG_OPENER)
351 end = text.index(self.TAG_CLOSER, start + 1) + 1
352 return text[start:end]
356 def _replace_tags(self, data, src_data=None):
357 """Replace tag(s) in the data by their values.
359 :param data: The data where the tags will be replaced by their values.
360 :param src_data: Data where the tags are defined. It is dictionary where
361 the key is the tag and the value is the tag value. If not given,
362 'data' is used instead.
363 :type data: str, list or dict
365 :returns: Data with the tags replaced.
366 :rtype: str, list or dict
367 :raises: PresentationError if it is not possible to replace the tag or
368 the data is not the supported data type (str, list or dict).
374 if isinstance(data, str):
375 tag = self._find_tag(data)
377 data = data.replace(tag, src_data[tag[1:-1]])
380 if isinstance(data, list):
383 new_list.append(self._replace_tags(item, src_data))
386 if isinstance(data, dict):
388 for key, value in data.items():
389 tag = self._find_tag(value)
392 data[key] = value.replace(tag, src_data[tag[1:-1]])
395 raise PresentationError(
396 f"Not possible to replace the tag {tag}"
399 self._replace_tags(data, src_data)
402 raise PresentationError(u"Replace tags: Not supported data type.")
404 def _parse_env(self):
405 """Parse environment specification in the specification YAML file.
408 logging.info(u"Parsing specification: ENVIRONMENT")
410 idx = self._get_type_index(u"environment")
414 self._specification[u"environment"][u"spec-files"] = \
415 self._cfg_yaml[idx].get(u"spec-files", None)
418 self._specification[u"environment"][u"paths"] = \
419 self._replace_tags(self._cfg_yaml[idx][u"paths"])
421 self._specification[u"environment"][u"paths"] = None
423 self._specification[u"environment"][u"data-sources"] = \
424 self._cfg_yaml[idx].get(u"data-sources", tuple())
426 for source in self._specification[u"environment"][u"data-sources"]:
427 source[u"successful-downloads"] = 0
429 self._specification[u"environment"][u"make-dirs"] = \
430 self._cfg_yaml[idx].get(u"make-dirs", None)
432 self._specification[u"environment"][u"remove-dirs"] = \
433 self._cfg_yaml[idx].get(u"remove-dirs", None)
435 self._specification[u"environment"][u"build-dirs"] = \
436 self._cfg_yaml[idx].get(u"build-dirs", None)
438 self._specification[u"environment"][u"testbeds"] = \
439 self._cfg_yaml[idx].get(u"testbeds", None)
441 self._specification[u"environment"][u"limits"] = \
442 self._cfg_yaml[idx].get(u"limits", None)
444 self._specification[u"environment"][u"urls"] = \
445 self._cfg_yaml[idx].get(u"urls", None)
447 self._specification[u"environment"][u"archive-inputs"] = \
448 self._cfg_yaml[idx].get(u"archive-inputs", False)
450 self._specification[u"environment"][u"reverse-input"] = \
451 self._cfg_yaml[idx].get(u"reverse-input", False)
453 self._specification[u"environment"][u"time-period"] = \
454 self._cfg_yaml[idx].get(u"time-period", None)
456 self._specification[u"environment"][u"alerting"] = \
457 self._cfg_yaml[idx].get(u"alerting", None)
459 self._specification[u"environment"][u"mapping-file"] = \
460 self._cfg_yaml[idx].get(u"mapping-file", None)
462 self._specification[u"environment"][u"ignore-list"] = \
463 self._cfg_yaml[idx].get(u"ignore-list", None)
466 self._load_mapping_table()
469 self._load_ignore_list()
471 def _parse_layouts(self):
472 """Parse layouts specification in the specification YAML file.
475 logging.info(u"Parsing specification: LAYOUTS")
477 idx = self._get_type_index(u"layouts")
482 self._specification[u"layouts"] = self._cfg_yaml[idx]
484 raise PresentationError(u"No layouts defined.")
486 def _parse_data_sets(self):
487 """Parse data sets specification in the specification YAML file.
490 logging.info(u"Parsing specification: DATA SETS")
492 idx = self._get_type_index(u"data-sets")
497 self._specification[u"data_sets"] = self._cfg_yaml[idx]
499 raise PresentationError(u"No Data sets defined.")
501 # Replace ranges by lists
502 for set_name, data_set in self.data_sets.items():
503 if not isinstance(data_set, dict):
505 for job, builds in data_set.items():
508 if isinstance(builds, dict):
509 build_end = builds.get(u"end", None)
510 max_builds = builds.get(u"max-builds", None)
511 reverse = builds.get(u"reverse", False)
513 build_end = int(build_end)
515 # defined as a range <start, build_type>
516 build_end = self._get_build_number(job, build_end)
517 builds = list(range(builds[u"start"], build_end + 1))
518 if max_builds and max_builds < len(builds):
519 builds = builds[-max_builds:]
522 self.data_sets[set_name][job] = builds
523 elif isinstance(builds, list):
524 for idx, item in enumerate(builds):
526 builds[idx] = int(item)
528 # defined as a range <build_type>
529 builds[idx] = self._get_build_number(job, item)
531 # Add sub-sets to sets (only one level):
532 for set_name, data_set in self.data_sets.items():
533 if isinstance(data_set, list):
535 for item in data_set:
537 for key, val in self.data_sets[item].items():
540 raise PresentationError(
541 f"Data set {item} is not defined."
543 self.data_sets[set_name] = new_set
545 def _load_mapping_table(self):
546 """Load a mapping table if it is specified. If not, use empty dict.
549 mapping_file_name = self.environment.get(u"mapping-file", None)
550 if mapping_file_name:
552 with open(mapping_file_name, u'r') as mfile:
553 mapping = load(mfile, Loader=FullLoader)
554 # Make sure everything is lowercase
555 self.environment[u"mapping"] = \
556 {key.lower(): val.lower() for key, val in
558 logging.debug(f"Loaded mapping table:\n{mapping}")
559 except (YAMLError, IOError) as err:
560 raise PresentationError(
561 msg=f"An error occurred while parsing the mapping file "
562 f"{mapping_file_name}",
566 self.environment[u"mapping"] = dict()
568 def _load_ignore_list(self):
569 """Load an ignore list if it is specified. If not, use empty list.
572 ignore_list_name = self.environment.get(u"ignore-list", None)
575 with open(ignore_list_name, u'r') as ifile:
576 ignore = load(ifile, Loader=FullLoader)
577 # Make sure everything is lowercase
578 self.environment[u"ignore"] = \
579 [item.lower() for item in ignore]
580 logging.debug(f"Loaded ignore list:\n{ignore}")
581 except (YAMLError, IOError) as err:
582 raise PresentationError(
583 msg=f"An error occurred while parsing the ignore list file "
584 f"{ignore_list_name}.",
588 self.environment[u"ignore"] = list()
590 def _parse_output(self):
591 """Parse output specification in the specification YAML file.
593 :raises: PresentationError if there is no output defined.
596 logging.info(u"Parsing specification: OUTPUT")
598 idx = self._get_type_index(u"output")
600 raise PresentationError(u"No output defined.")
603 self._specification[u"output"] = self._cfg_yaml[idx]
604 except (KeyError, IndexError):
605 raise PresentationError(u"No output defined.")
607 def _parse_static(self):
608 """Parse specification of the static content in the specification YAML
612 logging.info(u"Parsing specification: STATIC CONTENT")
614 idx = self._get_type_index(u"static")
616 logging.warning(u"No static content specified.")
617 self._specification[u"static"] = dict()
620 for key, value in self._cfg_yaml[idx].items():
621 if isinstance(value, str):
623 self._cfg_yaml[idx][key] = self._replace_tags(
624 value, self._specification[u"environment"][u"paths"])
628 self._specification[u"static"] = self._cfg_yaml[idx]
630 def _parse_elements_tables(self, table):
631 """Parse tables from the specification YAML file.
633 :param table: Table to be parsed from the specification file.
635 :raises PresentationError: If wrong data set is used.
639 table[u"template"] = self._replace_tags(
641 self._specification[u"environment"][u"paths"])
647 for item in (u"reference", u"compare"):
648 if table.get(item, None):
649 data_set = table[item].get(u"data", None)
650 if isinstance(data_set, str):
651 table[item][u"data"] = self.data_sets[data_set]
652 data_set = table[item].get(u"data-replacement", None)
653 if isinstance(data_set, str):
654 table[item][u"data-replacement"] = \
655 self.data_sets[data_set]
657 if table.get(u"columns", None):
658 for i in range(len(table[u"columns"])):
659 data_set = table[u"columns"][i].get(u"data-set", None)
660 if isinstance(data_set, str):
661 table[u"columns"][i][u"data-set"] = \
662 self.data_sets[data_set]
663 data_set = table[u"columns"][i].get(
664 u"data-replacement", None)
665 if isinstance(data_set, str):
666 table[u"columns"][i][u"data-replacement"] = \
667 self.data_sets[data_set]
670 raise PresentationError(
671 f"Wrong data set used in {table.get(u'title', u'')}."
674 self._specification[u"tables"].append(table)
676 def _parse_elements_plots(self, plot):
677 """Parse plots from the specification YAML file.
679 :param plot: Plot to be parsed from the specification file.
681 :raises PresentationError: If plot layout is not defined.
684 # Add layout to the plots:
685 layout = plot[u"layout"].get(u"layout", None)
686 if layout is not None:
687 plot[u"layout"].pop(u"layout")
689 for key, val in self.layouts[layout].items():
690 plot[u"layout"][key] = val
692 raise PresentationError(f"Layout {layout} is not defined.")
693 self._specification[u"plots"].append(plot)
695 def _parse_elements_files(self, file):
696 """Parse files from the specification YAML file.
698 :param file: File to be parsed from the specification file.
703 file[u"dir-tables"] = self._replace_tags(
705 self._specification[u"environment"][u"paths"])
708 self._specification[u"files"].append(file)
710 def _parse_elements_cpta(self, cpta):
711 """Parse cpta from the specification YAML file.
713 :param cpta: cpta to be parsed from the specification file.
715 :raises PresentationError: If wrong data set is used or if plot layout
719 for plot in cpta[u"plots"]:
720 # Add layout to the plots:
721 layout = plot.get(u"layout", None)
722 if layout is not None:
724 plot[u"layout"] = self.layouts[layout]
726 raise PresentationError(f"Layout {layout} is not defined.")
728 if isinstance(plot.get(u"data", None), str):
729 data_set = plot[u"data"]
731 plot[u"data"] = self.data_sets[data_set]
733 raise PresentationError(
734 f"Data set {data_set} is not defined."
736 self._specification[u"cpta"] = cpta
738 def _parse_elements(self):
739 """Parse elements (tables, plots, ..) specification in the specification
743 logging.info(u"Parsing specification: ELEMENTS")
746 for element in self._cfg_yaml:
750 element[u"output-file"] = self._replace_tags(
751 element[u"output-file"],
752 self.environment[u"paths"]
758 element[u"input-file"] = self._replace_tags(
759 element[u"input-file"],
760 self.environment[u"paths"]
766 element[u"output-file-links"] = self._replace_tags(
767 element[u"output-file-links"],
768 self.environment[u"paths"]
773 # Add data sets to the elements:
774 if isinstance(element.get(u"data", None), str):
775 data_set = element[u"data"]
777 element[u"data"] = self.data_sets[data_set]
779 raise PresentationError(
780 f"Data set {data_set} is not defined."
782 elif isinstance(element.get(u"data", None), list):
784 for item in element[u"data"]:
786 new_list.append(self.data_sets[item])
788 raise PresentationError(
789 f"Data set {item} is not defined."
791 element[u"data"] = new_list
794 if element[u"type"] == u"table":
795 logging.info(f" {count:3d} Processing a table ...")
796 self._parse_elements_tables(element)
798 elif element[u"type"] == u"plot":
799 logging.info(f" {count:3d} Processing a plot ...")
800 self._parse_elements_plots(element)
802 elif element[u"type"] == u"file":
803 logging.info(f" {count:3d} Processing a file ...")
804 self._parse_elements_files(element)
806 elif element[u"type"] == u"cpta":
808 f" {count:3d} Processing Continuous Performance Trending "
811 self._parse_elements_cpta(element)
814 def _prepare_input(self):
815 """Use information from data sets and generate list of jobs and builds
819 logging.info(u"Parsing specification: INPUT")
821 idx = self._get_type_index(u"input")
823 logging.info(u"Creating the list of inputs from data sets.")
824 for data_set in self.data_sets.values():
825 if data_set == "data-sets":
827 for job, builds in data_set.items():
839 logging.info(u"Reading pre-defined inputs.")
840 for job, builds in self._cfg_yaml[idx][u"builds"].items():
852 if self.environment[u"reverse-input"]:
853 for builds in self.input.values():
854 builds.sort(key=lambda k: k[u"build"], reverse=True)
856 def read_specification(self):
857 """Parse specification in the specification YAML files.
859 :raises: PresentationError if an error occurred while parsing the
863 # It always starts with environment.yaml file, it must be present.
864 spec_file = join(self._cfg_dir, u"environment.yaml")
865 logging.info(f"Reading {spec_file}")
866 if not exists(spec_file):
867 raise PresentationError(f"The file {spec_file} does not exist.")
869 with open(spec_file, u"r") as file_read:
871 self._cfg_yaml = load(file_read, Loader=FullLoader)
872 except YAMLError as err:
873 raise PresentationError(
874 f"An error occurred while parsing the specification file "
879 # Load the other specification files specified in the environment.yaml
880 idx = self._get_type_index(u"environment")
882 raise PresentationError(
883 f"No environment defined in the file {spec_file}"
885 for spec_file in self._cfg_yaml[idx].get(u"spec-files", tuple()):
886 logging.info(f"Reading {spec_file}")
887 if not exists(spec_file):
888 raise PresentationError(f"The file {spec_file} does not exist.")
890 with open(spec_file, u"r") as file_read:
892 spec = load(file_read, Loader=FullLoader)
893 except YAMLError as err:
894 raise PresentationError(
895 f"An error occurred while parsing the specification "
900 self._cfg_yaml.extend(spec)
903 self._parse_layouts()
904 self._parse_data_sets()
907 self._parse_elements()
908 self._prepare_input()
910 logging.debug(f"Specification: \n{pformat(self.specification)}")