1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
20 from os.path import join, exists
23 from pprint import pformat
25 from yaml import load, FullLoader, YAMLError
27 from pal_errors import PresentationError
28 from pal_utils import (
29 get_last_successful_build_nr, get_last_completed_build_number
34 """Specification of Presentation and analytics layer.
36 - based on specification specified in the specification YAML files
37 - presentation and analytics layer is model driven
40 # Tags are used in specification YAML file and replaced while the file is
45 def __init__(self, cfg_dir):
48 :param cfg_dir: Directory with the specification files.
51 self._cfg_dir = cfg_dir
54 self._specification = {
55 u"environment": dict(),
68 def specification(self):
69 """Getter - specification.
71 :returns: Specification.
74 return self._specification
77 def environment(self):
78 """Getter - environment.
80 :returns: Environment specification.
83 return self._specification[u"environment"]
87 """Getter - data_sets.
92 return self._specification[u"data_sets"]
101 return self._specification[u"layouts"]
105 """Getter - static content.
107 :returns: Static content specification.
110 return self._specification[u"static"]
116 :returns: Mapping of the old names of test cases to the new (actual)
120 return self.environment[u"mapping"]
124 """Getter - Ignore list.
126 :returns: List of ignored test cases.
129 return self.environment[u"ignore"]
133 """Getter - Alerting.
137 :returns: Specification of alerts.
140 return self.environment[u"alerting"]
144 """Getter - specification - inputs.
150 return self._specification[u"input"]
153 def input(self, new_value):
154 """Setter - specification - inputs.
156 :param new_value: New value to be set.
157 :type new_value: dict
159 self._specification[u"input"] = new_value
161 def add_build(self, job, build):
162 """Add a build to the list of builds if it does not exist there.
164 :param job: The job which run the build.
165 :param build: The build to be added.
169 if self.input.get(job, None) is None:
170 self.input[job] = list()
171 for existing_build in self.input[job]:
172 if existing_build[u"build"] == build[u"build"]:
175 self.input[job].append(build)
179 """Getter - specification - output formats and versions to be generated.
181 - versions: full, ...
183 :returns: Outputs to be generated.
186 return self._specification[u"output"]
190 """Getter - tables to be generated.
192 :returns: List of specifications of tables to be generated.
195 return self._specification[u"tables"]
199 """Getter - plots to be generated.
201 :returns: List of specifications of plots to be generated.
204 return self._specification[u"plots"]
208 """Getter - files to be generated.
210 :returns: List of specifications of files to be generated.
213 return self._specification[u"files"]
217 """Getter - Continuous Performance Trending and Analysis to be
220 :returns: List of specifications of Continuous Performance Trending and
221 Analysis to be generated.
224 return self._specification[u"cpta"]
226 def set_input_state(self, job, build_nr, state):
227 """Set the state of the input.
229 :param job: Job name.
230 :param build_nr: Build number.
231 :param state: The new input state.
235 :raises: PresentationError if wrong job and/or build is provided.
239 for build in self.input[job]:
240 if build[u"build"] == build_nr:
241 build[u"status"] = state
244 raise PresentationError(
245 f"Build {build_nr} is not defined for job {job} in "
246 f"specification file."
249 raise PresentationError(
250 f"Job {job} and build {build_nr} is not defined in "
251 f"specification file."
254 def set_input_file_name(self, job, build_nr, file_name):
255 """Set the file name for the input.
257 :param job: Job name.
258 :param build_nr: Build number.
259 :param file_name: The new file name.
263 :raises: PresentationError if wrong job and/or build is provided.
267 for build in self.input[job]:
268 if build[u"build"] == build_nr:
269 build[u"file-name"] = file_name
272 raise PresentationError(
273 f"Build {build_nr} is not defined for job {job} in "
274 f"specification file."
277 raise PresentationError(
278 f"Job {job} and build {build_nr} is not defined in "
279 f"specification file."
282 def _get_build_number(self, job, build_type):
283 """Get the number of the job defined by its name:
284 - lastSuccessfulBuild
287 :param job: Job name.
288 :param build_type: Build type:
289 - lastSuccessfulBuild
292 :raises PresentationError: If it is not possible to get the build
294 :returns: The build number.
298 # defined as a range <start, end>
299 if build_type == u"lastSuccessfulBuild":
300 # defined as a range <start, lastSuccessfulBuild>
301 ret_code, build_nr, _ = get_last_successful_build_nr(
302 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
303 elif build_type == u"lastCompletedBuild":
304 # defined as a range <start, lastCompletedBuild>
305 ret_code, build_nr, _ = get_last_completed_build_number(
306 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
308 raise PresentationError(f"Not supported build type: {build_type}")
310 raise PresentationError(
311 f"Not possible to get the build number of {job}."
314 build_nr = int(build_nr)
316 except ValueError as err:
317 raise PresentationError(
318 f"Not possible to get the build number of {job}. Reason:\n"
322 def _get_type_index(self, item_type):
323 """Get index of item type (environment, input, output, ...) in
324 specification YAML file.
326 :param item_type: Item type: Top level items in specification YAML file,
327 e.g.: environment, input, output.
329 :returns: Index of the given item type.
334 for item in self._cfg_yaml:
335 if item[u"type"] == item_type:
340 def _find_tag(self, text):
341 """Find the first tag in the given text. The tag is enclosed by the
342 TAG_OPENER and TAG_CLOSER.
344 :param text: Text to be searched.
346 :returns: The tag, or None if not found.
350 start = text.index(self.TAG_OPENER)
351 end = text.index(self.TAG_CLOSER, start + 1) + 1
352 return text[start:end]
356 def _replace_tags(self, data, src_data=None):
357 """Replace tag(s) in the data by their values.
359 :param data: The data where the tags will be replaced by their values.
360 :param src_data: Data where the tags are defined. It is dictionary where
361 the key is the tag and the value is the tag value. If not given,
362 'data' is used instead.
363 :type data: str, list or dict
365 :returns: Data with the tags replaced.
366 :rtype: str, list or dict
367 :raises: PresentationError if it is not possible to replace the tag or
368 the data is not the supported data type (str, list or dict).
374 if isinstance(data, str):
375 tag = self._find_tag(data)
377 data = data.replace(tag, src_data[tag[1:-1]])
380 if isinstance(data, list):
383 new_list.append(self._replace_tags(item, src_data))
386 if isinstance(data, dict):
388 for key, value in data.items():
389 tag = self._find_tag(value)
392 data[key] = value.replace(tag, src_data[tag[1:-1]])
395 raise PresentationError(
396 f"Not possible to replace the tag {tag}"
399 self._replace_tags(data, src_data)
402 raise PresentationError(u"Replace tags: Not supported data type.")
404 def _parse_env(self):
405 """Parse environment specification in the specification YAML file.
408 logging.info(u"Parsing specification: ENVIRONMENT")
410 idx = self._get_type_index(u"environment")
414 self._specification[u"environment"][u"spec-files"] = \
415 self._cfg_yaml[idx].get(u"spec-files", None)
418 self._specification[u"environment"][u"paths"] = \
419 self._replace_tags(self._cfg_yaml[idx][u"paths"])
421 self._specification[u"environment"][u"paths"] = None
423 self._specification[u"environment"][u"data-sources"] = \
424 self._cfg_yaml[idx].get(u"data-sources", tuple())
426 for source in self._specification[u"environment"][u"data-sources"]:
427 source[u"successful-downloads"] = 0
429 self._specification[u"environment"][u"make-dirs"] = \
430 self._cfg_yaml[idx].get(u"make-dirs", None)
432 self._specification[u"environment"][u"remove-dirs"] = \
433 self._cfg_yaml[idx].get(u"remove-dirs", None)
435 self._specification[u"environment"][u"build-dirs"] = \
436 self._cfg_yaml[idx].get(u"build-dirs", None)
438 self._specification[u"environment"][u"testbeds"] = \
439 self._cfg_yaml[idx].get(u"testbeds", None)
441 self._specification[u"environment"][u"limits"] = \
442 self._cfg_yaml[idx].get(u"limits", None)
444 self._specification[u"environment"][u"urls"] = \
445 self._cfg_yaml[idx].get(u"urls", None)
447 self._specification[u"environment"][u"archive-inputs"] = \
448 self._cfg_yaml[idx].get(u"archive-inputs", False)
450 self._specification[u"environment"][u"reverse-input"] = \
451 self._cfg_yaml[idx].get(u"reverse-input", False)
453 self._specification[u"environment"][u"time-period"] = \
454 self._cfg_yaml[idx].get(u"time-period", None)
456 self._specification[u"environment"][u"alerting"] = \
457 self._cfg_yaml[idx].get(u"alerting", None)
459 self._specification[u"environment"][u"mapping-file"] = \
460 self._cfg_yaml[idx].get(u"mapping-file", None)
462 self._specification[u"environment"][u"ignore-list"] = \
463 self._cfg_yaml[idx].get(u"ignore-list", None)
466 self._load_mapping_table()
469 self._load_ignore_list()
471 def _parse_layouts(self):
472 """Parse layouts specification in the specification YAML file.
475 logging.info(u"Parsing specification: LAYOUTS")
477 idx = self._get_type_index(u"layouts")
482 self._specification[u"layouts"] = self._cfg_yaml[idx]
484 raise PresentationError(u"No layouts defined.")
486 def _parse_data_sets(self):
487 """Parse data sets specification in the specification YAML file.
490 logging.info(u"Parsing specification: DATA SETS")
492 idx = self._get_type_index(u"data-sets")
497 self._specification[u"data_sets"] = self._cfg_yaml[idx]
499 raise PresentationError(u"No Data sets defined.")
501 # Replace ranges by lists
502 for set_name, data_set in self.data_sets.items():
503 if not isinstance(data_set, dict):
505 for job, builds in data_set.items():
508 if isinstance(builds, dict):
509 build_end = builds.get(u"end", None)
510 max_builds = builds.get(u"max-builds", None)
511 reverse = builds.get(u"reverse", False)
513 build_end = int(build_end)
515 # defined as a range <start, build_type>
516 build_end = self._get_build_number(job, build_end)
517 builds = list(range(builds[u"start"], build_end + 1))
518 if max_builds and max_builds < len(builds):
519 builds = builds[-max_builds:]
522 self.data_sets[set_name][job] = builds
523 elif isinstance(builds, list):
524 for idx, item in enumerate(builds):
526 builds[idx] = int(item)
528 # defined as a range <build_type>
529 builds[idx] = self._get_build_number(job, item)
531 # Add sub-sets to sets (only one level):
532 for set_name, data_set in self.data_sets.items():
533 if isinstance(data_set, list):
535 for item in data_set:
537 for key, val in self.data_sets[item].items():
540 raise PresentationError(
541 f"Data set {item} is not defined."
543 self.data_sets[set_name] = new_set
545 def _load_mapping_table(self):
546 """Load a mapping table if it is specified. If not, use empty dict.
549 mapping_file_name = self.environment.get(u"mapping-file", None)
550 if mapping_file_name:
552 with open(mapping_file_name, u'r') as mfile:
553 mapping = load(mfile, Loader=FullLoader)
554 # Make sure everything is lowercase
555 self.environment[u"mapping"] = \
556 {key.lower(): val.lower() for key, val in
558 logging.debug(f"Loaded mapping table:\n{mapping}")
559 except (YAMLError, IOError) as err:
560 raise PresentationError(
561 msg=f"An error occurred while parsing the mapping file "
562 f"{mapping_file_name}",
566 self.environment[u"mapping"] = dict()
568 def _load_ignore_list(self):
569 """Load an ignore list if it is specified. If not, use empty list.
572 ignore_list_name = self.environment.get(u"ignore-list", None)
575 with open(ignore_list_name, u'r') as ifile:
576 ignore = load(ifile, Loader=FullLoader)
577 # Make sure everything is lowercase
578 self.environment[u"ignore"] = \
579 [item.lower() for item in ignore]
580 logging.debug(f"Loaded ignore list:\n{ignore}")
581 except (YAMLError, IOError) as err:
582 raise PresentationError(
583 msg=f"An error occurred while parsing the ignore list file "
584 f"{ignore_list_name}.",
588 self.environment[u"ignore"] = list()
590 def _parse_output(self):
591 """Parse output specification in the specification YAML file.
593 :raises: PresentationError if there is no output defined.
596 logging.info(u"Parsing specification: OUTPUT")
598 idx = self._get_type_index(u"output")
600 raise PresentationError(u"No output defined.")
603 self._specification[u"output"] = self._cfg_yaml[idx]
604 except (KeyError, IndexError):
605 raise PresentationError(u"No output defined.")
607 def _parse_static(self):
608 """Parse specification of the static content in the specification YAML
612 logging.info(u"Parsing specification: STATIC CONTENT")
614 idx = self._get_type_index(u"static")
616 logging.warning(u"No static content specified.")
618 for key, value in self._cfg_yaml[idx].items():
619 if isinstance(value, str):
621 self._cfg_yaml[idx][key] = self._replace_tags(
622 value, self._specification[u"environment"][u"paths"])
626 self._specification[u"static"] = self._cfg_yaml[idx]
628 def _parse_elements_tables(self, table):
629 """Parse tables from the specification YAML file.
631 :param table: Table to be parsed from the specification file.
633 :raises PresentationError: If wrong data set is used.
637 table[u"template"] = self._replace_tags(
639 self._specification[u"environment"][u"paths"])
645 for item in (u"reference", u"compare"):
646 if table.get(item, None):
647 data_set = table[item].get(u"data", None)
648 if isinstance(data_set, str):
649 table[item][u"data"] = self.data_sets[data_set]
650 data_set = table[item].get(u"data-replacement", None)
651 if isinstance(data_set, str):
652 table[item][u"data-replacement"] = \
653 self.data_sets[data_set]
655 if table.get(u"columns", None):
656 for i in range(len(table[u"columns"])):
657 data_set = table[u"columns"][i].get(u"data-set", None)
658 if isinstance(data_set, str):
659 table[u"columns"][i][u"data-set"] = \
660 self.data_sets[data_set]
661 data_set = table[u"columns"][i].get(
662 u"data-replacement", None)
663 if isinstance(data_set, str):
664 table[u"columns"][i][u"data-replacement"] = \
665 self.data_sets[data_set]
668 raise PresentationError(
669 f"Wrong data set used in {table.get(u'title', u'')}."
672 self._specification[u"tables"].append(table)
674 def _parse_elements_plots(self, plot):
675 """Parse plots from the specification YAML file.
677 :param plot: Plot to be parsed from the specification file.
679 :raises PresentationError: If plot layout is not defined.
682 # Add layout to the plots:
683 layout = plot[u"layout"].get(u"layout", None)
684 if layout is not None:
685 plot[u"layout"].pop(u"layout")
687 for key, val in self.layouts[layout].items():
688 plot[u"layout"][key] = val
690 raise PresentationError(f"Layout {layout} is not defined.")
691 self._specification[u"plots"].append(plot)
693 def _parse_elements_files(self, file):
694 """Parse files from the specification YAML file.
696 :param file: File to be parsed from the specification file.
701 file[u"dir-tables"] = self._replace_tags(
703 self._specification[u"environment"][u"paths"])
706 self._specification[u"files"].append(file)
708 def _parse_elements_cpta(self, cpta):
709 """Parse cpta from the specification YAML file.
711 :param cpta: cpta to be parsed from the specification file.
713 :raises PresentationError: If wrong data set is used or if plot layout
717 for plot in cpta[u"plots"]:
718 # Add layout to the plots:
719 layout = plot.get(u"layout", None)
720 if layout is not None:
722 plot[u"layout"] = self.layouts[layout]
724 raise PresentationError(f"Layout {layout} is not defined.")
726 if isinstance(plot.get(u"data", None), str):
727 data_set = plot[u"data"]
729 plot[u"data"] = self.data_sets[data_set]
731 raise PresentationError(
732 f"Data set {data_set} is not defined."
734 self._specification[u"cpta"] = cpta
736 def _parse_elements(self):
737 """Parse elements (tables, plots, ..) specification in the specification
741 logging.info(u"Parsing specification: ELEMENTS")
744 for element in self._cfg_yaml:
748 element[u"output-file"] = self._replace_tags(
749 element[u"output-file"],
750 self.environment[u"paths"]
756 element[u"input-file"] = self._replace_tags(
757 element[u"input-file"],
758 self.environment[u"paths"]
764 element[u"output-file-links"] = self._replace_tags(
765 element[u"output-file-links"],
766 self.environment[u"paths"]
771 # Add data sets to the elements:
772 if isinstance(element.get(u"data", None), str):
773 data_set = element[u"data"]
775 element[u"data"] = self.data_sets[data_set]
777 raise PresentationError(
778 f"Data set {data_set} is not defined."
780 elif isinstance(element.get(u"data", None), list):
782 for item in element[u"data"]:
784 new_list.append(self.data_sets[item])
786 raise PresentationError(
787 f"Data set {item} is not defined."
789 element[u"data"] = new_list
792 if element[u"type"] == u"table":
793 logging.info(f" {count:3d} Processing a table ...")
794 self._parse_elements_tables(element)
796 elif element[u"type"] == u"plot":
797 logging.info(f" {count:3d} Processing a plot ...")
798 self._parse_elements_plots(element)
800 elif element[u"type"] == u"file":
801 logging.info(f" {count:3d} Processing a file ...")
802 self._parse_elements_files(element)
804 elif element[u"type"] == u"cpta":
806 f" {count:3d} Processing Continuous Performance Trending "
809 self._parse_elements_cpta(element)
812 def _prepare_input(self):
813 """Use information from data sets and generate list of jobs and builds
817 logging.info(u"Parsing specification: INPUT")
819 for data_set in self.data_sets.values():
820 if data_set == "data-sets":
822 for job, builds in data_set.items():
834 if self.environment[u"reverse-input"]:
835 for builds in self.input.values():
836 builds.sort(key=lambda k: k[u"build"], reverse=True)
838 def read_specification(self):
839 """Parse specification in the specification YAML files.
841 :raises: PresentationError if an error occurred while parsing the
845 # It always starts with environment.yaml file, it must be present.
846 spec_file = join(self._cfg_dir, u"environment.yaml")
847 logging.info(f"Reading {spec_file}")
848 if not exists(spec_file):
849 raise PresentationError(f"The file {spec_file} does not exist.")
851 with open(spec_file, u"r") as file_read:
853 self._cfg_yaml = load(file_read, Loader=FullLoader)
854 except YAMLError as err:
855 raise PresentationError(
856 f"An error occurred while parsing the specification file "
861 # Load the other specification files specified in the environment.yaml
862 idx = self._get_type_index(u"environment")
864 raise PresentationError(
865 f"No environment defined in the file {spec_file}"
867 for spec_file in self._cfg_yaml[idx].get(u"spec-files", tuple()):
868 logging.info(f"Reading {spec_file}")
869 if not exists(spec_file):
870 raise PresentationError(f"The file {spec_file} does not exist.")
872 with open(spec_file, u"r") as file_read:
874 spec = load(file_read, Loader=FullLoader)
875 except YAMLError as err:
876 raise PresentationError(
877 f"An error occurred while parsing the specification "
882 self._cfg_yaml.extend(spec)
885 self._parse_layouts()
886 self._parse_data_sets()
889 self._parse_elements()
890 self._prepare_input()
892 logging.debug(f"Specification: \n{pformat(self.specification)}")