1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
20 from os.path import join, exists
23 from pprint import pformat
25 from yaml import load, FullLoader, YAMLError
27 from pal_errors import PresentationError
28 from pal_utils import (
29 get_last_successful_build_nr, get_last_completed_build_number
34 """Specification of Presentation and analytics layer.
36 - based on specification specified in the specification YAML files
37 - presentation and analytics layer is model driven
40 # Tags are used in specification YAML file and replaced while the file is
45 def __init__(self, cfg_dir):
48 :param cfg_dir: Directory with the specification files.
51 self._cfg_dir = cfg_dir
54 self._specification = {
55 u"environment": dict(),
68 def specification(self):
69 """Getter - specification.
71 :returns: Specification.
74 return self._specification
77 def environment(self):
78 """Getter - environment.
80 :returns: Environment specification.
83 return self._specification[u"environment"]
87 """Getter - data_sets.
92 return self._specification[u"data_sets"]
101 return self._specification[u"layouts"]
105 """Getter - static content.
107 :returns: Static content specification.
110 return self._specification[u"static"]
116 :returns: Mapping of the old names of test cases to the new (actual)
120 return self.environment[u"mapping"]
124 """Getter - Ignore list.
126 :returns: List of ignored test cases.
129 return self.environment[u"ignore"]
133 """Getter - Alerting.
137 :returns: Specification of alerts.
140 return self.environment[u"alerting"]
144 """Getter - specification - inputs.
150 return self._specification[u"input"]
153 def input(self, new_value):
154 """Setter - specification - inputs.
156 :param new_value: New value to be set.
157 :type new_value: dict
159 self._specification[u"input"] = new_value
161 def add_build(self, job, build):
162 """Add a build to the list of builds if it does not exist there.
164 :param job: The job which run the build.
165 :param build: The build to be added.
169 if self.input.get(job, None) is None:
170 self.input[job] = list()
171 for existing_build in self.input[job]:
172 if existing_build[u"build"] == build[u"build"]:
175 self.input[job].append(build)
179 """Getter - specification - output formats and versions to be generated.
181 - versions: full, ...
183 :returns: Outputs to be generated.
186 return self._specification[u"output"]
190 """Getter - tables to be generated.
192 :returns: List of specifications of tables to be generated.
195 return self._specification.get(u"tables", list())
199 """Getter - plots to be generated.
201 :returns: List of specifications of plots to be generated.
204 return self._specification.get(u"plots", list())
208 """Getter - files to be generated.
210 :returns: List of specifications of files to be generated.
213 return self._specification.get(u"files", list())
217 """Getter - Continuous Performance Trending and Analysis to be
220 :returns: List of specifications of Continuous Performance Trending and
221 Analysis to be generated.
224 return self._specification[u"cpta"]
226 def set_input_state(self, job, build_nr, state):
227 """Set the state of the input.
229 :param job: Job name.
230 :param build_nr: Build number.
231 :param state: The new input state.
235 :raises: PresentationError if wrong job and/or build is provided.
239 for build in self.input[job]:
240 if build[u"build"] == build_nr:
241 build[u"status"] = state
244 raise PresentationError(
245 f"Build {build_nr} is not defined for job {job} in "
246 f"specification file."
249 raise PresentationError(
250 f"Job {job} and build {build_nr} is not defined in "
251 f"specification file."
254 def set_input_file_name(self, job, build_nr, file_name):
255 """Set the file name for the input.
257 :param job: Job name.
258 :param build_nr: Build number.
259 :param file_name: The new file name.
263 :raises: PresentationError if wrong job and/or build is provided.
267 for build in self.input[job]:
268 if build[u"build"] == build_nr:
269 build[u"file-name"] = file_name
272 raise PresentationError(
273 f"Build {build_nr} is not defined for job {job} in "
274 f"specification file."
277 raise PresentationError(
278 f"Job {job} and build {build_nr} is not defined in "
279 f"specification file."
282 def _get_build_number(self, job, build_type):
283 """Get the number of the job defined by its name:
284 - lastSuccessfulBuild
287 :param job: Job name.
288 :param build_type: Build type:
289 - lastSuccessfulBuild
292 :raises PresentationError: If it is not possible to get the build
294 :returns: The build number.
298 # defined as a range <start, end>
299 if build_type == u"lastSuccessfulBuild":
300 # defined as a range <start, lastSuccessfulBuild>
301 ret_code, build_nr, _ = get_last_successful_build_nr(
302 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
303 elif build_type == u"lastCompletedBuild":
304 # defined as a range <start, lastCompletedBuild>
305 ret_code, build_nr, _ = get_last_completed_build_number(
306 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
308 raise PresentationError(f"Not supported build type: {build_type}")
310 raise PresentationError(
311 f"Not possible to get the build number of {job}."
314 build_nr = int(build_nr)
316 except ValueError as err:
317 raise PresentationError(
318 f"Not possible to get the build number of {job}. Reason:\n"
322 def _get_type_index(self, item_type):
323 """Get index of item type (environment, input, output, ...) in
324 specification YAML file.
326 :param item_type: Item type: Top level items in specification YAML file,
327 e.g.: environment, input, output.
329 :returns: Index of the given item type.
334 for item in self._cfg_yaml:
335 if item[u"type"] == item_type:
340 def _find_tag(self, text):
341 """Find the first tag in the given text. The tag is enclosed by the
342 TAG_OPENER and TAG_CLOSER.
344 :param text: Text to be searched.
346 :returns: The tag, or None if not found.
350 start = text.index(self.TAG_OPENER)
351 end = text.index(self.TAG_CLOSER, start + 1) + 1
352 return text[start:end]
356 def _replace_tags(self, data, src_data=None):
357 """Replace tag(s) in the data by their values.
359 :param data: The data where the tags will be replaced by their values.
360 :param src_data: Data where the tags are defined. It is dictionary where
361 the key is the tag and the value is the tag value. If not given,
362 'data' is used instead.
363 :type data: str, list or dict
365 :returns: Data with the tags replaced.
366 :rtype: str, list or dict
367 :raises: PresentationError if it is not possible to replace the tag or
368 the data is not the supported data type (str, list or dict).
374 if isinstance(data, str):
375 tag = self._find_tag(data)
377 data = data.replace(tag, src_data[tag[1:-1]])
380 if isinstance(data, list):
383 new_list.append(self._replace_tags(item, src_data))
386 if isinstance(data, dict):
388 for key, value in data.items():
389 tag = self._find_tag(value)
392 data[key] = value.replace(tag, src_data[tag[1:-1]])
395 raise PresentationError(
396 f"Not possible to replace the tag {tag}"
399 self._replace_tags(data, src_data)
402 raise PresentationError(u"Replace tags: Not supported data type.")
404 def _parse_env(self):
405 """Parse environment specification in the specification YAML file.
408 logging.info(u"Parsing specification: ENVIRONMENT")
410 idx = self._get_type_index(u"environment")
414 self._specification[u"environment"][u"spec-files"] = \
415 self._cfg_yaml[idx].get(u"spec-files", None)
418 self._specification[u"environment"][u"paths"] = \
419 self._replace_tags(self._cfg_yaml[idx][u"paths"])
421 self._specification[u"environment"][u"paths"] = None
423 self._specification[u"environment"][u"data-sources"] = \
424 self._cfg_yaml[idx].get(u"data-sources", tuple())
426 for source in self._specification[u"environment"][u"data-sources"]:
427 source[u"successful-downloads"] = 0
429 self._specification[u"environment"][u"make-dirs"] = \
430 self._cfg_yaml[idx].get(u"make-dirs", None)
432 self._specification[u"environment"][u"remove-dirs"] = \
433 self._cfg_yaml[idx].get(u"remove-dirs", None)
435 self._specification[u"environment"][u"build-dirs"] = \
436 self._cfg_yaml[idx].get(u"build-dirs", None)
438 self._specification[u"environment"][u"testbeds"] = \
439 self._cfg_yaml[idx].get(u"testbeds", None)
441 self._specification[u"environment"][u"limits"] = \
442 self._cfg_yaml[idx].get(u"limits", None)
444 self._specification[u"environment"][u"urls"] = \
445 self._cfg_yaml[idx].get(u"urls", None)
447 self._specification[u"environment"][u"archive-inputs"] = \
448 self._cfg_yaml[idx].get(u"archive-inputs", False)
450 self._specification[u"environment"][u"reverse-input"] = \
451 self._cfg_yaml[idx].get(u"reverse-input", False)
453 self._specification[u"environment"][u"time-period"] = \
454 self._cfg_yaml[idx].get(u"time-period", None)
456 self._specification[u"environment"][u"alerting"] = \
457 self._cfg_yaml[idx].get(u"alerting", None)
459 self._specification[u"environment"][u"mapping-file"] = \
460 self._cfg_yaml[idx].get(u"mapping-file", None)
462 self._specification[u"environment"][u"ignore-list"] = \
463 self._cfg_yaml[idx].get(u"ignore-list", None)
466 self._load_mapping_table()
469 self._load_ignore_list()
471 def _parse_layouts(self):
472 """Parse layouts specification in the specification YAML file.
475 logging.info(u"Parsing specification: LAYOUTS")
477 idx = self._get_type_index(u"layouts")
482 self._specification[u"layouts"] = self._cfg_yaml[idx]
484 raise PresentationError(u"No layouts defined.")
486 def _parse_data_sets(self):
487 """Parse data sets specification in the specification YAML file.
490 logging.info(u"Parsing specification: DATA SETS")
492 idx = self._get_type_index(u"data-sets")
497 self._specification[u"data_sets"] = self._cfg_yaml[idx]
499 raise PresentationError(u"No Data sets defined.")
501 # Replace ranges by lists
502 for set_name, data_set in self.data_sets.items():
503 if not isinstance(data_set, dict):
505 for job, builds in data_set.items():
508 if isinstance(builds, dict):
509 build_end = builds.get(u"end", None)
510 max_builds = builds.get(u"max-builds", None)
511 reverse = builds.get(u"reverse", False)
513 build_end = int(build_end)
515 # defined as a range <start, build_type>
516 build_end = self._get_build_number(job, build_end)
517 builds = list(range(builds[u"start"], build_end + 1))
518 if max_builds and max_builds < len(builds):
519 builds = builds[-max_builds:]
522 self.data_sets[set_name][job] = builds
523 elif isinstance(builds, list):
524 for idx, item in enumerate(builds):
526 builds[idx] = int(item)
528 # defined as a range <build_type>
529 builds[idx] = self._get_build_number(job, item)
531 # Add sub-sets to sets (only one level):
532 for set_name, data_set in self.data_sets.items():
533 if isinstance(data_set, list):
535 for item in data_set:
537 for key, val in self.data_sets[item].items():
540 raise PresentationError(
541 f"Data set {item} is not defined."
543 self.data_sets[set_name] = new_set
545 def _load_mapping_table(self):
546 """Load a mapping table if it is specified. If not, use empty dict.
549 mapping_file_name = self.environment.get(u"mapping-file", None)
550 if mapping_file_name:
552 with open(mapping_file_name, u'r') as mfile:
553 mapping = load(mfile, Loader=FullLoader)
554 # Make sure everything is lowercase
555 self.environment[u"mapping"] = \
556 {key.lower(): val.lower() for key, val in
558 logging.debug(f"Loaded mapping table:\n{mapping}")
559 except (YAMLError, IOError) as err:
560 raise PresentationError(
561 msg=f"An error occurred while parsing the mapping file "
562 f"{mapping_file_name}",
566 self.environment[u"mapping"] = dict()
568 def _load_ignore_list(self):
569 """Load an ignore list if it is specified. If not, use empty list.
572 ignore_list_name = self.environment.get(u"ignore-list", None)
575 with open(ignore_list_name, u'r') as ifile:
576 ignore = load(ifile, Loader=FullLoader)
577 # Make sure everything is lowercase
578 self.environment[u"ignore"] = \
579 [item.lower() for item in ignore]
580 logging.debug(f"Loaded ignore list:\n{ignore}")
581 except (YAMLError, IOError) as err:
582 raise PresentationError(
583 msg=f"An error occurred while parsing the ignore list file "
584 f"{ignore_list_name}.",
588 self.environment[u"ignore"] = list()
590 def _parse_output(self):
591 """Parse output specification in the specification YAML file.
593 :raises: PresentationError if there is no output defined.
596 logging.info(u"Parsing specification: OUTPUT")
598 idx = self._get_type_index(u"output")
600 raise PresentationError(u"No output defined.")
603 self._specification[u"output"] = self._cfg_yaml[idx]
604 except (KeyError, IndexError):
605 raise PresentationError(u"No output defined.")
607 def _parse_static(self):
608 """Parse specification of the static content in the specification YAML
612 logging.info(u"Parsing specification: STATIC CONTENT")
614 idx = self._get_type_index(u"static")
616 logging.warning(u"No static content specified.")
617 self._specification[u"static"] = dict()
620 for key, value in self._cfg_yaml[idx].items():
621 if isinstance(value, str):
623 self._cfg_yaml[idx][key] = self._replace_tags(
624 value, self._specification[u"environment"][u"paths"])
628 self._specification[u"static"] = self._cfg_yaml[idx]
630 def _parse_elements_tables(self, table):
631 """Parse tables from the specification YAML file.
633 :param table: Table to be parsed from the specification file.
635 :raises PresentationError: If wrong data set is used.
639 table[u"template"] = self._replace_tags(
641 self._specification[u"environment"][u"paths"])
647 for item in (u"reference", u"compare"):
648 if table.get(item, None):
649 data_set = table[item].get(u"data", None)
650 if isinstance(data_set, str):
651 table[item][u"data"] = self.data_sets[data_set]
652 data_set = table[item].get(u"data-replacement", None)
653 if isinstance(data_set, str):
654 table[item][u"data-replacement"] = \
655 self.data_sets[data_set]
657 if table.get(u"columns", None):
658 for i in range(len(table[u"columns"])):
659 data_set = table[u"columns"][i].get(u"data-set", None)
660 if isinstance(data_set, str):
661 table[u"columns"][i][u"data-set"] = \
662 self.data_sets[data_set]
663 data_set = table[u"columns"][i].get(
664 u"data-replacement", None)
665 if isinstance(data_set, str):
666 table[u"columns"][i][u"data-replacement"] = \
667 self.data_sets[data_set]
669 if table.get(u"lines", None):
670 for i in range(len(table[u"lines"])):
671 data_set = table[u"lines"][i].get(u"data-set", None)
672 if isinstance(data_set, str):
673 table[u"lines"][i][u"data-set"] = \
674 self.data_sets[data_set]
677 raise PresentationError(
678 f"Wrong set '{data_set}' used in {table.get(u'title', u'')}."
681 self._specification[u"tables"].append(table)
683 def _parse_elements_plots(self, plot):
684 """Parse plots from the specification YAML file.
686 :param plot: Plot to be parsed from the specification file.
688 :raises PresentationError: If plot layout is not defined.
691 # Add layout to the plots:
692 layout = plot[u"layout"].get(u"layout", None)
693 if layout is not None:
694 plot[u"layout"].pop(u"layout")
696 for key, val in self.layouts[layout].items():
697 plot[u"layout"][key] = val
699 raise PresentationError(f"Layout {layout} is not defined.")
700 self._specification[u"plots"].append(plot)
702 def _parse_elements_files(self, file):
703 """Parse files from the specification YAML file.
705 :param file: File to be parsed from the specification file.
710 file[u"dir-tables"] = self._replace_tags(
712 self._specification[u"environment"][u"paths"])
715 self._specification[u"files"].append(file)
717 def _parse_elements_cpta(self, cpta):
718 """Parse cpta from the specification YAML file.
720 :param cpta: cpta to be parsed from the specification file.
722 :raises PresentationError: If wrong data set is used or if plot layout
726 for plot in cpta[u"plots"]:
727 # Add layout to the plots:
728 layout = plot.get(u"layout", None)
729 if layout is not None:
731 plot[u"layout"] = self.layouts[layout]
733 raise PresentationError(f"Layout {layout} is not defined.")
735 if isinstance(plot.get(u"data", None), str):
736 data_set = plot[u"data"]
738 plot[u"data"] = self.data_sets[data_set]
740 raise PresentationError(
741 f"Data set {data_set} is not defined."
743 self._specification[u"cpta"] = cpta
745 def _parse_elements(self):
746 """Parse elements (tables, plots, ..) specification in the specification
750 logging.info(u"Parsing specification: ELEMENTS")
753 for element in self._cfg_yaml:
757 element[u"output-file"] = self._replace_tags(
758 element[u"output-file"],
759 self.environment[u"paths"]
765 element[u"input-file"] = self._replace_tags(
766 element[u"input-file"],
767 self.environment[u"paths"]
773 element[u"output-file-links"] = self._replace_tags(
774 element[u"output-file-links"],
775 self.environment[u"paths"]
780 # Add data sets to the elements:
781 if isinstance(element.get(u"data", None), str):
782 data_set = element[u"data"]
784 element[u"data"] = self.data_sets[data_set]
786 raise PresentationError(
787 f"Data set {data_set} is not defined."
789 elif isinstance(element.get(u"data", None), list):
791 for item in element[u"data"]:
793 new_list.append(self.data_sets[item])
795 raise PresentationError(
796 f"Data set {item} is not defined."
798 element[u"data"] = new_list
801 if element[u"type"] == u"table":
802 logging.info(f" {count:3d} Processing a table ...")
803 self._parse_elements_tables(element)
805 elif element[u"type"] == u"plot":
806 logging.info(f" {count:3d} Processing a plot ...")
807 self._parse_elements_plots(element)
809 elif element[u"type"] == u"file":
810 logging.info(f" {count:3d} Processing a file ...")
811 self._parse_elements_files(element)
813 elif element[u"type"] == u"cpta":
815 f" {count:3d} Processing Continuous Performance Trending "
818 self._parse_elements_cpta(element)
821 def _prepare_input(self):
822 """Use information from data sets and generate list of jobs and builds
826 logging.info(u"Parsing specification: INPUT")
828 idx = self._get_type_index(u"input")
830 logging.info(u"Creating the list of inputs from data sets.")
831 for data_set in self.data_sets.values():
832 if data_set == "data-sets":
834 for job, builds in data_set.items():
846 logging.info(u"Reading pre-defined inputs.")
847 for job, builds in self._cfg_yaml[idx][u"builds"].items():
859 if self.environment[u"reverse-input"]:
860 for builds in self.input.values():
861 builds.sort(key=lambda k: k[u"build"], reverse=True)
863 def read_specification(self):
864 """Parse specification in the specification YAML files.
866 :raises: PresentationError if an error occurred while parsing the
870 # It always starts with environment.yaml file, it must be present.
871 spec_file = join(self._cfg_dir, u"environment.yaml")
872 logging.info(f"Reading {spec_file}")
873 if not exists(spec_file):
874 raise PresentationError(f"The file {spec_file} does not exist.")
876 with open(spec_file, u"r") as file_read:
878 self._cfg_yaml = load(file_read, Loader=FullLoader)
879 except YAMLError as err:
880 raise PresentationError(
881 f"An error occurred while parsing the specification file "
886 # Load the other specification files specified in the environment.yaml
887 idx = self._get_type_index(u"environment")
889 raise PresentationError(
890 f"No environment defined in the file {spec_file}"
892 for spec_file in self._cfg_yaml[idx].get(u"spec-files", tuple()):
893 logging.info(f"Reading {spec_file}")
894 if not exists(spec_file):
895 raise PresentationError(f"The file {spec_file} does not exist.")
897 with open(spec_file, u"r") as file_read:
899 spec = load(file_read, Loader=FullLoader)
900 except YAMLError as err:
901 raise PresentationError(
902 f"An error occurred while parsing the specification "
907 self._cfg_yaml.extend(spec)
910 self._parse_layouts()
911 self._parse_data_sets()
914 self._parse_elements()
915 self._prepare_input()
917 logging.debug(f"Specification: \n{pformat(self.specification)}")