1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
20 from os.path import join, exists
23 from pprint import pformat
25 from yaml import load, FullLoader, YAMLError
27 from pal_errors import PresentationError
28 from pal_utils import (
29 get_last_successful_build_nr, get_last_completed_build_number
34 """Specification of Presentation and analytics layer.
36 - based on specification specified in the specification YAML files
37 - presentation and analytics layer is model driven
40 # Tags are used in specification YAML file and replaced while the file is
45 def __init__(self, cfg_dir):
48 :param cfg_dir: Directory with the specification files.
51 self._cfg_dir = cfg_dir
54 self._specification = {
55 u"environment": dict(),
68 def specification(self):
69 """Getter - specification.
71 :returns: Specification.
74 return self._specification
77 def environment(self):
78 """Getter - environment.
80 :returns: Environment specification.
83 return self._specification[u"environment"]
87 """Getter - data_sets.
92 return self._specification[u"data_sets"]
101 return self._specification[u"layouts"]
105 """Getter - static content.
107 :returns: Static content specification.
110 return self._specification[u"static"]
116 :returns: Mapping of the old names of test cases to the new (actual)
120 return self.environment[u"mapping"]
124 """Getter - Ignore list.
126 :returns: List of ignored test cases.
129 return self.environment[u"ignore"]
133 """Getter - Alerting.
137 :returns: Specification of alerts.
140 return self.environment[u"alerting"]
144 """Getter - specification - inputs.
150 return self._specification[u"input"]
153 def input(self, new_value):
154 """Setter - specification - inputs.
156 :param new_value: New value to be set.
157 :type new_value: dict
159 self._specification[u"input"] = new_value
161 def add_build(self, job, build):
162 """Add a build to the list of builds if it does not exist there.
164 :param job: The job which run the build.
165 :param build: The build to be added.
169 if self.input.get(job, None) is None:
170 self.input[job] = list()
171 for existing_build in self.input[job]:
172 if existing_build[u"build"] == build[u"build"]:
175 self.input[job].append(build)
179 """Getter - specification - output formats and versions to be generated.
181 - versions: full, ...
183 :returns: Outputs to be generated.
186 return self._specification[u"output"]
190 """Getter - tables to be generated.
192 :returns: List of specifications of tables to be generated.
195 return self._specification.get(u"tables", list())
199 """Getter - plots to be generated.
201 :returns: List of specifications of plots to be generated.
204 return self._specification.get(u"plots", list())
208 """Getter - files to be generated.
210 :returns: List of specifications of files to be generated.
213 return self._specification.get(u"files", list())
217 """Getter - Continuous Performance Trending and Analysis to be
220 :returns: List of specifications of Continuous Performance Trending and
221 Analysis to be generated.
224 return self._specification[u"cpta"]
226 def set_input_state(self, job, build_nr, state):
227 """Set the state of the input.
229 :param job: Job name.
230 :param build_nr: Build number.
231 :param state: The new input state.
235 :raises: PresentationError if wrong job and/or build is provided.
239 for build in self.input[job]:
240 if build[u"build"] == build_nr:
241 build[u"status"] = state
244 raise PresentationError(
245 f"Build {build_nr} is not defined for job {job} in "
246 f"specification file."
249 raise PresentationError(
250 f"Job {job} and build {build_nr} is not defined in "
251 f"specification file."
254 def set_input_file_name(self, job, build_nr, file_name):
255 """Set the file name for the input.
257 :param job: Job name.
258 :param build_nr: Build number.
259 :param file_name: The new file name.
263 :raises: PresentationError if wrong job and/or build is provided.
267 for build in self.input[job]:
268 if build[u"build"] == build_nr:
269 build[u"file-name"] = file_name
272 raise PresentationError(
273 f"Build {build_nr} is not defined for job {job} in "
274 f"specification file."
277 raise PresentationError(
278 f"Job {job} and build {build_nr} is not defined in "
279 f"specification file."
282 def _get_build_number(self, job, build_type):
283 """Get the number of the job defined by its name:
284 - lastSuccessfulBuild
287 :param job: Job name.
288 :param build_type: Build type:
289 - lastSuccessfulBuild
292 :raises PresentationError: If it is not possible to get the build
294 :returns: The build number.
298 # defined as a range <start, end>
299 if build_type == u"lastSuccessfulBuild":
300 # defined as a range <start, lastSuccessfulBuild>
301 ret_code, build_nr, _ = get_last_successful_build_nr(
302 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
303 elif build_type == u"lastCompletedBuild":
304 # defined as a range <start, lastCompletedBuild>
305 ret_code, build_nr, _ = get_last_completed_build_number(
306 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
308 raise PresentationError(f"Not supported build type: {build_type}")
310 raise PresentationError(
311 f"Not possible to get the build number of {job}."
314 build_nr = int(build_nr)
316 except ValueError as err:
317 raise PresentationError(
318 f"Not possible to get the build number of {job}. Reason:\n"
322 def _get_type_index(self, item_type):
323 """Get index of item type (environment, input, output, ...) in
324 specification YAML file.
326 :param item_type: Item type: Top level items in specification YAML file,
327 e.g.: environment, input, output.
329 :returns: Index of the given item type.
334 for item in self._cfg_yaml:
335 if item[u"type"] == item_type:
340 def _find_tag(self, text):
341 """Find the first tag in the given text. The tag is enclosed by the
342 TAG_OPENER and TAG_CLOSER.
344 :param text: Text to be searched.
346 :returns: The tag, or None if not found.
350 start = text.index(self.TAG_OPENER)
351 end = text.index(self.TAG_CLOSER, start + 1) + 1
352 return text[start:end]
356 def _replace_tags(self, data, src_data=None):
357 """Replace tag(s) in the data by their values.
359 :param data: The data where the tags will be replaced by their values.
360 :param src_data: Data where the tags are defined. It is dictionary where
361 the key is the tag and the value is the tag value. If not given,
362 'data' is used instead.
363 :type data: str, list or dict
365 :returns: Data with the tags replaced.
366 :rtype: str, list or dict
367 :raises: PresentationError if it is not possible to replace the tag or
368 the data is not the supported data type (str, list or dict).
374 if isinstance(data, str):
375 tag = self._find_tag(data)
377 data = data.replace(tag, src_data[tag[1:-1]])
380 if isinstance(data, list):
383 new_list.append(self._replace_tags(item, src_data))
386 if isinstance(data, dict):
388 for key, value in data.items():
389 tag = self._find_tag(value)
392 data[key] = value.replace(tag, src_data[tag[1:-1]])
395 raise PresentationError(
396 f"Not possible to replace the tag {tag}"
399 self._replace_tags(data, src_data)
402 raise PresentationError(u"Replace tags: Not supported data type.")
404 def _parse_env(self):
405 """Parse environment specification in the specification YAML file.
408 logging.info(u"Parsing specification: ENVIRONMENT")
410 idx = self._get_type_index(u"environment")
414 self._specification[u"environment"][u"spec-files"] = \
415 self._cfg_yaml[idx].get(u"spec-files", None)
418 self._specification[u"environment"][u"paths"] = \
419 self._replace_tags(self._cfg_yaml[idx][u"paths"])
421 self._specification[u"environment"][u"paths"] = None
423 self._specification[u"environment"][u"data-sources"] = \
424 self._cfg_yaml[idx].get(u"data-sources", tuple())
426 for source in self._specification[u"environment"][u"data-sources"]:
427 source[u"successful-downloads"] = 0
429 self._specification[u"environment"][u"make-dirs"] = \
430 self._cfg_yaml[idx].get(u"make-dirs", None)
432 self._specification[u"environment"][u"remove-dirs"] = \
433 self._cfg_yaml[idx].get(u"remove-dirs", None)
435 self._specification[u"environment"][u"build-dirs"] = \
436 self._cfg_yaml[idx].get(u"build-dirs", None)
438 self._specification[u"environment"][u"testbeds"] = \
439 self._cfg_yaml[idx].get(u"testbeds", None)
441 self._specification[u"environment"][u"limits"] = \
442 self._cfg_yaml[idx].get(u"limits", None)
444 self._specification[u"environment"][u"frequency"] = \
445 self._cfg_yaml[idx].get(u"frequency", dict())
447 self._specification[u"environment"][u"urls"] = \
448 self._cfg_yaml[idx].get(u"urls", None)
450 self._specification[u"environment"][u"archive-inputs"] = \
451 self._cfg_yaml[idx].get(u"archive-inputs", False)
453 self._specification[u"environment"][u"reverse-input"] = \
454 self._cfg_yaml[idx].get(u"reverse-input", False)
456 self._specification[u"environment"][u"time-period"] = \
457 self._cfg_yaml[idx].get(u"time-period", None)
459 self._specification[u"environment"][u"alerting"] = \
460 self._cfg_yaml[idx].get(u"alerting", None)
462 self._specification[u"environment"][u"mapping-file"] = \
463 self._cfg_yaml[idx].get(u"mapping-file", None)
465 self._specification[u"environment"][u"ignore-list"] = \
466 self._cfg_yaml[idx].get(u"ignore-list", None)
469 self._load_mapping_table()
472 self._load_ignore_list()
474 def _parse_layouts(self):
475 """Parse layouts specification in the specification YAML file.
478 logging.info(u"Parsing specification: LAYOUTS")
480 idx = self._get_type_index(u"layouts")
485 self._specification[u"layouts"] = self._cfg_yaml[idx]
487 raise PresentationError(u"No layouts defined.")
489 def _parse_data_sets(self):
490 """Parse data sets specification in the specification YAML file.
493 logging.info(u"Parsing specification: DATA SETS")
495 idx = self._get_type_index(u"data-sets")
500 self._specification[u"data_sets"] = self._cfg_yaml[idx]
502 raise PresentationError(u"No Data sets defined.")
504 # Replace ranges by lists
505 for set_name, data_set in self.data_sets.items():
506 if not isinstance(data_set, dict):
508 for job, builds in data_set.items():
511 if isinstance(builds, dict):
512 build_end = builds.get(u"end", None)
513 max_builds = builds.get(u"max-builds", None)
514 reverse = builds.get(u"reverse", False)
516 build_end = int(build_end)
518 # defined as a range <start, build_type>
519 build_end = self._get_build_number(job, build_end)
520 builds = list(range(builds[u"start"], build_end + 1))
521 if max_builds and max_builds < len(builds):
522 builds = builds[-max_builds:]
525 self.data_sets[set_name][job] = builds
526 elif isinstance(builds, list):
527 for idx, item in enumerate(builds):
529 builds[idx] = int(item)
531 # defined as a range <build_type>
532 builds[idx] = self._get_build_number(job, item)
534 # Add sub-sets to sets (only one level):
535 for set_name, data_set in self.data_sets.items():
536 if isinstance(data_set, list):
538 for item in data_set:
540 for key, val in self.data_sets[item].items():
543 raise PresentationError(
544 f"Data set {item} is not defined."
546 self.data_sets[set_name] = new_set
548 def _load_mapping_table(self):
549 """Load a mapping table if it is specified. If not, use empty dict.
552 mapping_file_name = self.environment.get(u"mapping-file", None)
553 if mapping_file_name:
555 with open(mapping_file_name, u'r') as mfile:
556 mapping = load(mfile, Loader=FullLoader)
557 # Make sure everything is lowercase
558 self.environment[u"mapping"] = \
559 {key.lower(): val.lower() for key, val in
561 logging.debug(f"Loaded mapping table:\n{mapping}")
562 except (YAMLError, IOError) as err:
563 raise PresentationError(
564 msg=f"An error occurred while parsing the mapping file "
565 f"{mapping_file_name}",
569 self.environment[u"mapping"] = dict()
571 def _load_ignore_list(self):
572 """Load an ignore list if it is specified. If not, use empty list.
575 ignore_list_name = self.environment.get(u"ignore-list", None)
578 with open(ignore_list_name, u'r') as ifile:
579 ignore = load(ifile, Loader=FullLoader)
580 # Make sure everything is lowercase
581 self.environment[u"ignore"] = \
582 [item.lower() for item in ignore]
583 logging.debug(f"Loaded ignore list:\n{ignore}")
584 except (YAMLError, IOError) as err:
585 raise PresentationError(
586 msg=f"An error occurred while parsing the ignore list file "
587 f"{ignore_list_name}.",
591 self.environment[u"ignore"] = list()
593 def _parse_output(self):
594 """Parse output specification in the specification YAML file.
596 :raises: PresentationError if there is no output defined.
599 logging.info(u"Parsing specification: OUTPUT")
601 idx = self._get_type_index(u"output")
603 raise PresentationError(u"No output defined.")
606 self._specification[u"output"] = self._cfg_yaml[idx]
607 except (KeyError, IndexError):
608 raise PresentationError(u"No output defined.")
610 def _parse_static(self):
611 """Parse specification of the static content in the specification YAML
615 logging.info(u"Parsing specification: STATIC CONTENT")
617 idx = self._get_type_index(u"static")
619 logging.warning(u"No static content specified.")
620 self._specification[u"static"] = dict()
623 for key, value in self._cfg_yaml[idx].items():
624 if isinstance(value, str):
626 self._cfg_yaml[idx][key] = self._replace_tags(
627 value, self._specification[u"environment"][u"paths"])
631 self._specification[u"static"] = self._cfg_yaml[idx]
633 def _parse_elements_tables(self, table):
634 """Parse tables from the specification YAML file.
636 :param table: Table to be parsed from the specification file.
638 :raises PresentationError: If wrong data set is used.
642 table[u"template"] = self._replace_tags(
644 self._specification[u"environment"][u"paths"])
650 for item in (u"reference", u"compare"):
651 if table.get(item, None):
652 data_set = table[item].get(u"data", None)
653 if isinstance(data_set, str):
654 table[item][u"data"] = self.data_sets[data_set]
655 data_set = table[item].get(u"data-replacement", None)
656 if isinstance(data_set, str):
657 table[item][u"data-replacement"] = \
658 self.data_sets[data_set]
660 if table.get(u"columns", None):
661 for i in range(len(table[u"columns"])):
662 data_set = table[u"columns"][i].get(u"data-set", None)
663 if isinstance(data_set, str):
664 table[u"columns"][i][u"data-set"] = \
665 self.data_sets[data_set]
666 data_set = table[u"columns"][i].get(
667 u"data-replacement", None)
668 if isinstance(data_set, str):
669 table[u"columns"][i][u"data-replacement"] = \
670 self.data_sets[data_set]
672 if table.get(u"lines", None):
673 for i in range(len(table[u"lines"])):
674 data_set = table[u"lines"][i].get(u"data-set", None)
675 if isinstance(data_set, str):
676 table[u"lines"][i][u"data-set"] = \
677 self.data_sets[data_set]
680 raise PresentationError(
681 f"Wrong set '{data_set}' used in {table.get(u'title', u'')}."
684 self._specification[u"tables"].append(table)
686 def _parse_elements_plots(self, plot):
687 """Parse plots from the specification YAML file.
689 :param plot: Plot to be parsed from the specification file.
691 :raises PresentationError: If plot layout is not defined.
694 # Add layout to the plots:
695 layout = plot[u"layout"].get(u"layout", None)
696 if layout is not None:
697 plot[u"layout"].pop(u"layout")
699 for key, val in self.layouts[layout].items():
700 plot[u"layout"][key] = val
702 raise PresentationError(f"Layout {layout} is not defined.")
703 self._specification[u"plots"].append(plot)
705 def _parse_elements_files(self, file):
706 """Parse files from the specification YAML file.
708 :param file: File to be parsed from the specification file.
713 file[u"dir-tables"] = self._replace_tags(
715 self._specification[u"environment"][u"paths"])
718 self._specification[u"files"].append(file)
720 def _parse_elements_cpta(self, cpta):
721 """Parse cpta from the specification YAML file.
723 :param cpta: cpta to be parsed from the specification file.
725 :raises PresentationError: If wrong data set is used or if plot layout
729 for plot in cpta[u"plots"]:
730 # Add layout to the plots:
731 layout = plot.get(u"layout", None)
732 if layout is not None:
734 plot[u"layout"] = self.layouts[layout]
736 raise PresentationError(f"Layout {layout} is not defined.")
738 if isinstance(plot.get(u"data", None), str):
739 data_set = plot[u"data"]
741 plot[u"data"] = self.data_sets[data_set]
743 raise PresentationError(
744 f"Data set {data_set} is not defined."
746 self._specification[u"cpta"] = cpta
748 def _parse_elements(self):
749 """Parse elements (tables, plots, ..) specification in the specification
753 logging.info(u"Parsing specification: ELEMENTS")
756 for element in self._cfg_yaml:
760 element[u"output-file"] = self._replace_tags(
761 element[u"output-file"],
762 self.environment[u"paths"]
768 element[u"input-file"] = self._replace_tags(
769 element[u"input-file"],
770 self.environment[u"paths"]
776 element[u"output-file-links"] = self._replace_tags(
777 element[u"output-file-links"],
778 self.environment[u"paths"]
783 # Add data sets to the elements:
784 if isinstance(element.get(u"data", None), str):
785 data_set = element[u"data"]
787 element[u"data"] = self.data_sets[data_set]
789 raise PresentationError(
790 f"Data set {data_set} is not defined."
792 elif isinstance(element.get(u"data", None), list):
794 for item in element[u"data"]:
796 new_list.append(self.data_sets[item])
798 raise PresentationError(
799 f"Data set {item} is not defined."
801 element[u"data"] = new_list
804 if element[u"type"] == u"table":
805 logging.info(f" {count:3d} Processing a table ...")
806 self._parse_elements_tables(element)
808 elif element[u"type"] == u"plot":
809 logging.info(f" {count:3d} Processing a plot ...")
810 self._parse_elements_plots(element)
812 elif element[u"type"] == u"file":
813 logging.info(f" {count:3d} Processing a file ...")
814 self._parse_elements_files(element)
816 elif element[u"type"] == u"cpta":
818 f" {count:3d} Processing Continuous Performance Trending "
821 self._parse_elements_cpta(element)
824 def _prepare_input(self):
825 """Use information from data sets and generate list of jobs and builds
829 logging.info(u"Parsing specification: INPUT")
831 idx = self._get_type_index(u"input")
833 logging.info(u"Creating the list of inputs from data sets.")
834 for data_set in self.data_sets.values():
835 if data_set == "data-sets":
837 for job, builds in data_set.items():
849 logging.info(u"Reading pre-defined inputs.")
850 for job, builds in self._cfg_yaml[idx][u"builds"].items():
862 if self.environment[u"reverse-input"]:
863 for builds in self.input.values():
864 builds.sort(key=lambda k: k[u"build"], reverse=True)
866 def read_specification(self):
867 """Parse specification in the specification YAML files.
869 :raises: PresentationError if an error occurred while parsing the
873 # It always starts with environment.yaml file, it must be present.
874 spec_file = join(self._cfg_dir, u"environment.yaml")
875 logging.info(f"Reading {spec_file}")
876 if not exists(spec_file):
877 raise PresentationError(f"The file {spec_file} does not exist.")
879 with open(spec_file, u"r") as file_read:
881 self._cfg_yaml = load(file_read, Loader=FullLoader)
882 except YAMLError as err:
883 raise PresentationError(
884 f"An error occurred while parsing the specification file "
889 # Load the other specification files specified in the environment.yaml
890 idx = self._get_type_index(u"environment")
892 raise PresentationError(
893 f"No environment defined in the file {spec_file}"
895 for spec_file in self._cfg_yaml[idx].get(u"spec-files", tuple()):
896 logging.info(f"Reading {spec_file}")
897 if not exists(spec_file):
898 raise PresentationError(f"The file {spec_file} does not exist.")
900 with open(spec_file, u"r") as file_read:
902 spec = load(file_read, Loader=FullLoader)
903 except YAMLError as err:
904 raise PresentationError(
905 f"An error occurred while parsing the specification "
910 self._cfg_yaml.extend(spec)
913 self._parse_layouts()
914 self._parse_data_sets()
917 self._parse_elements()
918 self._prepare_input()
920 logging.debug(f"Specification: \n{pformat(self.specification)}")