1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from pprint import pformat
23 from yaml import load, FullLoader, YAMLError
25 from pal_errors import PresentationError
26 from pal_utils import (
27 get_last_successful_build_nr, get_last_completed_build_number
32 """Specification of Presentation and analytics layer.
34 - based on specification specified in the specification YAML file
35 - presentation and analytics layer is model driven
38 # Tags are used in specification YAML file and replaced while the file is
43 def __init__(self, cfg_file):
46 :param cfg_file: File handler for the specification YAML file.
47 :type cfg_file: BinaryIO
49 self._cfg_file = cfg_file
52 self._specification = {
53 u"environment": dict(),
54 u"configuration": dict(),
65 def specification(self):
66 """Getter - specification.
68 :returns: Specification.
71 return self._specification
74 def environment(self):
75 """Getter - environment.
77 :returns: Environment specification.
80 return self._specification[u"environment"]
83 def configuration(self):
84 """Getter - configuration.
86 :returns: Configuration of PAL.
89 return self._specification[u"configuration"]
93 """Getter - static content.
95 :returns: Static content specification.
98 return self._specification[u"static"]
104 :returns: Mapping of the old names of test cases to the new (actual)
108 return self._specification[u"configuration"][u"mapping"]
112 """Getter - Ignore list.
114 :returns: List of ignored test cases.
117 return self._specification[u"configuration"][u"ignore"]
121 """Getter - Alerting.
123 :returns: Specification of alerts.
126 return self._specification[u"configuration"][u"alerting"]
130 """Getter - specification - inputs.
136 return self._specification[u"input"]
140 """Getter - builds defined in specification.
142 :returns: Builds defined in the specification.
145 return self.input[u"builds"]
149 """Getter - specification - output formats and versions to be generated.
151 - versions: full, ...
153 :returns: Outputs to be generated.
156 return self._specification[u"output"]
160 """Getter - tables to be generated.
162 :returns: List of specifications of tables to be generated.
165 return self._specification[u"tables"]
169 """Getter - plots to be generated.
171 :returns: List of specifications of plots to be generated.
174 return self._specification[u"plots"]
178 """Getter - files to be generated.
180 :returns: List of specifications of files to be generated.
183 return self._specification[u"files"]
187 """Getter - Continuous Performance Trending and Analysis to be
190 :returns: List of specifications of Continuous Performance Trending and
191 Analysis to be generated.
194 return self._specification[u"cpta"]
196 def set_input_state(self, job, build_nr, state):
197 """Set the state of input
199 :param job: Job name.
200 :param build_nr: Build number.
201 :param state: The new input state.
205 :raises: PresentationError if wrong job and/or build is provided.
209 for build in self._specification[u"input"][u"builds"][job]:
210 if build[u"build"] == build_nr:
211 build[u"status"] = state
214 raise PresentationError(
215 f"Build {build_nr} is not defined for job {job} in "
216 f"specification file."
219 raise PresentationError(
220 f"Job {job} and build {build_nr} is not defined in "
221 f"specification file."
224 def set_input_file_name(self, job, build_nr, file_name):
225 """Set the state of input
227 :param job: Job name.
228 :param build_nr: Build number.
229 :param file_name: The new file name.
233 :raises: PresentationError if wrong job and/or build is provided.
237 for build in self._specification[u"input"][u"builds"][job]:
238 if build[u"build"] == build_nr:
239 build[u"file-name"] = file_name
242 raise PresentationError(
243 f"Build {build_nr} is not defined for job {job} in "
244 f"specification file."
247 raise PresentationError(
248 f"Job {job} and build {build_nr} is not defined in "
249 f"specification file."
252 def _get_build_number(self, job, build_type):
253 """Get the number of the job defined by its name:
254 - lastSuccessfulBuild
257 :param job: Job name.
258 :param build_type: Build type:
259 - lastSuccessfulBuild
262 :raises PresentationError: If it is not possible to get the build
264 :returns: The build number.
268 # defined as a range <start, end>
269 if build_type == u"lastSuccessfulBuild":
270 # defined as a range <start, lastSuccessfulBuild>
271 ret_code, build_nr, _ = get_last_successful_build_nr(
272 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
273 elif build_type == u"lastCompletedBuild":
274 # defined as a range <start, lastCompletedBuild>
275 ret_code, build_nr, _ = get_last_completed_build_number(
276 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
278 raise PresentationError(f"Not supported build type: {build_type}")
280 raise PresentationError(u"Not possible to get the number of the "
283 build_nr = int(build_nr)
285 except ValueError as err:
286 raise PresentationError(
287 f"Not possible to get the number of the build number. Reason:\n"
291 def _get_type_index(self, item_type):
292 """Get index of item type (environment, input, output, ...) in
293 specification YAML file.
295 :param item_type: Item type: Top level items in specification YAML file,
296 e.g.: environment, input, output.
298 :returns: Index of the given item type.
303 for item in self._cfg_yaml:
304 if item[u"type"] == item_type:
309 def _find_tag(self, text):
310 """Find the first tag in the given text. The tag is enclosed by the
311 TAG_OPENER and TAG_CLOSER.
313 :param text: Text to be searched.
315 :returns: The tag, or None if not found.
319 start = text.index(self.TAG_OPENER)
320 end = text.index(self.TAG_CLOSER, start + 1) + 1
321 return text[start:end]
325 def _replace_tags(self, data, src_data=None):
326 """Replace tag(s) in the data by their values.
328 :param data: The data where the tags will be replaced by their values.
329 :param src_data: Data where the tags are defined. It is dictionary where
330 the key is the tag and the value is the tag value. If not given,
331 'data' is used instead.
332 :type data: str, list or dict
334 :returns: Data with the tags replaced.
335 :rtype: str, list or dict
336 :raises: PresentationError if it is not possible to replace the tag or
337 the data is not the supported data type (str, list or dict).
343 if isinstance(data, str):
344 tag = self._find_tag(data)
346 data = data.replace(tag, src_data[tag[1:-1]])
349 if isinstance(data, list):
352 new_list.append(self._replace_tags(item, src_data))
355 if isinstance(data, dict):
357 for key, value in data.items():
358 tag = self._find_tag(value)
361 data[key] = value.replace(tag, src_data[tag[1:-1]])
364 raise PresentationError(
365 f"Not possible to replace the tag {tag}"
368 self._replace_tags(data, src_data)
371 raise PresentationError(u"Replace tags: Not supported data type.")
373 def _parse_env(self):
374 """Parse environment specification in the specification YAML file.
377 logging.info(u"Parsing specification file: environment ...")
379 idx = self._get_type_index(u"environment")
384 self._specification[u"environment"][u"configuration"] = \
385 self._cfg_yaml[idx][u"configuration"]
387 self._specification[u"environment"][u"configuration"] = None
390 self._specification[u"environment"][u"paths"] = \
391 self._replace_tags(self._cfg_yaml[idx][u"paths"])
393 self._specification[u"environment"][u"paths"] = None
396 self._specification[u"environment"][u"urls"] = \
397 self._cfg_yaml[idx][u"urls"]
399 self._specification[u"environment"][u"urls"] = None
402 self._specification[u"environment"][u"make-dirs"] = \
403 self._cfg_yaml[idx][u"make-dirs"]
405 self._specification[u"environment"][u"make-dirs"] = None
408 self._specification[u"environment"][u"remove-dirs"] = \
409 self._cfg_yaml[idx][u"remove-dirs"]
411 self._specification[u"environment"][u"remove-dirs"] = None
414 self._specification[u"environment"][u"build-dirs"] = \
415 self._cfg_yaml[idx][u"build-dirs"]
417 self._specification[u"environment"][u"build-dirs"] = None
420 self._specification[u"environment"][u"testbeds"] = \
421 self._cfg_yaml[idx][u"testbeds"]
423 self._specification[u"environment"][u"testbeds"] = None
425 logging.info(u"Done.")
427 def _load_mapping_table(self):
428 """Load a mapping table if it is specified. If not, use empty list.
431 mapping_file_name = self._specification[u"configuration"].\
432 get(u"mapping-file", None)
433 if mapping_file_name:
435 with open(mapping_file_name, u'r') as mfile:
436 mapping = load(mfile, Loader=FullLoader)
437 # Make sure everything is lowercase
438 self._specification[u"configuration"][u"mapping"] = \
439 {key.lower(): val.lower() for key, val in
441 logging.debug(f"Loaded mapping table:\n{mapping}")
442 except (YAMLError, IOError) as err:
443 raise PresentationError(
444 msg=f"An error occurred while parsing the mapping file "
445 f"{mapping_file_name}",
449 self._specification[u"configuration"][u"mapping"] = dict()
451 def _load_ignore_list(self):
452 """Load an ignore list if it is specified. If not, use empty list.
455 ignore_list_name = self._specification[u"configuration"].\
456 get(u"ignore-list", None)
459 with open(ignore_list_name, u'r') as ifile:
460 ignore = load(ifile, Loader=FullLoader)
461 # Make sure everything is lowercase
462 self._specification[u"configuration"][u"ignore"] = \
463 [item.lower() for item in ignore]
464 logging.debug(f"Loaded ignore list:\n{ignore}")
465 except (YAMLError, IOError) as err:
466 raise PresentationError(
467 msg=f"An error occurred while parsing the ignore list file "
468 f"{ignore_list_name}.",
472 self._specification[u"configuration"][u"ignore"] = list()
474 def _parse_configuration(self):
475 """Parse configuration of PAL in the specification YAML file.
478 logging.info(u"Parsing specification file: configuration ...")
480 idx = self._get_type_index("configuration")
483 u"No configuration information in the specification file."
488 self._specification[u"configuration"] = self._cfg_yaml[idx]
490 raise PresentationError(u"No configuration defined.")
492 # Data sets: Replace ranges by lists
493 for set_name, data_set in self.configuration[u"data-sets"].items():
494 if not isinstance(data_set, dict):
496 for job, builds in data_set.items():
499 if isinstance(builds, dict):
500 build_end = builds.get(u"end", None)
502 build_end = int(build_end)
504 # defined as a range <start, build_type>
505 build_end = self._get_build_number(job, build_end)
506 builds = [x for x in range(builds[u"start"],
508 if x not in builds.get(u"skip", list())]
509 self.configuration[u"data-sets"][set_name][job] = builds
510 elif isinstance(builds, list):
511 for idx, item in enumerate(builds):
513 builds[idx] = int(item)
515 # defined as a range <build_type>
516 builds[idx] = self._get_build_number(job, item)
518 # Data sets: add sub-sets to sets (only one level):
519 for set_name, data_set in self.configuration[u"data-sets"].items():
520 if isinstance(data_set, list):
522 for item in data_set:
524 for key, val in self.configuration[u"data-sets"][item].\
528 raise PresentationError(
529 f"Data set {item} is not defined in "
530 f"the configuration section."
532 self.configuration[u"data-sets"][set_name] = new_set
535 self._load_mapping_table()
538 self._load_ignore_list()
540 logging.info(u"Done.")
542 def _parse_input(self):
543 """Parse input specification in the specification YAML file.
545 :raises: PresentationError if there are no data to process.
548 logging.info(u"Parsing specification file: input ...")
550 idx = self._get_type_index(u"input")
552 raise PresentationError(u"No data to process.")
555 for key, value in self._cfg_yaml[idx][u"general"].items():
556 self._specification[u"input"][key] = value
557 self._specification[u"input"][u"builds"] = dict()
559 for job, builds in self._cfg_yaml[idx][u"builds"].items():
561 if isinstance(builds, dict):
562 build_end = builds.get(u"end", None)
564 build_end = int(build_end)
566 # defined as a range <start, build_type>
567 build_end = self._get_build_number(job, build_end)
568 builds = [x for x in range(builds[u"start"],
570 if x not in builds.get(u"skip", list())]
571 self._specification[u"input"][u"builds"][job] = list()
573 self._specification[u"input"][u"builds"][job]. \
574 append({u"build": build, u"status": None})
578 f"No build is defined for the job {job}. Trying to "
579 f"continue without it."
582 raise PresentationError(u"No data to process.")
584 logging.info(u"Done.")
586 def _parse_output(self):
587 """Parse output specification in the specification YAML file.
589 :raises: PresentationError if there is no output defined.
592 logging.info(u"Parsing specification file: output ...")
594 idx = self._get_type_index(u"output")
596 raise PresentationError(u"No output defined.")
599 self._specification[u"output"] = self._cfg_yaml[idx]
600 except (KeyError, IndexError):
601 raise PresentationError(u"No output defined.")
603 logging.info(u"Done.")
605 def _parse_static(self):
606 """Parse specification of the static content in the specification YAML
610 logging.info(u"Parsing specification file: static content ...")
612 idx = self._get_type_index(u"static")
614 logging.warning(u"No static content specified.")
616 for key, value in self._cfg_yaml[idx].items():
617 if isinstance(value, str):
619 self._cfg_yaml[idx][key] = self._replace_tags(
620 value, self._specification[u"environment"][u"paths"])
624 self._specification[u"static"] = self._cfg_yaml[idx]
626 logging.info(u"Done.")
628 def _parse_elements_tables(self, table):
629 """Parse tables from the specification YAML file.
631 :param table: Table to be parsed from the specification file.
633 :raises PresentationError: If wrong data set is used.
637 table[u"template"] = self._replace_tags(
639 self._specification[u"environment"][u"paths"])
645 for item in (u"reference", u"compare"):
646 if table.get(item, None):
647 data_set = table[item].get(u"data", None)
648 if isinstance(data_set, str):
649 table[item][u"data"] = \
650 self.configuration[u"data-sets"][data_set]
651 data_set = table[item].get(u"data-replacement", None)
652 if isinstance(data_set, str):
653 table[item][u"data-replacement"] = \
654 self.configuration[u"data-sets"][data_set]
656 if table.get(u"history", None):
657 for i in range(len(table[u"history"])):
658 data_set = table[u"history"][i].get(u"data", None)
659 if isinstance(data_set, str):
660 table[u"history"][i][u"data"] = \
661 self.configuration[u"data-sets"][data_set]
662 data_set = table[u"history"][i].get(
663 u"data-replacement", None)
664 if isinstance(data_set, str):
665 table[u"history"][i][u"data-replacement"] = \
666 self.configuration[u"data-sets"][data_set]
668 raise PresentationError(
669 f"Wrong data set used in {table.get(u'title', u'')}."
672 self._specification[u"tables"].append(table)
674 def _parse_elements_plots(self, plot):
675 """Parse plots from the specification YAML file.
677 :param plot: Plot to be parsed from the specification file.
679 :raises PresentationError: If plot layout is not defined.
682 # Add layout to the plots:
683 layout = plot[u"layout"].get(u"layout", None)
684 if layout is not None:
685 plot[u"layout"].pop(u"layout")
687 for key, val in (self.configuration[u"plot-layouts"]
689 plot[u"layout"][key] = val
691 raise PresentationError(
692 f"Layout {layout} is not defined in the "
693 f"configuration section."
695 self._specification[u"plots"].append(plot)
697 def _parse_elements_files(self, file):
698 """Parse files from the specification YAML file.
700 :param file: File to be parsed from the specification file.
705 file[u"dir-tables"] = self._replace_tags(
707 self._specification[u"environment"][u"paths"])
710 self._specification[u"files"].append(file)
712 def _parse_elements_cpta(self, cpta):
713 """Parse cpta from the specification YAML file.
715 :param cpta: cpta to be parsed from the specification file.
717 :raises PresentationError: If wrong data set is used or if plot layout
721 for plot in cpta[u"plots"]:
722 # Add layout to the plots:
723 layout = plot.get(u"layout", None)
724 if layout is not None:
727 self.configuration[u"plot-layouts"][layout]
729 raise PresentationError(
730 f"Layout {layout} is not defined in the "
731 f"configuration section."
734 if isinstance(plot.get(u"data", None), str):
735 data_set = plot[u"data"]
738 self.configuration[u"data-sets"][data_set]
740 raise PresentationError(
741 f"Data set {data_set} is not defined in "
742 f"the configuration section."
744 self._specification[u"cpta"] = cpta
746 def _parse_elements(self):
747 """Parse elements (tables, plots, ..) specification in the specification
751 logging.info(u"Parsing specification file: elements ...")
754 for element in self._cfg_yaml:
758 element[u"output-file"] = self._replace_tags(
759 element[u"output-file"],
760 self._specification[u"environment"][u"paths"])
765 element[u"input-file"] = self._replace_tags(
766 element[u"input-file"],
767 self._specification[u"environment"][u"paths"])
772 element[u"output-file-links"] = self._replace_tags(
773 element[u"output-file-links"],
774 self._specification[u"environment"][u"paths"])
778 # Add data sets to the elements:
779 if isinstance(element.get(u"data", None), str):
780 data_set = element[u"data"]
783 self.configuration[u"data-sets"][data_set]
785 raise PresentationError(
786 f"Data set {data_set} is not defined in the "
787 f"configuration section."
789 elif isinstance(element.get(u"data", None), list):
791 for item in element[u"data"]:
794 self.configuration[u"data-sets"][item]
797 raise PresentationError(
798 f"Data set {item} is not defined in the "
799 f"configuration section."
801 element[u"data"] = new_list
804 if element[u"type"] == u"table":
806 logging.info(f" {count:3d} Processing a table ...")
807 self._parse_elements_tables(element)
810 elif element[u"type"] == u"plot":
812 logging.info(f" {count:3d} Processing a plot ...")
813 self._parse_elements_plots(element)
816 elif element[u"type"] == u"file":
818 logging.info(f" {count:3d} Processing a file ...")
819 self._parse_elements_files(element)
822 elif element[u"type"] == u"cpta":
825 f" {count:3d} Processing Continuous Performance Trending "
828 self._parse_elements_cpta(element)
831 logging.info(u"Done.")
833 def read_specification(self):
834 """Parse specification in the specification YAML file.
836 :raises: PresentationError if an error occurred while parsing the
840 self._cfg_yaml = load(self._cfg_file, Loader=FullLoader)
841 except YAMLError as err:
842 raise PresentationError(msg=u"An error occurred while parsing the "
843 u"specification file.",
847 self._parse_configuration()
851 self._parse_elements()
853 logging.debug(f"Specification: \n{pformat(self._specification)}")