1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from pprint import pformat
23 from yaml import load, FullLoader, YAMLError
25 from pal_errors import PresentationError
26 from pal_utils import (
27 get_last_successful_build_nr, get_last_completed_build_number
32 """Specification of Presentation and analytics layer.
34 - based on specification specified in the specification YAML file
35 - presentation and analytics layer is model driven
38 # Tags are used in specification YAML file and replaced while the file is
43 def __init__(self, cfg_file):
46 :param cfg_file: File handler for the specification YAML file.
47 :type cfg_file: BinaryIO
49 self._cfg_file = cfg_file
52 self._specification = {
53 u"environment": dict(),
54 u"configuration": dict(),
65 def specification(self):
66 """Getter - specification.
68 :returns: Specification.
71 return self._specification
74 def environment(self):
75 """Getter - environment.
77 :returns: Environment specification.
80 return self._specification[u"environment"]
83 def configuration(self):
84 """Getter - configuration.
86 :returns: Configuration of PAL.
89 return self._specification[u"configuration"]
93 """Getter - static content.
95 :returns: Static content specification.
98 return self._specification[u"static"]
104 :returns: Mapping of the old names of test cases to the new (actual)
108 return self._specification[u"configuration"][u"mapping"]
112 """Getter - Ignore list.
114 :returns: List of ignored test cases.
117 return self._specification[u"configuration"][u"ignore"]
121 """Getter - Alerting.
123 :returns: Specification of alerts.
126 return self._specification[u"configuration"][u"alerting"]
130 """Getter - specification - inputs.
136 return self._specification[u"input"]
139 def input(self, new_value):
140 """Setter - specification - inputs.
142 :param new_value: New value to be set.
143 :type new_value: dict
145 self._specification[u"input"] = new_value
149 """Getter - builds defined in specification.
151 :returns: Builds defined in the specification.
154 return self.input[u"builds"]
157 def builds(self, new_value):
158 """Setter - builds defined in specification.
160 :param new_value: New value to be set.
161 :type new_value: dict
163 self.input[u"builds"] = new_value
165 def add_build(self, job, build):
166 """Add a build to the specification.
168 :param job: The job which run the build.
169 :param build: The build to be added.
173 if self._specification[u"input"][u"builds"].get(job, None) is None:
174 self._specification[u"input"][u"builds"][job] = list()
175 self._specification[u"input"][u"builds"][job].append(build)
179 """Getter - specification - output formats and versions to be generated.
181 - versions: full, ...
183 :returns: Outputs to be generated.
186 return self._specification[u"output"]
190 """Getter - tables to be generated.
192 :returns: List of specifications of tables to be generated.
195 return self._specification[u"tables"]
199 """Getter - plots to be generated.
201 :returns: List of specifications of plots to be generated.
204 return self._specification[u"plots"]
208 """Getter - files to be generated.
210 :returns: List of specifications of files to be generated.
213 return self._specification[u"files"]
217 """Getter - Continuous Performance Trending and Analysis to be
220 :returns: List of specifications of Continuous Performance Trending and
221 Analysis to be generated.
224 return self._specification[u"cpta"]
226 def set_input_state(self, job, build_nr, state):
227 """Set the state of input
229 :param job: Job name.
230 :param build_nr: Build number.
231 :param state: The new input state.
235 :raises: PresentationError if wrong job and/or build is provided.
239 for build in self._specification[u"input"][u"builds"][job]:
240 if build[u"build"] == build_nr:
241 build[u"status"] = state
244 raise PresentationError(
245 f"Build {build_nr} is not defined for job {job} in "
246 f"specification file."
249 raise PresentationError(
250 f"Job {job} and build {build_nr} is not defined in "
251 f"specification file."
254 def set_input_file_name(self, job, build_nr, file_name):
255 """Set the state of input
257 :param job: Job name.
258 :param build_nr: Build number.
259 :param file_name: The new file name.
263 :raises: PresentationError if wrong job and/or build is provided.
267 for build in self._specification[u"input"][u"builds"][job]:
268 if build[u"build"] == build_nr:
269 build[u"file-name"] = file_name
272 raise PresentationError(
273 f"Build {build_nr} is not defined for job {job} in "
274 f"specification file."
277 raise PresentationError(
278 f"Job {job} and build {build_nr} is not defined in "
279 f"specification file."
282 def _get_build_number(self, job, build_type):
283 """Get the number of the job defined by its name:
284 - lastSuccessfulBuild
287 :param job: Job name.
288 :param build_type: Build type:
289 - lastSuccessfulBuild
292 :raises PresentationError: If it is not possible to get the build
294 :returns: The build number.
298 # defined as a range <start, end>
299 if build_type == u"lastSuccessfulBuild":
300 # defined as a range <start, lastSuccessfulBuild>
301 ret_code, build_nr, _ = get_last_successful_build_nr(
302 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
303 elif build_type == u"lastCompletedBuild":
304 # defined as a range <start, lastCompletedBuild>
305 ret_code, build_nr, _ = get_last_completed_build_number(
306 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
308 raise PresentationError(f"Not supported build type: {build_type}")
310 raise PresentationError(u"Not possible to get the number of the "
313 build_nr = int(build_nr)
315 except ValueError as err:
316 raise PresentationError(
317 f"Not possible to get the number of the build number. Reason:\n"
321 def _get_type_index(self, item_type):
322 """Get index of item type (environment, input, output, ...) in
323 specification YAML file.
325 :param item_type: Item type: Top level items in specification YAML file,
326 e.g.: environment, input, output.
328 :returns: Index of the given item type.
333 for item in self._cfg_yaml:
334 if item[u"type"] == item_type:
339 def _find_tag(self, text):
340 """Find the first tag in the given text. The tag is enclosed by the
341 TAG_OPENER and TAG_CLOSER.
343 :param text: Text to be searched.
345 :returns: The tag, or None if not found.
349 start = text.index(self.TAG_OPENER)
350 end = text.index(self.TAG_CLOSER, start + 1) + 1
351 return text[start:end]
355 def _replace_tags(self, data, src_data=None):
356 """Replace tag(s) in the data by their values.
358 :param data: The data where the tags will be replaced by their values.
359 :param src_data: Data where the tags are defined. It is dictionary where
360 the key is the tag and the value is the tag value. If not given,
361 'data' is used instead.
362 :type data: str, list or dict
364 :returns: Data with the tags replaced.
365 :rtype: str, list or dict
366 :raises: PresentationError if it is not possible to replace the tag or
367 the data is not the supported data type (str, list or dict).
373 if isinstance(data, str):
374 tag = self._find_tag(data)
376 data = data.replace(tag, src_data[tag[1:-1]])
379 if isinstance(data, list):
382 new_list.append(self._replace_tags(item, src_data))
385 if isinstance(data, dict):
387 for key, value in data.items():
388 tag = self._find_tag(value)
391 data[key] = value.replace(tag, src_data[tag[1:-1]])
394 raise PresentationError(
395 f"Not possible to replace the tag {tag}"
398 self._replace_tags(data, src_data)
401 raise PresentationError(u"Replace tags: Not supported data type.")
403 def _parse_env(self):
404 """Parse environment specification in the specification YAML file.
407 logging.info(u"Parsing specification file: environment ...")
409 idx = self._get_type_index(u"environment")
414 self._specification[u"environment"][u"configuration"] = \
415 self._cfg_yaml[idx][u"configuration"]
417 self._specification[u"environment"][u"configuration"] = None
420 self._specification[u"environment"][u"paths"] = \
421 self._replace_tags(self._cfg_yaml[idx][u"paths"])
423 self._specification[u"environment"][u"paths"] = None
426 self._specification[u"environment"][u"urls"] = \
427 self._cfg_yaml[idx][u"urls"]
429 self._specification[u"environment"][u"urls"] = None
432 self._specification[u"environment"][u"make-dirs"] = \
433 self._cfg_yaml[idx][u"make-dirs"]
435 self._specification[u"environment"][u"make-dirs"] = None
438 self._specification[u"environment"][u"remove-dirs"] = \
439 self._cfg_yaml[idx][u"remove-dirs"]
441 self._specification[u"environment"][u"remove-dirs"] = None
444 self._specification[u"environment"][u"build-dirs"] = \
445 self._cfg_yaml[idx][u"build-dirs"]
447 self._specification[u"environment"][u"build-dirs"] = None
450 self._specification[u"environment"][u"testbeds"] = \
451 self._cfg_yaml[idx][u"testbeds"]
453 self._specification[u"environment"][u"testbeds"] = None
455 logging.info(u"Done.")
457 def _load_mapping_table(self):
458 """Load a mapping table if it is specified. If not, use empty list.
461 mapping_file_name = self._specification[u"configuration"].\
462 get(u"mapping-file", None)
463 if mapping_file_name:
465 with open(mapping_file_name, u'r') as mfile:
466 mapping = load(mfile, Loader=FullLoader)
467 # Make sure everything is lowercase
468 self._specification[u"configuration"][u"mapping"] = \
469 {key.lower(): val.lower() for key, val in
471 logging.debug(f"Loaded mapping table:\n{mapping}")
472 except (YAMLError, IOError) as err:
473 raise PresentationError(
474 msg=f"An error occurred while parsing the mapping file "
475 f"{mapping_file_name}",
479 self._specification[u"configuration"][u"mapping"] = dict()
481 def _load_ignore_list(self):
482 """Load an ignore list if it is specified. If not, use empty list.
485 ignore_list_name = self._specification[u"configuration"].\
486 get(u"ignore-list", None)
489 with open(ignore_list_name, u'r') as ifile:
490 ignore = load(ifile, Loader=FullLoader)
491 # Make sure everything is lowercase
492 self._specification[u"configuration"][u"ignore"] = \
493 [item.lower() for item in ignore]
494 logging.debug(f"Loaded ignore list:\n{ignore}")
495 except (YAMLError, IOError) as err:
496 raise PresentationError(
497 msg=f"An error occurred while parsing the ignore list file "
498 f"{ignore_list_name}.",
502 self._specification[u"configuration"][u"ignore"] = list()
504 def _parse_configuration(self):
505 """Parse configuration of PAL in the specification YAML file.
508 logging.info(u"Parsing specification file: configuration ...")
510 idx = self._get_type_index("configuration")
513 u"No configuration information in the specification file."
518 self._specification[u"configuration"] = self._cfg_yaml[idx]
520 raise PresentationError(u"No configuration defined.")
522 # Data sets: Replace ranges by lists
523 for set_name, data_set in self.configuration[u"data-sets"].items():
524 if not isinstance(data_set, dict):
526 for job, builds in data_set.items():
529 if isinstance(builds, dict):
530 build_end = builds.get(u"end", None)
532 build_end = int(build_end)
534 # defined as a range <start, build_type>
535 build_end = self._get_build_number(job, build_end)
536 builds = [x for x in range(builds[u"start"],
538 if x not in builds.get(u"skip", list())]
539 self.configuration[u"data-sets"][set_name][job] = builds
540 elif isinstance(builds, list):
541 for idx, item in enumerate(builds):
543 builds[idx] = int(item)
545 # defined as a range <build_type>
546 builds[idx] = self._get_build_number(job, item)
548 # Data sets: add sub-sets to sets (only one level):
549 for set_name, data_set in self.configuration[u"data-sets"].items():
550 if isinstance(data_set, list):
552 for item in data_set:
554 for key, val in self.configuration[u"data-sets"][item].\
558 raise PresentationError(
559 f"Data set {item} is not defined in "
560 f"the configuration section."
562 self.configuration[u"data-sets"][set_name] = new_set
565 self._load_mapping_table()
568 self._load_ignore_list()
570 logging.info(u"Done.")
572 def _parse_input(self):
573 """Parse input specification in the specification YAML file.
575 :raises: PresentationError if there are no data to process.
578 logging.info(u"Parsing specification file: input ...")
580 idx = self._get_type_index(u"input")
582 raise PresentationError(u"No data to process.")
585 for key, value in self._cfg_yaml[idx][u"general"].items():
586 self._specification[u"input"][key] = value
587 self._specification[u"input"][u"builds"] = dict()
589 for job, builds in self._cfg_yaml[idx][u"builds"].items():
591 if isinstance(builds, dict):
592 build_end = builds.get(u"end", None)
594 build_end = int(build_end)
596 # defined as a range <start, build_type>
597 build_end = self._get_build_number(job, build_end)
598 builds = [x for x in range(builds[u"start"],
600 if x not in builds.get(u"skip", list())]
601 self._specification[u"input"][u"builds"][job] = list()
603 self._specification[u"input"][u"builds"][job]. \
604 append({u"build": build, u"status": None})
608 f"No build is defined for the job {job}. Trying to "
609 f"continue without it."
612 raise PresentationError(u"No data to process.")
614 logging.info(u"Done.")
616 def _parse_output(self):
617 """Parse output specification in the specification YAML file.
619 :raises: PresentationError if there is no output defined.
622 logging.info(u"Parsing specification file: output ...")
624 idx = self._get_type_index(u"output")
626 raise PresentationError(u"No output defined.")
629 self._specification[u"output"] = self._cfg_yaml[idx]
630 except (KeyError, IndexError):
631 raise PresentationError(u"No output defined.")
633 logging.info(u"Done.")
635 def _parse_static(self):
636 """Parse specification of the static content in the specification YAML
640 logging.info(u"Parsing specification file: static content ...")
642 idx = self._get_type_index(u"static")
644 logging.warning(u"No static content specified.")
646 for key, value in self._cfg_yaml[idx].items():
647 if isinstance(value, str):
649 self._cfg_yaml[idx][key] = self._replace_tags(
650 value, self._specification[u"environment"][u"paths"])
654 self._specification[u"static"] = self._cfg_yaml[idx]
656 logging.info(u"Done.")
658 def _parse_elements_tables(self, table):
659 """Parse tables from the specification YAML file.
661 :param table: Table to be parsed from the specification file.
663 :raises PresentationError: If wrong data set is used.
667 table[u"template"] = self._replace_tags(
669 self._specification[u"environment"][u"paths"])
675 for item in (u"reference", u"compare"):
676 if table.get(item, None):
677 data_set = table[item].get(u"data", None)
678 if isinstance(data_set, str):
679 table[item][u"data"] = \
680 self.configuration[u"data-sets"][data_set]
681 data_set = table[item].get(u"data-replacement", None)
682 if isinstance(data_set, str):
683 table[item][u"data-replacement"] = \
684 self.configuration[u"data-sets"][data_set]
686 if table.get(u"history", None):
687 for i in range(len(table[u"history"])):
688 data_set = table[u"history"][i].get(u"data", None)
689 if isinstance(data_set, str):
690 table[u"history"][i][u"data"] = \
691 self.configuration[u"data-sets"][data_set]
692 data_set = table[u"history"][i].get(
693 u"data-replacement", None)
694 if isinstance(data_set, str):
695 table[u"history"][i][u"data-replacement"] = \
696 self.configuration[u"data-sets"][data_set]
698 if table.get(u"columns", None):
699 for i in range(len(table[u"columns"])):
700 data_set = table[u"columns"][i].get(u"data-set", None)
701 if isinstance(data_set, str):
702 table[u"columns"][i][u"data-set"] = \
703 self.configuration[u"data-sets"][data_set]
704 data_set = table[u"columns"][i].get(
705 u"data-replacement", None)
706 if isinstance(data_set, str):
707 table[u"columns"][i][u"data-replacement"] = \
708 self.configuration[u"data-sets"][data_set]
711 raise PresentationError(
712 f"Wrong data set used in {table.get(u'title', u'')}."
715 self._specification[u"tables"].append(table)
717 def _parse_elements_plots(self, plot):
718 """Parse plots from the specification YAML file.
720 :param plot: Plot to be parsed from the specification file.
722 :raises PresentationError: If plot layout is not defined.
725 # Add layout to the plots:
726 layout = plot[u"layout"].get(u"layout", None)
727 if layout is not None:
728 plot[u"layout"].pop(u"layout")
730 for key, val in (self.configuration[u"plot-layouts"]
732 plot[u"layout"][key] = val
734 raise PresentationError(
735 f"Layout {layout} is not defined in the "
736 f"configuration section."
738 self._specification[u"plots"].append(plot)
740 def _parse_elements_files(self, file):
741 """Parse files from the specification YAML file.
743 :param file: File to be parsed from the specification file.
748 file[u"dir-tables"] = self._replace_tags(
750 self._specification[u"environment"][u"paths"])
753 self._specification[u"files"].append(file)
755 def _parse_elements_cpta(self, cpta):
756 """Parse cpta from the specification YAML file.
758 :param cpta: cpta to be parsed from the specification file.
760 :raises PresentationError: If wrong data set is used or if plot layout
764 for plot in cpta[u"plots"]:
765 # Add layout to the plots:
766 layout = plot.get(u"layout", None)
767 if layout is not None:
770 self.configuration[u"plot-layouts"][layout]
772 raise PresentationError(
773 f"Layout {layout} is not defined in the "
774 f"configuration section."
777 if isinstance(plot.get(u"data", None), str):
778 data_set = plot[u"data"]
781 self.configuration[u"data-sets"][data_set]
783 raise PresentationError(
784 f"Data set {data_set} is not defined in "
785 f"the configuration section."
787 self._specification[u"cpta"] = cpta
789 def _parse_elements(self):
790 """Parse elements (tables, plots, ..) specification in the specification
794 logging.info(u"Parsing specification file: elements ...")
797 for element in self._cfg_yaml:
801 element[u"output-file"] = self._replace_tags(
802 element[u"output-file"],
803 self._specification[u"environment"][u"paths"])
808 element[u"input-file"] = self._replace_tags(
809 element[u"input-file"],
810 self._specification[u"environment"][u"paths"])
815 element[u"output-file-links"] = self._replace_tags(
816 element[u"output-file-links"],
817 self._specification[u"environment"][u"paths"])
821 # Add data sets to the elements:
822 if isinstance(element.get(u"data", None), str):
823 data_set = element[u"data"]
826 self.configuration[u"data-sets"][data_set]
828 raise PresentationError(
829 f"Data set {data_set} is not defined in the "
830 f"configuration section."
832 elif isinstance(element.get(u"data", None), list):
834 for item in element[u"data"]:
837 self.configuration[u"data-sets"][item]
840 raise PresentationError(
841 f"Data set {item} is not defined in the "
842 f"configuration section."
844 element[u"data"] = new_list
847 if element[u"type"] == u"table":
849 logging.info(f" {count:3d} Processing a table ...")
850 self._parse_elements_tables(element)
853 elif element[u"type"] == u"plot":
855 logging.info(f" {count:3d} Processing a plot ...")
856 self._parse_elements_plots(element)
859 elif element[u"type"] == u"file":
861 logging.info(f" {count:3d} Processing a file ...")
862 self._parse_elements_files(element)
865 elif element[u"type"] == u"cpta":
868 f" {count:3d} Processing Continuous Performance Trending "
871 self._parse_elements_cpta(element)
874 logging.info(u"Done.")
876 def read_specification(self):
877 """Parse specification in the specification YAML file.
879 :raises: PresentationError if an error occurred while parsing the
883 self._cfg_yaml = load(self._cfg_file, Loader=FullLoader)
884 except YAMLError as err:
885 raise PresentationError(msg=u"An error occurred while parsing the "
886 u"specification file.",
890 self._parse_configuration()
894 self._parse_elements()
896 logging.debug(f"Specification: \n{pformat(self._specification)}")