1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from pprint import pformat
23 from yaml import load, FullLoader, YAMLError
25 from pal_errors import PresentationError
26 from pal_utils import (
27 get_last_successful_build_nr, get_last_completed_build_number
32 """Specification of Presentation and analytics layer.
34 - based on specification specified in the specification YAML file
35 - presentation and analytics layer is model driven
38 # Tags are used in specification YAML file and replaced while the file is
43 def __init__(self, cfg_file):
46 :param cfg_file: File handler for the specification YAML file.
47 :type cfg_file: BinaryIO
49 self._cfg_file = cfg_file
52 self._specification = {
53 u"environment": dict(),
54 u"configuration": dict(),
65 def specification(self):
66 """Getter - specification.
68 :returns: Specification.
71 return self._specification
74 def environment(self):
75 """Getter - environment.
77 :returns: Environment specification.
80 return self._specification[u"environment"]
83 def configuration(self):
84 """Getter - configuration.
86 :returns: Configuration of PAL.
89 return self._specification[u"configuration"]
93 """Getter - static content.
95 :returns: Static content specification.
98 return self._specification[u"static"]
104 :returns: Mapping of the old names of test cases to the new (actual)
108 return self._specification[u"configuration"][u"mapping"]
112 """Getter - Ignore list.
114 :returns: List of ignored test cases.
117 return self._specification[u"configuration"][u"ignore"]
121 """Getter - Alerting.
123 :returns: Specification of alerts.
126 return self._specification[u"configuration"][u"alerting"]
130 """Getter - specification - inputs.
136 return self._specification[u"input"]
140 """Getter - builds defined in specification.
142 :returns: Builds defined in the specification.
145 return self.input[u"builds"]
149 """Getter - specification - output formats and versions to be generated.
151 - versions: full, ...
153 :returns: Outputs to be generated.
156 return self._specification[u"output"]
160 """Getter - tables to be generated.
162 :returns: List of specifications of tables to be generated.
165 return self._specification[u"tables"]
169 """Getter - plots to be generated.
171 :returns: List of specifications of plots to be generated.
174 return self._specification[u"plots"]
178 """Getter - files to be generated.
180 :returns: List of specifications of files to be generated.
183 return self._specification[u"files"]
187 """Getter - Continuous Performance Trending and Analysis to be
190 :returns: List of specifications of Continuous Performance Trending and
191 Analysis to be generated.
194 return self._specification[u"cpta"]
196 def set_input_state(self, job, build_nr, state):
197 """Set the state of input
206 for build in self._specification[u"input"][u"builds"][job]:
207 if build[u"build"] == build_nr:
208 build[u"status"] = state
211 raise PresentationError(
212 f"Build {build_nr} is not defined for job {job} in "
213 f"specification file."
216 raise PresentationError(
217 f"Job {job} and build {build_nr} is not defined in "
218 f"specification file."
221 def set_input_file_name(self, job, build_nr, file_name):
222 """Set the state of input
231 for build in self._specification[u"input"][u"builds"][job]:
232 if build[u"build"] == build_nr:
233 build[u"file-name"] = file_name
236 raise PresentationError(
237 f"Build {build_nr} is not defined for job {job} in "
238 f"specification file."
241 raise PresentationError(
242 f"Job {job} and build {build_nr} is not defined in "
243 f"specification file."
246 def _get_build_number(self, job, build_type):
247 """Get the number of the job defined by its name:
248 - lastSuccessfulBuild
251 :param job: Job name.
252 :param build_type: Build type:
253 - lastSuccessfulBuild
256 :raises PresentationError: If it is not possible to get the build
258 :returns: The build number.
262 # defined as a range <start, end>
263 if build_type == u"lastSuccessfulBuild":
264 # defined as a range <start, lastSuccessfulBuild>
265 ret_code, build_nr, _ = get_last_successful_build_nr(
266 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
267 elif build_type == u"lastCompletedBuild":
268 # defined as a range <start, lastCompletedBuild>
269 ret_code, build_nr, _ = get_last_completed_build_number(
270 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
272 raise PresentationError(f"Not supported build type: {build_type}")
274 raise PresentationError(u"Not possible to get the number of the "
277 build_nr = int(build_nr)
279 except ValueError as err:
280 raise PresentationError(
281 f"Not possible to get the number of the build number. Reason:\n"
285 def _get_type_index(self, item_type):
286 """Get index of item type (environment, input, output, ...) in
287 specification YAML file.
289 :param item_type: Item type: Top level items in specification YAML file,
290 e.g.: environment, input, output.
292 :returns: Index of the given item type.
297 for item in self._cfg_yaml:
298 if item[u"type"] == item_type:
303 def _find_tag(self, text):
304 """Find the first tag in the given text. The tag is enclosed by the
305 TAG_OPENER and TAG_CLOSER.
307 :param text: Text to be searched.
309 :returns: The tag, or None if not found.
313 start = text.index(self.TAG_OPENER)
314 end = text.index(self.TAG_CLOSER, start + 1) + 1
315 return text[start:end]
319 def _replace_tags(self, data, src_data=None):
320 """Replace tag(s) in the data by their values.
322 :param data: The data where the tags will be replaced by their values.
323 :param src_data: Data where the tags are defined. It is dictionary where
324 the key is the tag and the value is the tag value. If not given, 'data'
326 :type data: str or dict
328 :returns: Data with the tags replaced.
330 :raises: PresentationError if it is not possible to replace the tag or
331 the data is not the supported data type (str, dict).
337 if isinstance(data, str):
338 tag = self._find_tag(data)
340 data = data.replace(tag, src_data[tag[1:-1]])
342 elif isinstance(data, dict):
344 for key, value in data.items():
345 tag = self._find_tag(value)
348 data[key] = value.replace(tag, src_data[tag[1:-1]])
351 raise PresentationError(
352 f"Not possible to replace the tag {tag}"
355 self._replace_tags(data, src_data)
357 raise PresentationError(u"Replace tags: Not supported data type.")
361 def _parse_env(self):
362 """Parse environment specification in the specification YAML file.
365 logging.info(u"Parsing specification file: environment ...")
367 idx = self._get_type_index(u"environment")
372 self._specification[u"environment"][u"configuration"] = \
373 self._cfg_yaml[idx][u"configuration"]
375 self._specification[u"environment"][u"configuration"] = None
378 self._specification[u"environment"][u"paths"] = \
379 self._replace_tags(self._cfg_yaml[idx][u"paths"])
381 self._specification[u"environment"][u"paths"] = None
384 self._specification[u"environment"][u"urls"] = \
385 self._cfg_yaml[idx][u"urls"]
387 self._specification[u"environment"][u"urls"] = None
390 self._specification[u"environment"][u"make-dirs"] = \
391 self._cfg_yaml[idx][u"make-dirs"]
393 self._specification[u"environment"][u"make-dirs"] = None
396 self._specification[u"environment"][u"remove-dirs"] = \
397 self._cfg_yaml[idx][u"remove-dirs"]
399 self._specification[u"environment"][u"remove-dirs"] = None
402 self._specification[u"environment"][u"build-dirs"] = \
403 self._cfg_yaml[idx][u"build-dirs"]
405 self._specification[u"environment"][u"build-dirs"] = None
408 self._specification[u"environment"][u"testbeds"] = \
409 self._cfg_yaml[idx][u"testbeds"]
411 self._specification[u"environment"][u"testbeds"] = None
413 logging.info(u"Done.")
415 def _load_mapping_table(self):
416 """Load a mapping table if it is specified. If not, use empty list.
419 mapping_file_name = self._specification[u"configuration"].\
420 get(u"mapping-file", None)
421 if mapping_file_name:
423 with open(mapping_file_name, u'r') as mfile:
424 mapping = load(mfile, Loader=FullLoader)
425 # Make sure everything is lowercase
426 self._specification[u"configuration"][u"mapping"] = \
427 {key.lower(): val.lower() for key, val in
429 logging.debug(f"Loaded mapping table:\n{mapping}")
430 except (YAMLError, IOError) as err:
431 raise PresentationError(
432 msg=f"An error occurred while parsing the mapping file "
433 f"{mapping_file_name}",
437 self._specification[u"configuration"][u"mapping"] = dict()
439 def _load_ignore_list(self):
440 """Load an ignore list if it is specified. If not, use empty list.
443 ignore_list_name = self._specification[u"configuration"].\
444 get(u"ignore-list", None)
447 with open(ignore_list_name, u'r') as ifile:
448 ignore = load(ifile, Loader=FullLoader)
449 # Make sure everything is lowercase
450 self._specification[u"configuration"][u"ignore"] = \
451 [item.lower() for item in ignore]
452 logging.debug(f"Loaded ignore list:\n{ignore}")
453 except (YAMLError, IOError) as err:
454 raise PresentationError(
455 msg=f"An error occurred while parsing the ignore list file "
456 f"{ignore_list_name}.",
460 self._specification[u"configuration"][u"ignore"] = list()
462 def _parse_configuration(self):
463 """Parse configuration of PAL in the specification YAML file.
466 logging.info(u"Parsing specification file: configuration ...")
468 idx = self._get_type_index("configuration")
471 u"No configuration information in the specification file."
476 self._specification[u"configuration"] = self._cfg_yaml[idx]
478 raise PresentationError(u"No configuration defined.")
480 # Data sets: Replace ranges by lists
481 for set_name, data_set in self.configuration[u"data-sets"].items():
482 if not isinstance(data_set, dict):
484 for job, builds in data_set.items():
487 if isinstance(builds, dict):
488 build_end = builds.get(u"end", None)
490 build_end = int(build_end)
492 # defined as a range <start, build_type>
493 build_end = self._get_build_number(job, build_end)
494 builds = [x for x in range(builds[u"start"],
496 if x not in builds.get(u"skip", list())]
497 self.configuration[u"data-sets"][set_name][job] = builds
498 elif isinstance(builds, list):
499 for idx, item in enumerate(builds):
501 builds[idx] = int(item)
503 # defined as a range <build_type>
504 builds[idx] = self._get_build_number(job, item)
506 # Data sets: add sub-sets to sets (only one level):
507 for set_name, data_set in self.configuration[u"data-sets"].items():
508 if isinstance(data_set, list):
510 for item in data_set:
512 for key, val in self.configuration[u"data-sets"][item].\
516 raise PresentationError(
517 f"Data set {item} is not defined in "
518 f"the configuration section."
520 self.configuration[u"data-sets"][set_name] = new_set
523 self._load_mapping_table()
526 self._load_ignore_list()
528 logging.info(u"Done.")
530 def _parse_input(self):
531 """Parse input specification in the specification YAML file.
533 :raises: PresentationError if there are no data to process.
536 logging.info(u"Parsing specification file: input ...")
538 idx = self._get_type_index(u"input")
540 raise PresentationError(u"No data to process.")
543 for key, value in self._cfg_yaml[idx][u"general"].items():
544 self._specification[u"input"][key] = value
545 self._specification[u"input"][u"builds"] = dict()
547 for job, builds in self._cfg_yaml[idx][u"builds"].items():
549 if isinstance(builds, dict):
550 build_end = builds.get(u"end", None)
552 build_end = int(build_end)
554 # defined as a range <start, build_type>
555 build_end = self._get_build_number(job, build_end)
556 builds = [x for x in range(builds[u"start"],
558 if x not in builds.get(u"skip", list())]
559 self._specification[u"input"][u"builds"][job] = list()
561 self._specification[u"input"][u"builds"][job]. \
562 append({u"build": build, u"status": None})
566 f"No build is defined for the job {job}. Trying to "
567 f"continue without it."
570 raise PresentationError(u"No data to process.")
572 logging.info(u"Done.")
574 def _parse_output(self):
575 """Parse output specification in the specification YAML file.
577 :raises: PresentationError if there is no output defined.
580 logging.info(u"Parsing specification file: output ...")
582 idx = self._get_type_index(u"output")
584 raise PresentationError(u"No output defined.")
587 self._specification[u"output"] = self._cfg_yaml[idx]
588 except (KeyError, IndexError):
589 raise PresentationError(u"No output defined.")
591 logging.info(u"Done.")
593 def _parse_static(self):
594 """Parse specification of the static content in the specification YAML
598 logging.info(u"Parsing specification file: static content ...")
600 idx = self._get_type_index(u"static")
602 logging.warning(u"No static content specified.")
604 for key, value in self._cfg_yaml[idx].items():
605 if isinstance(value, str):
607 self._cfg_yaml[idx][key] = self._replace_tags(
608 value, self._specification[u"environment"][u"paths"])
612 self._specification[u"static"] = self._cfg_yaml[idx]
614 logging.info(u"Done.")
616 def _parse_elements_tables(self, table):
617 """Parse tables from the specification YAML file.
619 :param table: Table to be parsed from the specification file.
621 :raises PresentationError: If wrong data set is used.
625 table[u"template"] = self._replace_tags(
627 self._specification[u"environment"][u"paths"])
633 for item in (u"reference", u"compare"):
634 if table.get(item, None):
635 data_set = table[item].get(u"data", None)
636 if isinstance(data_set, str):
637 table[item][u"data"] = \
638 self.configuration[u"data-sets"][data_set]
639 data_set = table[item].get(u"data-replacement", None)
640 if isinstance(data_set, str):
641 table[item][u"data-replacement"] = \
642 self.configuration[u"data-sets"][data_set]
644 if table.get(u"history", None):
645 for i in range(len(table[u"history"])):
646 data_set = table[u"history"][i].get(u"data", None)
647 if isinstance(data_set, str):
648 table[u"history"][i][u"data"] = \
649 self.configuration[u"data-sets"][data_set]
650 data_set = table[u"history"][i].get(
651 u"data-replacement", None)
652 if isinstance(data_set, str):
653 table[u"history"][i][u"data-replacement"] = \
654 self.configuration[u"data-sets"][data_set]
656 raise PresentationError(
657 f"Wrong data set used in {table.get(u'title', u'')}."
660 self._specification[u"tables"].append(table)
662 def _parse_elements_plots(self, plot):
663 """Parse plots from the specification YAML file.
665 :param plot: Plot to be parsed from the specification file.
667 :raises PresentationError: If plot layout is not defined.
670 # Add layout to the plots:
671 layout = plot[u"layout"].get(u"layout", None)
672 if layout is not None:
673 plot[u"layout"].pop(u"layout")
675 for key, val in (self.configuration[u"plot-layouts"]
677 plot[u"layout"][key] = val
679 raise PresentationError(
680 f"Layout {layout} is not defined in the "
681 f"configuration section."
683 self._specification[u"plots"].append(plot)
685 def _parse_elements_files(self, file):
686 """Parse files from the specification YAML file.
688 :param file: File to be parsed from the specification file.
693 file[u"dir-tables"] = self._replace_tags(
695 self._specification[u"environment"][u"paths"])
698 self._specification[u"files"].append(file)
700 def _parse_elements_cpta(self, cpta):
701 """Parse cpta from the specification YAML file.
703 :param cpta: cpta to be parsed from the specification file.
705 :raises PresentationError: If wrong data set is used or if plot layout
709 for plot in cpta[u"plots"]:
710 # Add layout to the plots:
711 layout = plot.get(u"layout", None)
712 if layout is not None:
715 self.configuration[u"plot-layouts"][layout]
717 raise PresentationError(
718 f"Layout {layout} is not defined in the "
719 f"configuration section."
722 if isinstance(plot.get(u"data", None), str):
723 data_set = plot[u"data"]
726 self.configuration[u"data-sets"][data_set]
728 raise PresentationError(
729 f"Data set {data_set} is not defined in "
730 f"the configuration section."
732 self._specification[u"cpta"] = cpta
734 def _parse_elements(self):
735 """Parse elements (tables, plots, ..) specification in the specification
739 logging.info(u"Parsing specification file: elements ...")
742 for element in self._cfg_yaml:
746 element[u"output-file"] = self._replace_tags(
747 element[u"output-file"],
748 self._specification[u"environment"][u"paths"])
753 element[u"input-file"] = self._replace_tags(
754 element[u"input-file"],
755 self._specification[u"environment"][u"paths"])
759 # Add data sets to the elements:
760 if isinstance(element.get(u"data", None), str):
761 data_set = element[u"data"]
764 self.configuration[u"data-sets"][data_set]
766 raise PresentationError(
767 f"Data set {data_set} is not defined in the "
768 f"configuration section."
772 if element[u"type"] == u"table":
774 logging.info(f" {count:3d} Processing a table ...")
775 self._parse_elements_tables(element)
778 elif element[u"type"] == u"plot":
780 logging.info(f" {count:3d} Processing a plot ...")
781 self._parse_elements_plots(element)
784 elif element[u"type"] == u"file":
786 logging.info(f" {count:3d} Processing a file ...")
787 self._parse_elements_files(element)
790 elif element[u"type"] == u"cpta":
793 f" {count:3d} Processing Continuous Performance Trending "
796 self._parse_elements_cpta(element)
799 logging.info(u"Done.")
801 def read_specification(self):
802 """Parse specification in the specification YAML file.
804 :raises: PresentationError if an error occurred while parsing the
808 self._cfg_yaml = load(self._cfg_file, Loader=FullLoader)
809 except YAMLError as err:
810 raise PresentationError(msg=u"An error occurred while parsing the "
811 u"specification file.",
815 self._parse_configuration()
819 self._parse_elements()
821 logging.debug(f"Specification: \n{pformat(self._specification)}")