1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from yaml import load, YAMLError
22 from pprint import pformat
24 from errors import PresentationError
25 from utils import get_last_successful_build_number
26 from utils import get_last_completed_build_number
29 class Specification(object):
30 """Specification of Presentation and analytics layer.
32 - based on specification specified in the specification YAML file
33 - presentation and analytics layer is model driven
36 # Tags are used in specification YAML file and replaced while the file is
41 def __init__(self, cfg_file):
44 :param cfg_file: File handler for the specification YAML file.
45 :type cfg_file: BinaryIO
47 self._cfg_file = cfg_file
50 self._specification = {"environment": dict(),
51 "configuration": dict(),
61 def specification(self):
62 """Getter - specification.
64 :returns: Specification.
67 return self._specification
70 def environment(self):
71 """Getter - environment.
73 :returns: Environment specification.
76 return self._specification["environment"]
79 def configuration(self):
80 """Getter - configuration.
82 :returns: Configuration of PAL.
85 return self._specification["configuration"]
89 """Getter - static content.
91 :returns: Static content specification.
94 return self._specification["static"]
100 :returns: Mapping of the old names of test cases to the new (actual)
104 return self._specification["configuration"]["mapping"]
108 """Getter - Ignore list.
110 :returns: List of ignored test cases.
113 return self._specification["configuration"]["ignore"]
117 """Getter - specification - inputs.
123 return self._specification["input"]
127 """Getter - builds defined in specification.
129 :returns: Builds defined in the specification.
132 return self.input["builds"]
136 """Getter - specification - output formats and versions to be generated.
138 - versions: full, ...
140 :returns: Outputs to be generated.
143 return self._specification["output"]
147 """Getter - tables to be generated.
149 :returns: List of specifications of tables to be generated.
152 return self._specification["tables"]
156 """Getter - plots to be generated.
158 :returns: List of specifications of plots to be generated.
161 return self._specification["plots"]
165 """Getter - files to be generated.
167 :returns: List of specifications of files to be generated.
170 return self._specification["files"]
174 """Getter - Continuous Performance Trending and Analysis to be
177 :returns: List of specifications of Continuous Performance Trending and
178 Analysis to be generated.
181 return self._specification["cpta"]
183 def set_input_state(self, job, build_nr, state):
184 """Set the state of input
193 for build in self._specification["input"]["builds"][job]:
194 if build["build"] == build_nr:
195 build["status"] = state
198 raise PresentationError("Build '{}' is not defined for job '{}'"
199 " in specification file.".
200 format(build_nr, job))
202 raise PresentationError("Job '{}' and build '{}' is not defined in "
203 "specification file.".format(job, build_nr))
205 def set_input_file_name(self, job, build_nr, file_name):
206 """Set the state of input
215 for build in self._specification["input"]["builds"][job]:
216 if build["build"] == build_nr:
217 build["file-name"] = file_name
220 raise PresentationError("Build '{}' is not defined for job '{}'"
221 " in specification file.".
222 format(build_nr, job))
224 raise PresentationError("Job '{}' and build '{}' is not defined in "
225 "specification file.".format(job, build_nr))
227 def _get_build_number(self, job, build_type):
228 """Get the number of the job defined by its name:
229 - lastSuccessfulBuild
232 :param job: Job name.
233 :param build_type: Build type:
234 - lastSuccessfulBuild
237 :raises PresentationError: If it is not possible to get the build
239 :returns: The build number.
243 # defined as a range <start, end>
244 if build_type == "lastSuccessfulBuild":
245 # defined as a range <start, lastSuccessfulBuild>
246 ret_code, build_nr, _ = get_last_successful_build_number(
247 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
248 elif build_type == "lastCompletedBuild":
249 # defined as a range <start, lastCompletedBuild>
250 ret_code, build_nr, _ = get_last_completed_build_number(
251 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
253 raise PresentationError("Not supported build type: '{0}'".
256 raise PresentationError("Not possible to get the number of the "
259 build_nr = int(build_nr)
261 except ValueError as err:
262 raise PresentationError("Not possible to get the number of the "
263 "build number.\nReason: {0}".format(err))
265 def _get_type_index(self, item_type):
266 """Get index of item type (environment, input, output, ...) in
267 specification YAML file.
269 :param item_type: Item type: Top level items in specification YAML file,
270 e.g.: environment, input, output.
272 :returns: Index of the given item type.
277 for item in self._cfg_yaml:
278 if item["type"] == item_type:
283 def _find_tag(self, text):
284 """Find the first tag in the given text. The tag is enclosed by the
285 TAG_OPENER and TAG_CLOSER.
287 :param text: Text to be searched.
289 :returns: The tag, or None if not found.
293 start = text.index(self.TAG_OPENER)
294 end = text.index(self.TAG_CLOSER, start + 1) + 1
295 return text[start:end]
299 def _replace_tags(self, data, src_data=None):
300 """Replace tag(s) in the data by their values.
302 :param data: The data where the tags will be replaced by their values.
303 :param src_data: Data where the tags are defined. It is dictionary where
304 the key is the tag and the value is the tag value. If not given, 'data'
306 :type data: str or dict
308 :returns: Data with the tags replaced.
310 :raises: PresentationError if it is not possible to replace the tag or
311 the data is not the supported data type (str, dict).
317 if isinstance(data, str):
318 tag = self._find_tag(data)
320 data = data.replace(tag, src_data[tag[1:-1]])
322 elif isinstance(data, dict):
324 for key, value in data.items():
325 tag = self._find_tag(value)
328 data[key] = value.replace(tag, src_data[tag[1:-1]])
331 raise PresentationError("Not possible to replace the "
332 "tag '{}'".format(tag))
334 self._replace_tags(data, src_data)
336 raise PresentationError("Replace tags: Not supported data type.")
340 def _parse_env(self):
341 """Parse environment specification in the specification YAML file.
344 logging.info("Parsing specification file: environment ...")
346 idx = self._get_type_index("environment")
351 self._specification["environment"]["configuration"] = \
352 self._cfg_yaml[idx]["configuration"]
354 self._specification["environment"]["configuration"] = None
357 self._specification["environment"]["paths"] = \
358 self._replace_tags(self._cfg_yaml[idx]["paths"])
360 self._specification["environment"]["paths"] = None
363 self._specification["environment"]["urls"] = \
364 self._replace_tags(self._cfg_yaml[idx]["urls"])
366 self._specification["environment"]["urls"] = None
369 self._specification["environment"]["make-dirs"] = \
370 self._cfg_yaml[idx]["make-dirs"]
372 self._specification["environment"]["make-dirs"] = None
375 self._specification["environment"]["remove-dirs"] = \
376 self._cfg_yaml[idx]["remove-dirs"]
378 self._specification["environment"]["remove-dirs"] = None
381 self._specification["environment"]["build-dirs"] = \
382 self._cfg_yaml[idx]["build-dirs"]
384 self._specification["environment"]["build-dirs"] = None
386 logging.info("Done.")
388 def _parse_configuration(self):
389 """Parse configuration of PAL in the specification YAML file.
392 logging.info("Parsing specification file: configuration ...")
394 idx = self._get_type_index("configuration")
396 logging.warning("No configuration information in the specification "
401 self._specification["configuration"] = self._cfg_yaml[idx]
404 raise PresentationError("No configuration defined.")
406 # Data sets: Replace ranges by lists
407 for set_name, data_set in self.configuration["data-sets"].items():
408 for job, builds in data_set.items():
410 if isinstance(builds, dict):
411 build_nr = builds.get("end", None)
413 build_nr = int(build_nr)
415 # defined as a range <start, build_type>
416 build_nr = self._get_build_number(job, build_nr)
417 builds = [x for x in range(builds["start"], build_nr+1)]
418 self.configuration["data-sets"][set_name][job] = builds
422 mapping_file_name = self._specification["configuration"].\
423 get("mapping-file", None)
424 if mapping_file_name:
425 logging.debug("Mapping file: '{0}'".format(mapping_file_name))
427 with open(mapping_file_name, 'r') as mfile:
428 mapping = load(mfile)
429 logging.debug("Loaded mapping table:\n{0}".format(mapping))
430 except (YAMLError, IOError) as err:
431 raise PresentationError(
432 msg="An error occurred while parsing the mapping file "
433 "'{0}'.".format(mapping_file_name),
435 # Make sure everything is lowercase
437 self._specification["configuration"]["mapping"] = \
438 {key.lower(): val.lower() for key, val in mapping.iteritems()}
440 self._specification["configuration"]["mapping"] = dict()
444 ignore_list_name = self._specification["configuration"].\
445 get("ignore-list", None)
447 logging.debug("Ignore list file: '{0}'".format(ignore_list_name))
449 with open(ignore_list_name, 'r') as ifile:
451 logging.debug("Loaded ignore list:\n{0}".format(ignore))
452 except (YAMLError, IOError) as err:
453 raise PresentationError(
454 msg="An error occurred while parsing the ignore list file "
455 "'{0}'.".format(ignore_list_name),
457 # Make sure everything is lowercase
459 self._specification["configuration"]["ignore"] = \
460 [item.lower() for item in ignore]
462 self._specification["configuration"]["ignore"] = list()
464 logging.info("Done.")
466 def _parse_input(self):
467 """Parse input specification in the specification YAML file.
469 :raises: PresentationError if there are no data to process.
472 logging.info("Parsing specification file: input ...")
474 idx = self._get_type_index("input")
476 raise PresentationError("No data to process.")
479 for key, value in self._cfg_yaml[idx]["general"].items():
480 self._specification["input"][key] = value
481 self._specification["input"]["builds"] = dict()
483 for job, builds in self._cfg_yaml[idx]["builds"].items():
485 if isinstance(builds, dict):
486 build_nr = builds.get("end", None)
488 build_nr = int(build_nr)
490 # defined as a range <start, build_type>
491 build_nr = self._get_build_number(job, build_nr)
492 builds = [x for x in range(builds["start"], build_nr+1)]
493 self._specification["input"]["builds"][job] = list()
495 self._specification["input"]["builds"][job]. \
496 append({"build": build, "status": None})
499 logging.warning("No build is defined for the job '{}'. "
500 "Trying to continue without it.".
503 raise PresentationError("No data to process.")
505 logging.info("Done.")
507 def _parse_output(self):
508 """Parse output specification in the specification YAML file.
510 :raises: PresentationError if there is no output defined.
513 logging.info("Parsing specification file: output ...")
515 idx = self._get_type_index("output")
517 raise PresentationError("No output defined.")
520 self._specification["output"] = self._cfg_yaml[idx]
521 except (KeyError, IndexError):
522 raise PresentationError("No output defined.")
524 logging.info("Done.")
526 def _parse_static(self):
527 """Parse specification of the static content in the specification YAML
531 logging.info("Parsing specification file: static content ...")
533 idx = self._get_type_index("static")
535 logging.warning("No static content specified.")
537 for key, value in self._cfg_yaml[idx].items():
538 if isinstance(value, str):
540 self._cfg_yaml[idx][key] = self._replace_tags(
541 value, self._specification["environment"]["paths"])
545 self._specification["static"] = self._cfg_yaml[idx]
547 logging.info("Done.")
549 def _parse_elements(self):
550 """Parse elements (tables, plots) specification in the specification
554 logging.info("Parsing specification file: elements ...")
557 for element in self._cfg_yaml:
559 element["output-file"] = self._replace_tags(
560 element["output-file"],
561 self._specification["environment"]["paths"])
566 element["input-file"] = self._replace_tags(
567 element["input-file"],
568 self._specification["environment"]["paths"])
572 # add data sets to the elements:
573 if isinstance(element.get("data", None), str):
574 data_set = element["data"]
576 element["data"] = self.configuration["data-sets"][data_set]
578 raise PresentationError("Data set {0} is not defined in "
579 "the configuration section.".
582 if element["type"] == "table":
583 logging.info(" {:3d} Processing a table ...".format(count))
585 element["template"] = self._replace_tags(
587 self._specification["environment"]["paths"])
590 self._specification["tables"].append(element)
593 elif element["type"] == "plot":
594 logging.info(" {:3d} Processing a plot ...".format(count))
596 # Add layout to the plots:
597 layout = element["layout"].get("layout", None)
598 if layout is not None:
599 element["layout"].pop("layout")
601 for key, val in (self.configuration["plot-layouts"]
603 element["layout"][key] = val
605 raise PresentationError("Layout {0} is not defined in "
606 "the configuration section.".
608 self._specification["plots"].append(element)
611 elif element["type"] == "file":
612 logging.info(" {:3d} Processing a file ...".format(count))
614 element["dir-tables"] = self._replace_tags(
615 element["dir-tables"],
616 self._specification["environment"]["paths"])
619 self._specification["files"].append(element)
622 elif element["type"] == "cpta":
623 logging.info(" {:3d} Processing Continuous Performance "
624 "Trending and Analysis ...".format(count))
626 for plot in element["plots"]:
627 # Add layout to the plots:
628 layout = plot.get("layout", None)
629 if layout is not None:
632 self.configuration["plot-layouts"][layout]
634 raise PresentationError(
635 "Layout {0} is not defined in the "
636 "configuration section.".format(layout))
638 if isinstance(plot.get("data", None), str):
639 data_set = plot["data"]
642 self.configuration["data-sets"][data_set]
644 raise PresentationError(
645 "Data set {0} is not defined in "
646 "the configuration section.".
648 self._specification["cpta"] = element
651 logging.info("Done.")
653 def read_specification(self):
654 """Parse specification in the specification YAML file.
656 :raises: PresentationError if an error occurred while parsing the
660 self._cfg_yaml = load(self._cfg_file)
661 except YAMLError as err:
662 raise PresentationError(msg="An error occurred while parsing the "
663 "specification file.",
667 self._parse_configuration()
671 self._parse_elements()
673 logging.debug("Specification: \n{}".
674 format(pformat(self._specification)))