1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from yaml import load, YAMLError
22 from pprint import pformat
24 from errors import PresentationError
25 from utils import get_last_successful_build_number
26 from utils import get_last_completed_build_number
29 class Specification(object):
30 """Specification of Presentation and analytics layer.
32 - based on specification specified in the specification YAML file
33 - presentation and analytics layer is model driven
36 # Tags are used in specification YAML file and replaced while the file is
41 def __init__(self, cfg_file):
44 :param cfg_file: File handler for the specification YAML file.
45 :type cfg_file: BinaryIO
47 self._cfg_file = cfg_file
50 self._specification = {"environment": dict(),
51 "configuration": dict(),
62 def specification(self):
63 """Getter - specification.
65 :returns: Specification.
68 return self._specification
71 def environment(self):
72 """Getter - environment.
74 :returns: Environment specification.
77 return self._specification["environment"]
80 def configuration(self):
81 """Getter - configuration.
83 :returns: Configuration of PAL.
86 return self._specification["configuration"]
90 """Getter - static content.
92 :returns: Static content specification.
95 return self._specification["static"]
101 :returns: Debug specification
104 return self._specification["debug"]
108 """Getter - debug mode
110 :returns: True if debug mode is on, otherwise False.
115 if self.environment["configuration"]["CFG[DEBUG]"] == 1:
124 """Getter - specification - inputs.
130 return self._specification["input"]
134 """Getter - builds defined in specification.
136 :returns: Builds defined in the specification.
139 return self.input["builds"]
143 """Getter - specification - output formats and versions to be generated.
145 - versions: full, ...
147 :returns: Outputs to be generated.
150 return self._specification["output"]
154 """Getter - tables to be generated.
156 :returns: List of specifications of tables to be generated.
159 return self._specification["tables"]
163 """Getter - plots to be generated.
165 :returns: List of specifications of plots to be generated.
168 return self._specification["plots"]
172 """Getter - files to be generated.
174 :returns: List of specifications of files to be generated.
177 return self._specification["files"]
181 """Getter - Continuous Performance Trending and Analysis to be
184 :returns: List of specifications of Continuous Performance Trending and
185 Analysis to be generated.
188 return self._specification["cpta"]
190 def set_input_state(self, job, build_nr, state):
191 """Set the state of input
200 for build in self._specification["input"]["builds"][job]:
201 if build["build"] == build_nr:
202 build["status"] = state
205 raise PresentationError("Build '{}' is not defined for job '{}'"
206 " in specification file.".
207 format(build_nr, job))
209 raise PresentationError("Job '{}' and build '{}' is not defined in "
210 "specification file.".format(job, build_nr))
212 def set_input_file_name(self, job, build_nr, file_name):
213 """Set the state of input
222 for build in self._specification["input"]["builds"][job]:
223 if build["build"] == build_nr:
224 build["file-name"] = file_name
227 raise PresentationError("Build '{}' is not defined for job '{}'"
228 " in specification file.".
229 format(build_nr, job))
231 raise PresentationError("Job '{}' and build '{}' is not defined in "
232 "specification file.".format(job, build_nr))
234 def _get_build_number(self, job, build_type):
235 """Get the number of the job defined by its name:
236 - lastSuccessfulBuild
239 :param job: Job name.
240 :param build_type: Build type:
241 - lastSuccessfulBuild
244 :raises PresentationError: If it is not possible to get the build
246 :returns: The build number.
250 # defined as a range <start, end>
251 if build_type == "lastSuccessfulBuild":
252 # defined as a range <start, lastSuccessfulBuild>
253 ret_code, build_nr, _ = get_last_successful_build_number(
254 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
255 elif build_type == "lastCompletedBuild":
256 # defined as a range <start, lastCompletedBuild>
257 ret_code, build_nr, _ = get_last_completed_build_number(
258 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
260 raise PresentationError("Not supported build type: '{0}'".
263 raise PresentationError("Not possible to get the number of the "
266 build_nr = int(build_nr)
268 except ValueError as err:
269 raise PresentationError("Not possible to get the number of the "
270 "build number.\nReason: {0}".format(err))
272 def _get_type_index(self, item_type):
273 """Get index of item type (environment, input, output, ...) in
274 specification YAML file.
276 :param item_type: Item type: Top level items in specification YAML file,
277 e.g.: environment, input, output.
279 :returns: Index of the given item type.
284 for item in self._cfg_yaml:
285 if item["type"] == item_type:
290 def _find_tag(self, text):
291 """Find the first tag in the given text. The tag is enclosed by the
292 TAG_OPENER and TAG_CLOSER.
294 :param text: Text to be searched.
296 :returns: The tag, or None if not found.
300 start = text.index(self.TAG_OPENER)
301 end = text.index(self.TAG_CLOSER, start + 1) + 1
302 return text[start:end]
306 def _replace_tags(self, data, src_data=None):
307 """Replace tag(s) in the data by their values.
309 :param data: The data where the tags will be replaced by their values.
310 :param src_data: Data where the tags are defined. It is dictionary where
311 the key is the tag and the value is the tag value. If not given, 'data'
313 :type data: str or dict
315 :returns: Data with the tags replaced.
317 :raises: PresentationError if it is not possible to replace the tag or
318 the data is not the supported data type (str, dict).
324 if isinstance(data, str):
325 tag = self._find_tag(data)
327 data = data.replace(tag, src_data[tag[1:-1]])
329 elif isinstance(data, dict):
331 for key, value in data.items():
332 tag = self._find_tag(value)
335 data[key] = value.replace(tag, src_data[tag[1:-1]])
338 raise PresentationError("Not possible to replace the "
339 "tag '{}'".format(tag))
341 self._replace_tags(data, src_data)
343 raise PresentationError("Replace tags: Not supported data type.")
347 def _parse_env(self):
348 """Parse environment specification in the specification YAML file.
351 logging.info("Parsing specification file: environment ...")
353 idx = self._get_type_index("environment")
358 self._specification["environment"]["configuration"] = \
359 self._cfg_yaml[idx]["configuration"]
361 self._specification["environment"]["configuration"] = None
364 self._specification["environment"]["paths"] = \
365 self._replace_tags(self._cfg_yaml[idx]["paths"])
367 self._specification["environment"]["paths"] = None
370 self._specification["environment"]["urls"] = \
371 self._replace_tags(self._cfg_yaml[idx]["urls"])
373 self._specification["environment"]["urls"] = None
376 self._specification["environment"]["make-dirs"] = \
377 self._cfg_yaml[idx]["make-dirs"]
379 self._specification["environment"]["make-dirs"] = None
382 self._specification["environment"]["remove-dirs"] = \
383 self._cfg_yaml[idx]["remove-dirs"]
385 self._specification["environment"]["remove-dirs"] = None
388 self._specification["environment"]["build-dirs"] = \
389 self._cfg_yaml[idx]["build-dirs"]
391 self._specification["environment"]["build-dirs"] = None
393 logging.info("Done.")
395 def _parse_configuration(self):
396 """Parse configuration of PAL in the specification YAML file.
399 logging.info("Parsing specification file: configuration ...")
401 idx = self._get_type_index("configuration")
403 logging.warning("No configuration information in the specification "
408 self._specification["configuration"] = self._cfg_yaml[idx]
411 raise PresentationError("No configuration defined.")
413 # Data sets: Replace ranges by lists
414 for set_name, data_set in self.configuration["data-sets"].items():
415 for job, builds in data_set.items():
417 if isinstance(builds, dict):
418 build_nr = builds.get("end", None)
420 build_nr = int(build_nr)
422 # defined as a range <start, build_type>
423 build_nr = self._get_build_number(job, build_nr)
424 builds = [x for x in range(builds["start"], build_nr+1)]
425 self.configuration["data-sets"][set_name][job] = builds
426 logging.info("Done.")
428 def _parse_debug(self):
429 """Parse debug specification in the specification YAML file.
432 if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
435 logging.info("Parsing specification file: debug ...")
437 idx = self._get_type_index("debug")
439 self.environment["configuration"]["CFG[DEBUG]"] = 0
443 for key, value in self._cfg_yaml[idx]["general"].items():
444 self._specification["debug"][key] = value
446 self._specification["input"]["builds"] = dict()
447 for job, builds in self._cfg_yaml[idx]["builds"].items():
449 self._specification["input"]["builds"][job] = list()
451 self._specification["input"]["builds"][job].\
452 append({"build": build["build"],
453 "status": "downloaded",
454 "file-name": self._replace_tags(
456 self.environment["paths"])})
458 logging.warning("No build is defined for the job '{}'. "
459 "Trying to continue without it.".
463 raise PresentationError("No data to process.")
465 def _parse_input(self):
466 """Parse input specification in the specification YAML file.
468 :raises: PresentationError if there are no data to process.
471 logging.info("Parsing specification file: input ...")
473 idx = self._get_type_index("input")
475 raise PresentationError("No data to process.")
478 for key, value in self._cfg_yaml[idx]["general"].items():
479 self._specification["input"][key] = value
480 self._specification["input"]["builds"] = dict()
482 for job, builds in self._cfg_yaml[idx]["builds"].items():
484 if isinstance(builds, dict):
485 build_nr = builds.get("end", None)
487 build_nr = int(build_nr)
489 # defined as a range <start, build_type>
490 build_nr = self._get_build_number(job, build_nr)
491 builds = [x for x in range(builds["start"], build_nr+1)]
492 self._specification["input"]["builds"][job] = list()
494 self._specification["input"]["builds"][job]. \
495 append({"build": build, "status": None})
498 logging.warning("No build is defined for the job '{}'. "
499 "Trying to continue without it.".
502 raise PresentationError("No data to process.")
504 logging.info("Done.")
506 def _parse_output(self):
507 """Parse output specification in the specification YAML file.
509 :raises: PresentationError if there is no output defined.
512 logging.info("Parsing specification file: output ...")
514 idx = self._get_type_index("output")
516 raise PresentationError("No output defined.")
519 self._specification["output"] = self._cfg_yaml[idx]
520 except (KeyError, IndexError):
521 raise PresentationError("No output defined.")
523 logging.info("Done.")
525 def _parse_static(self):
526 """Parse specification of the static content in the specification YAML
530 logging.info("Parsing specification file: static content ...")
532 idx = self._get_type_index("static")
534 logging.warning("No static content specified.")
536 for key, value in self._cfg_yaml[idx].items():
537 if isinstance(value, str):
539 self._cfg_yaml[idx][key] = self._replace_tags(
540 value, self._specification["environment"]["paths"])
544 self._specification["static"] = self._cfg_yaml[idx]
546 logging.info("Done.")
548 def _parse_elements(self):
549 """Parse elements (tables, plots) specification in the specification
553 logging.info("Parsing specification file: elements ...")
556 for element in self._cfg_yaml:
558 element["output-file"] = self._replace_tags(
559 element["output-file"],
560 self._specification["environment"]["paths"])
565 element["input-file"] = self._replace_tags(
566 element["input-file"],
567 self._specification["environment"]["paths"])
571 # add data sets to the elements:
572 if isinstance(element.get("data", None), str):
573 data_set = element["data"]
575 element["data"] = self.configuration["data-sets"][data_set]
577 raise PresentationError("Data set {0} is not defined in "
578 "the configuration section.".
581 if element["type"] == "table":
582 logging.info(" {:3d} Processing a table ...".format(count))
584 element["template"] = self._replace_tags(
586 self._specification["environment"]["paths"])
589 self._specification["tables"].append(element)
592 elif element["type"] == "plot":
593 logging.info(" {:3d} Processing a plot ...".format(count))
595 # Add layout to the plots:
596 layout = element["layout"].get("layout", None)
597 if layout is not None:
598 element["layout"].pop("layout")
600 for key, val in (self.configuration["plot-layouts"]
602 element["layout"][key] = val
604 raise PresentationError("Layout {0} is not defined in "
605 "the configuration section.".
607 self._specification["plots"].append(element)
610 elif element["type"] == "file":
611 logging.info(" {:3d} Processing a file ...".format(count))
613 element["dir-tables"] = self._replace_tags(
614 element["dir-tables"],
615 self._specification["environment"]["paths"])
618 self._specification["files"].append(element)
621 elif element["type"] == "cpta":
622 logging.info(" {:3d} Processing Continuous Performance "
623 "Trending and Analysis ...".format(count))
625 for plot in element["plots"]:
626 # Add layout to the plots:
627 layout = plot.get("layout", None)
628 if layout is not None:
631 self.configuration["plot-layouts"][layout]
633 raise PresentationError(
634 "Layout {0} is not defined in the "
635 "configuration section.".format(layout))
637 if isinstance(plot.get("data", None), str):
638 data_set = plot["data"]
641 self.configuration["data-sets"][data_set]
643 raise PresentationError(
644 "Data set {0} is not defined in "
645 "the configuration section.".
647 self._specification["cpta"] = element
650 logging.info("Done.")
652 def read_specification(self):
653 """Parse specification in the specification YAML file.
655 :raises: PresentationError if an error occurred while parsing the
659 self._cfg_yaml = load(self._cfg_file)
660 except YAMLError as err:
661 raise PresentationError(msg="An error occurred while parsing the "
662 "specification file.",
666 self._parse_configuration()
672 self._parse_elements()
674 logging.debug("Specification: \n{}".
675 format(pformat(self._specification)))