1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from yaml import load, YAMLError
22 from pprint import pformat
24 from errors import PresentationError
25 from utils import get_last_successful_build_number
26 from utils import get_last_completed_build_number
29 class Specification(object):
30 """Specification of Presentation and analytics layer.
32 - based on specification specified in the specification YAML file
33 - presentation and analytics layer is model driven
36 # Tags are used in specification YAML file and replaced while the file is
41 def __init__(self, cfg_file):
44 :param cfg_file: File handler for the specification YAML file.
45 :type cfg_file: BinaryIO
47 self._cfg_file = cfg_file
50 self._specification = {"environment": dict(),
51 "configuration": dict(),
62 def specification(self):
63 """Getter - specification.
65 :returns: Specification.
68 return self._specification
71 def environment(self):
72 """Getter - environment.
74 :returns: Environment specification.
77 return self._specification["environment"]
80 def configuration(self):
81 """Getter - configuration.
83 :returns: Configuration of PAL.
86 return self._specification["configuration"]
90 """Getter - static content.
92 :returns: Static content specification.
95 return self._specification["static"]
101 :returns: Debug specification
104 return self._specification["debug"]
108 """Getter - debug mode
110 :returns: True if debug mode is on, otherwise False.
115 if self.environment["configuration"]["CFG[DEBUG]"] == 1:
124 """Getter - specification - inputs.
130 return self._specification["input"]
134 """Getter - builds defined in specification.
136 :returns: Builds defined in the specification.
139 return self.input["builds"]
143 """Getter - specification - output formats and versions to be generated.
145 - versions: full, ...
147 :returns: Outputs to be generated.
150 return self._specification["output"]
154 """Getter - tables to be generated.
156 :returns: List of specifications of tables to be generated.
159 return self._specification["tables"]
163 """Getter - plots to be generated.
165 :returns: List of specifications of plots to be generated.
168 return self._specification["plots"]
172 """Getter - files to be generated.
174 :returns: List of specifications of files to be generated.
177 return self._specification["files"]
181 """Getter - Continuous Performance Trending and Analysis to be
184 :returns: List of specifications of Continuous Performance Trending and
185 Analysis to be generated.
188 return self._specification["cpta"]
190 def set_input_state(self, job, build_nr, state):
191 """Set the state of input
200 for build in self._specification["input"]["builds"][job]:
201 if build["build"] == build_nr:
202 build["status"] = state
205 raise PresentationError("Build '{}' is not defined for job '{}'"
206 " in specification file.".
207 format(build_nr, job))
209 raise PresentationError("Job '{}' and build '{}' is not defined in "
210 "specification file.".format(job, build_nr))
212 def set_input_file_name(self, job, build_nr, file_name):
213 """Set the state of input
222 for build in self._specification["input"]["builds"][job]:
223 if build["build"] == build_nr:
224 build["file-name"] = file_name
227 raise PresentationError("Build '{}' is not defined for job '{}'"
228 " in specification file.".
229 format(build_nr, job))
231 raise PresentationError("Job '{}' and build '{}' is not defined in "
232 "specification file.".format(job, build_nr))
234 def _get_build_number(self, job, build_type):
235 """Get the number of the job defined by its name:
236 - lastSuccessfulBuild
239 :param job: Job name.
240 :param build_type: Build type:
241 - lastSuccessfulBuild
244 :raises PresentationError: If it is not possible to get the build
246 :returns: The build number.
250 # defined as a range <start, end>
251 if build_type == "lastSuccessfulBuild":
252 # defined as a range <start, lastSuccessfulBuild>
253 ret_code, build_nr, _ = get_last_successful_build_number(
254 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
255 elif build_type == "lastCompletedBuild":
256 # defined as a range <start, lastCompletedBuild>
257 ret_code, build_nr, _ = get_last_completed_build_number(
258 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
260 raise PresentationError("Not supported build type: '{0}'".
263 raise PresentationError("Not possible to get the number of the "
266 build_nr = int(build_nr)
268 except ValueError as err:
269 raise PresentationError("Not possible to get the number of the "
270 "build number.\nReason: {0}".format(err))
272 def _get_type_index(self, item_type):
273 """Get index of item type (environment, input, output, ...) in
274 specification YAML file.
276 :param item_type: Item type: Top level items in specification YAML file,
277 e.g.: environment, input, output.
279 :returns: Index of the given item type.
284 for item in self._cfg_yaml:
285 if item["type"] == item_type:
290 def _find_tag(self, text):
291 """Find the first tag in the given text. The tag is enclosed by the
292 TAG_OPENER and TAG_CLOSER.
294 :param text: Text to be searched.
296 :returns: The tag, or None if not found.
300 start = text.index(self.TAG_OPENER)
301 end = text.index(self.TAG_CLOSER, start + 1) + 1
302 return text[start:end]
306 def _replace_tags(self, data, src_data=None):
307 """Replace tag(s) in the data by their values.
309 :param data: The data where the tags will be replaced by their values.
310 :param src_data: Data where the tags are defined. It is dictionary where
311 the key is the tag and the value is the tag value. If not given, 'data'
313 :type data: str or dict
315 :returns: Data with the tags replaced.
317 :raises: PresentationError if it is not possible to replace the tag or
318 the data is not the supported data type (str, dict).
324 if isinstance(data, str):
325 tag = self._find_tag(data)
327 data = data.replace(tag, src_data[tag[1:-1]])
329 elif isinstance(data, dict):
331 for key, value in data.items():
332 tag = self._find_tag(value)
335 data[key] = value.replace(tag, src_data[tag[1:-1]])
338 raise PresentationError("Not possible to replace the "
339 "tag '{}'".format(tag))
341 self._replace_tags(data, src_data)
343 raise PresentationError("Replace tags: Not supported data type.")
347 def _parse_env(self):
348 """Parse environment specification in the specification YAML file.
351 logging.info("Parsing specification file: environment ...")
353 idx = self._get_type_index("environment")
358 self._specification["environment"]["configuration"] = \
359 self._cfg_yaml[idx]["configuration"]
361 self._specification["environment"]["configuration"] = None
364 self._specification["environment"]["paths"] = \
365 self._replace_tags(self._cfg_yaml[idx]["paths"])
367 self._specification["environment"]["paths"] = None
370 self._specification["environment"]["urls"] = \
371 self._replace_tags(self._cfg_yaml[idx]["urls"])
373 self._specification["environment"]["urls"] = None
376 self._specification["environment"]["make-dirs"] = \
377 self._cfg_yaml[idx]["make-dirs"]
379 self._specification["environment"]["make-dirs"] = None
382 self._specification["environment"]["remove-dirs"] = \
383 self._cfg_yaml[idx]["remove-dirs"]
385 self._specification["environment"]["remove-dirs"] = None
388 self._specification["environment"]["build-dirs"] = \
389 self._cfg_yaml[idx]["build-dirs"]
391 self._specification["environment"]["build-dirs"] = None
393 logging.info("Done.")
395 def _parse_configuration(self):
396 """Parse configuration of PAL in the specification YAML file.
399 logging.info("Parsing specification file: configuration ...")
401 idx = self._get_type_index("configuration")
403 logging.warning("No configuration information in the specification "
408 self._specification["configuration"] = self._cfg_yaml[idx]
411 raise PresentationError("No configuration defined.")
413 # Data sets: Replace ranges by lists
414 for set_name, data_set in self.configuration["data-sets"].items():
415 for job, builds in data_set.items():
417 if isinstance(builds, dict):
418 build_nr = builds.get("end", None)
420 build_nr = int(build_nr)
422 # defined as a range <start, build_type>
423 build_nr = self._get_build_number(job, build_nr)
424 builds = [x for x in range(builds["start"], build_nr+1)]
425 self.configuration["data-sets"][set_name][job] = builds
426 logging.info("Done.")
428 def _parse_debug(self):
429 """Parse debug specification in the specification YAML file.
432 if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
435 logging.info("Parsing specification file: debug ...")
437 idx = self._get_type_index("debug")
439 self.environment["configuration"]["CFG[DEBUG]"] = 0
443 for key, value in self._cfg_yaml[idx]["general"].items():
444 self._specification["debug"][key] = value
446 self._specification["input"]["builds"] = dict()
447 for job, builds in self._cfg_yaml[idx]["builds"].items():
449 self._specification["input"]["builds"][job] = list()
451 self._specification["input"]["builds"][job].\
452 append({"build": build["build"],
453 "status": "downloaded",
454 "file-name": self._replace_tags(
456 self.environment["paths"])})
458 logging.warning("No build is defined for the job '{}'. "
459 "Trying to continue without it.".
463 raise PresentationError("No data to process.")
465 def _parse_input(self):
466 """Parse input specification in the specification YAML file.
468 :raises: PresentationError if there are no data to process.
471 logging.info("Parsing specification file: input ...")
473 idx = self._get_type_index("input")
475 raise PresentationError("No data to process.")
478 for key, value in self._cfg_yaml[idx]["general"].items():
479 self._specification["input"][key] = value
480 self._specification["input"]["builds"] = dict()
482 for job, builds in self._cfg_yaml[idx]["builds"].items():
484 if isinstance(builds, dict):
485 build_nr = builds.get("end", None)
487 build_nr = int(build_nr)
489 # defined as a range <start, build_type>
490 build_nr = self._get_build_number(job, build_nr)
491 builds = [x for x in range(builds["start"], build_nr+1)]
492 self._specification["input"]["builds"][job] = list()
494 self._specification["input"]["builds"][job]. \
495 append({"build": build, "status": None})
498 logging.warning("No build is defined for the job '{}'. "
499 "Trying to continue without it.".
502 raise PresentationError("No data to process.")
504 logging.info("Done.")
506 def _parse_output(self):
507 """Parse output specification in the specification YAML file.
509 :raises: PresentationError if there is no output defined.
512 logging.info("Parsing specification file: output ...")
514 idx = self._get_type_index("output")
516 raise PresentationError("No output defined.")
519 self._specification["output"] = self._cfg_yaml[idx]
520 except (KeyError, IndexError):
521 raise PresentationError("No output defined.")
523 logging.info("Done.")
525 def _parse_static(self):
526 """Parse specification of the static content in the specification YAML
530 logging.info("Parsing specification file: static content ...")
532 idx = self._get_type_index("static")
534 logging.warning("No static content specified.")
536 for key, value in self._cfg_yaml[idx].items():
537 if isinstance(value, str):
539 self._cfg_yaml[idx][key] = self._replace_tags(
540 value, self._specification["environment"]["paths"])
544 self._specification["static"] = self._cfg_yaml[idx]
546 logging.info("Done.")
548 def _parse_elements(self):
549 """Parse elements (tables, plots) specification in the specification
553 logging.info("Parsing specification file: elements ...")
556 for element in self._cfg_yaml:
558 element["output-file"] = self._replace_tags(
559 element["output-file"],
560 self._specification["environment"]["paths"])
564 # add data sets to the elements:
565 if isinstance(element.get("data", None), str):
566 data_set = element["data"]
568 element["data"] = self.configuration["data-sets"][data_set]
570 raise PresentationError("Data set {0} is not defined in "
571 "the configuration section.".
574 if element["type"] == "table":
575 logging.info(" {:3d} Processing a table ...".format(count))
577 element["template"] = self._replace_tags(
579 self._specification["environment"]["paths"])
582 self._specification["tables"].append(element)
585 elif element["type"] == "plot":
586 logging.info(" {:3d} Processing a plot ...".format(count))
588 # Add layout to the plots:
589 layout = element["layout"].get("layout", None)
590 if layout is not None:
591 element["layout"].pop("layout")
593 for key, val in (self.configuration["plot-layouts"]
595 element["layout"][key] = val
597 raise PresentationError("Layout {0} is not defined in "
598 "the configuration section.".
600 self._specification["plots"].append(element)
603 elif element["type"] == "file":
604 logging.info(" {:3d} Processing a file ...".format(count))
606 element["dir-tables"] = self._replace_tags(
607 element["dir-tables"],
608 self._specification["environment"]["paths"])
611 self._specification["files"].append(element)
614 elif element["type"] == "cpta":
615 logging.info(" {:3d} Processing Continuous Performance "
616 "Trending and Analysis ...".format(count))
618 for plot in element["plots"]:
619 # Add layout to the plots:
620 layout = plot.get("layout", None)
621 if layout is not None:
624 self.configuration["plot-layouts"][layout]
626 raise PresentationError(
627 "Layout {0} is not defined in the "
628 "configuration section.".format(layout))
630 if isinstance(plot.get("data", None), str):
631 data_set = plot["data"]
634 self.configuration["data-sets"][data_set]
636 raise PresentationError(
637 "Data set {0} is not defined in "
638 "the configuration section.".
640 self._specification["cpta"] = element
643 logging.info("Done.")
645 def read_specification(self):
646 """Parse specification in the specification YAML file.
648 :raises: PresentationError if an error occurred while parsing the
652 self._cfg_yaml = load(self._cfg_file)
653 except YAMLError as err:
654 raise PresentationError(msg="An error occurred while parsing the "
655 "specification file.",
659 self._parse_configuration()
665 self._parse_elements()
667 logging.debug("Specification: \n{}".
668 format(pformat(self._specification)))