1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from yaml import load, YAMLError
22 from pprint import pformat
24 from errors import PresentationError
25 from utils import get_last_build_number
28 class Specification(object):
29 """Specification of Presentation and analytics layer.
31 - based on specification specified in the specification YAML file
32 - presentation and analytics layer is model driven
35 # Tags are used in specification YAML file and replaced while the file is
40 def __init__(self, cfg_file):
43 :param cfg_file: File handler for the specification YAML file.
44 :type cfg_file: BinaryIO
46 self._cfg_file = cfg_file
49 self._specification = {"environment": dict(),
50 "configuration": dict(),
61 def specification(self):
62 """Getter - specification.
64 :returns: Specification.
67 return self._specification
70 def environment(self):
71 """Getter - environment.
73 :returns: Environment specification.
76 return self._specification["environment"]
79 def configuration(self):
80 """Getter - configuration.
82 :returns: Configuration of PAL.
85 return self._specification["configuration"]
89 """Getter - static content.
91 :returns: Static content specification.
94 return self._specification["static"]
100 :returns: Debug specification
103 return self._specification["debug"]
107 """Getter - debug mode
109 :returns: True if debug mode is on, otherwise False.
114 if self.environment["configuration"]["CFG[DEBUG]"] == 1:
123 """Getter - specification - inputs.
129 return self._specification["input"]
133 """Getter - builds defined in specification.
135 :returns: Builds defined in the specification.
138 return self.input["builds"]
142 """Getter - specification - output formats and versions to be generated.
144 - versions: full, ...
146 :returns: Outputs to be generated.
149 return self._specification["output"]
153 """Getter - tables to be generated.
155 :returns: List of specifications of tables to be generated.
158 return self._specification["tables"]
162 """Getter - plots to be generated.
164 :returns: List of specifications of plots to be generated.
167 return self._specification["plots"]
171 """Getter - files to be generated.
173 :returns: List of specifications of files to be generated.
176 return self._specification["files"]
180 """Getter - Continuous Performance Trending and Analysis to be
183 :returns: List of specifications of Continuous Performance Trending and
184 Analysis to be generated.
187 return self._specification["cpta"]
189 def set_input_state(self, job, build_nr, state):
190 """Set the state of input
199 for build in self._specification["input"]["builds"][job]:
200 if build["build"] == build_nr:
201 build["status"] = state
204 raise PresentationError("Build '{}' is not defined for job '{}'"
205 " in specification file.".
206 format(build_nr, job))
208 raise PresentationError("Job '{}' and build '{}' is not defined in "
209 "specification file.".format(job, build_nr))
211 def set_input_file_name(self, job, build_nr, file_name):
212 """Set the state of input
221 for build in self._specification["input"]["builds"][job]:
222 if build["build"] == build_nr:
223 build["file-name"] = file_name
226 raise PresentationError("Build '{}' is not defined for job '{}'"
227 " in specification file.".
228 format(build_nr, job))
230 raise PresentationError("Job '{}' and build '{}' is not defined in "
231 "specification file.".format(job, build_nr))
233 def _get_type_index(self, item_type):
234 """Get index of item type (environment, input, output, ...) in
235 specification YAML file.
237 :param item_type: Item type: Top level items in specification YAML file,
238 e.g.: environment, input, output.
240 :returns: Index of the given item type.
245 for item in self._cfg_yaml:
246 if item["type"] == item_type:
251 def _find_tag(self, text):
252 """Find the first tag in the given text. The tag is enclosed by the
253 TAG_OPENER and TAG_CLOSER.
255 :param text: Text to be searched.
257 :returns: The tag, or None if not found.
261 start = text.index(self.TAG_OPENER)
262 end = text.index(self.TAG_CLOSER, start + 1) + 1
263 return text[start:end]
267 def _replace_tags(self, data, src_data=None):
268 """Replace tag(s) in the data by their values.
270 :param data: The data where the tags will be replaced by their values.
271 :param src_data: Data where the tags are defined. It is dictionary where
272 the key is the tag and the value is the tag value. If not given, 'data'
274 :type data: str or dict
276 :returns: Data with the tags replaced.
278 :raises: PresentationError if it is not possible to replace the tag or
279 the data is not the supported data type (str, dict).
285 if isinstance(data, str):
286 tag = self._find_tag(data)
288 data = data.replace(tag, src_data[tag[1:-1]])
290 elif isinstance(data, dict):
292 for key, value in data.items():
293 tag = self._find_tag(value)
296 data[key] = value.replace(tag, src_data[tag[1:-1]])
299 raise PresentationError("Not possible to replace the "
300 "tag '{}'".format(tag))
302 self._replace_tags(data, src_data)
304 raise PresentationError("Replace tags: Not supported data type.")
308 def _parse_env(self):
309 """Parse environment specification in the specification YAML file.
312 logging.info("Parsing specification file: environment ...")
314 idx = self._get_type_index("environment")
319 self._specification["environment"]["configuration"] = \
320 self._cfg_yaml[idx]["configuration"]
322 self._specification["environment"]["configuration"] = None
325 self._specification["environment"]["paths"] = \
326 self._replace_tags(self._cfg_yaml[idx]["paths"])
328 self._specification["environment"]["paths"] = None
331 self._specification["environment"]["urls"] = \
332 self._replace_tags(self._cfg_yaml[idx]["urls"])
334 self._specification["environment"]["urls"] = None
337 self._specification["environment"]["make-dirs"] = \
338 self._cfg_yaml[idx]["make-dirs"]
340 self._specification["environment"]["make-dirs"] = None
343 self._specification["environment"]["remove-dirs"] = \
344 self._cfg_yaml[idx]["remove-dirs"]
346 self._specification["environment"]["remove-dirs"] = None
349 self._specification["environment"]["build-dirs"] = \
350 self._cfg_yaml[idx]["build-dirs"]
352 self._specification["environment"]["build-dirs"] = None
354 logging.info("Done.")
356 def _parse_configuration(self):
357 """Parse configuration of PAL in the specification YAML file.
360 logging.info("Parsing specification file: configuration ...")
362 idx = self._get_type_index("configuration")
364 logging.warning("No configuration information in the specification "
369 self._specification["configuration"] = self._cfg_yaml[idx]
372 raise PresentationError("No configuration defined.")
374 # Data sets: Replace ranges by lists
375 for set_name, data_set in self.configuration["data-sets"].items():
376 for job, builds in data_set.items():
378 if isinstance(builds, dict):
379 # defined as a range <start, end>
380 if builds.get("end", None) == "lastSuccessfulBuild":
381 # defined as a range <start, lastSuccessfulBuild>
382 ret_code, build_nr, _ = get_last_build_number(
383 self.environment["urls"]["URL[JENKINS,CSIT]"],
386 raise PresentationError(
387 "Not possible to get the number of the "
388 "last successful build.")
390 # defined as a range <start, end (build number)>
391 build_nr = builds.get("end", None)
392 builds = [x for x in range(1, int(build_nr)+1)]
393 self.configuration["data-sets"][set_name][job] = builds
395 logging.info("Done.")
397 def _parse_debug(self):
398 """Parse debug specification in the specification YAML file.
401 if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
404 logging.info("Parsing specification file: debug ...")
406 idx = self._get_type_index("debug")
408 self.environment["configuration"]["CFG[DEBUG]"] = 0
412 for key, value in self._cfg_yaml[idx]["general"].items():
413 self._specification["debug"][key] = value
415 self._specification["input"]["builds"] = dict()
416 for job, builds in self._cfg_yaml[idx]["builds"].items():
418 self._specification["input"]["builds"][job] = list()
420 self._specification["input"]["builds"][job].\
421 append({"build": build["build"],
422 "status": "downloaded",
423 "file-name": self._replace_tags(
425 self.environment["paths"])})
427 logging.warning("No build is defined for the job '{}'. "
428 "Trying to continue without it.".
432 raise PresentationError("No data to process.")
434 def _parse_input(self):
435 """Parse input specification in the specification YAML file.
437 :raises: PresentationError if there are no data to process.
440 logging.info("Parsing specification file: input ...")
442 idx = self._get_type_index("input")
444 raise PresentationError("No data to process.")
447 for key, value in self._cfg_yaml[idx]["general"].items():
448 self._specification["input"][key] = value
449 self._specification["input"]["builds"] = dict()
451 for job, builds in self._cfg_yaml[idx]["builds"].items():
453 if isinstance(builds, dict):
454 # defined as a range <start, end>
455 if builds.get("end", None) == "lastSuccessfulBuild":
456 # defined as a range <start, lastSuccessfulBuild>
457 ret_code, build_nr, _ = get_last_build_number(
458 self.environment["urls"]["URL[JENKINS,CSIT]"],
461 raise PresentationError(
462 "Not possible to get the number of the "
463 "last successful build.")
465 # defined as a range <start, end (build number)>
466 build_nr = builds.get("end", None)
467 builds = [x for x in range(builds["start"],
469 self._specification["input"]["builds"][job] = list()
471 self._specification["input"]["builds"][job].\
472 append({"build": build, "status": None})
474 logging.warning("No build is defined for the job '{}'. "
475 "Trying to continue without it.".
478 raise PresentationError("No data to process.")
480 logging.info("Done.")
482 def _parse_output(self):
483 """Parse output specification in the specification YAML file.
485 :raises: PresentationError if there is no output defined.
488 logging.info("Parsing specification file: output ...")
490 idx = self._get_type_index("output")
492 raise PresentationError("No output defined.")
495 self._specification["output"] = self._cfg_yaml[idx]
496 except (KeyError, IndexError):
497 raise PresentationError("No output defined.")
499 logging.info("Done.")
501 def _parse_static(self):
502 """Parse specification of the static content in the specification YAML
506 logging.info("Parsing specification file: static content ...")
508 idx = self._get_type_index("static")
510 logging.warning("No static content specified.")
512 for key, value in self._cfg_yaml[idx].items():
513 if isinstance(value, str):
515 self._cfg_yaml[idx][key] = self._replace_tags(
516 value, self._specification["environment"]["paths"])
520 self._specification["static"] = self._cfg_yaml[idx]
522 logging.info("Done.")
524 def _parse_elements(self):
525 """Parse elements (tables, plots) specification in the specification
529 logging.info("Parsing specification file: elements ...")
532 for element in self._cfg_yaml:
534 element["output-file"] = self._replace_tags(
535 element["output-file"],
536 self._specification["environment"]["paths"])
540 # add data sets to the elements:
541 if isinstance(element.get("data", None), str):
542 data_set = element["data"]
544 element["data"] = self.configuration["data-sets"][data_set]
546 raise PresentationError("Data set {0} is not defined in "
547 "the configuration section.".
550 if element["type"] == "table":
551 logging.info(" {:3d} Processing a table ...".format(count))
553 element["template"] = self._replace_tags(
555 self._specification["environment"]["paths"])
558 self._specification["tables"].append(element)
561 elif element["type"] == "plot":
562 logging.info(" {:3d} Processing a plot ...".format(count))
564 # Add layout to the plots:
565 layout = element["layout"].get("layout", None)
566 if layout is not None:
567 element["layout"].pop("layout")
569 for key, val in (self.configuration["plot-layouts"]
571 element["layout"][key] = val
573 raise PresentationError("Layout {0} is not defined in "
574 "the configuration section.".
576 self._specification["plots"].append(element)
579 elif element["type"] == "file":
580 logging.info(" {:3d} Processing a file ...".format(count))
582 element["dir-tables"] = self._replace_tags(
583 element["dir-tables"],
584 self._specification["environment"]["paths"])
587 self._specification["files"].append(element)
590 elif element["type"] == "cpta":
591 logging.info(" {:3d} Processing Continuous Performance "
592 "Trending and Analysis ...".format(count))
594 for plot in element["plots"]:
595 # Add layout to the plots:
596 layout = plot.get("layout", None)
597 if layout is not None:
600 self.configuration["plot-layouts"][layout]
602 raise PresentationError(
603 "Layout {0} is not defined in the "
604 "configuration section.".format(layout))
606 if isinstance(plot.get("data", None), str):
607 data_set = plot["data"]
610 self.configuration["data-sets"][data_set]
612 raise PresentationError(
613 "Data set {0} is not defined in "
614 "the configuration section.".
616 self._specification["cpta"] = element
619 logging.info("Done.")
621 def read_specification(self):
622 """Parse specification in the specification YAML file.
624 :raises: PresentationError if an error occurred while parsing the
628 self._cfg_yaml = load(self._cfg_file)
629 except YAMLError as err:
630 raise PresentationError(msg="An error occurred while parsing the "
631 "specification file.",
635 self._parse_configuration()
641 self._parse_elements()
643 logging.debug("Specification: \n{}".
644 format(pformat(self._specification)))