1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from yaml import load, YAMLError
22 from pprint import pformat
24 from errors import PresentationError
25 from utils import get_last_successful_build_number
26 from utils import get_last_completed_build_number
29 class Specification(object):
30 """Specification of Presentation and analytics layer.
32 - based on specification specified in the specification YAML file
33 - presentation and analytics layer is model driven
36 # Tags are used in specification YAML file and replaced while the file is
41 def __init__(self, cfg_file):
44 :param cfg_file: File handler for the specification YAML file.
45 :type cfg_file: BinaryIO
47 self._cfg_file = cfg_file
50 self._specification = {"environment": dict(),
51 "configuration": dict(),
61 def specification(self):
62 """Getter - specification.
64 :returns: Specification.
67 return self._specification
70 def environment(self):
71 """Getter - environment.
73 :returns: Environment specification.
76 return self._specification["environment"]
79 def configuration(self):
80 """Getter - configuration.
82 :returns: Configuration of PAL.
85 return self._specification["configuration"]
89 """Getter - static content.
91 :returns: Static content specification.
94 return self._specification["static"]
98 """Getter - debug mode
100 :returns: True if debug mode is on, otherwise False.
105 if self.environment["configuration"]["CFG[DEBUG]"] == 1:
114 """Getter - specification - inputs.
120 return self._specification["input"]
124 """Getter - builds defined in specification.
126 :returns: Builds defined in the specification.
129 return self.input["builds"]
133 """Getter - specification - output formats and versions to be generated.
135 - versions: full, ...
137 :returns: Outputs to be generated.
140 return self._specification["output"]
144 """Getter - tables to be generated.
146 :returns: List of specifications of tables to be generated.
149 return self._specification["tables"]
153 """Getter - plots to be generated.
155 :returns: List of specifications of plots to be generated.
158 return self._specification["plots"]
162 """Getter - files to be generated.
164 :returns: List of specifications of files to be generated.
167 return self._specification["files"]
171 """Getter - Continuous Performance Trending and Analysis to be
174 :returns: List of specifications of Continuous Performance Trending and
175 Analysis to be generated.
178 return self._specification["cpta"]
180 def set_input_state(self, job, build_nr, state):
181 """Set the state of input
190 for build in self._specification["input"]["builds"][job]:
191 if build["build"] == build_nr:
192 build["status"] = state
195 raise PresentationError("Build '{}' is not defined for job '{}'"
196 " in specification file.".
197 format(build_nr, job))
199 raise PresentationError("Job '{}' and build '{}' is not defined in "
200 "specification file.".format(job, build_nr))
202 def set_input_file_name(self, job, build_nr, file_name):
203 """Set the state of input
212 for build in self._specification["input"]["builds"][job]:
213 if build["build"] == build_nr:
214 build["file-name"] = file_name
217 raise PresentationError("Build '{}' is not defined for job '{}'"
218 " in specification file.".
219 format(build_nr, job))
221 raise PresentationError("Job '{}' and build '{}' is not defined in "
222 "specification file.".format(job, build_nr))
224 def _get_build_number(self, job, build_type):
225 """Get the number of the job defined by its name:
226 - lastSuccessfulBuild
229 :param job: Job name.
230 :param build_type: Build type:
231 - lastSuccessfulBuild
234 :raises PresentationError: If it is not possible to get the build
236 :returns: The build number.
240 # defined as a range <start, end>
241 if build_type == "lastSuccessfulBuild":
242 # defined as a range <start, lastSuccessfulBuild>
243 ret_code, build_nr, _ = get_last_successful_build_number(
244 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
245 elif build_type == "lastCompletedBuild":
246 # defined as a range <start, lastCompletedBuild>
247 ret_code, build_nr, _ = get_last_completed_build_number(
248 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
250 raise PresentationError("Not supported build type: '{0}'".
253 raise PresentationError("Not possible to get the number of the "
256 build_nr = int(build_nr)
258 except ValueError as err:
259 raise PresentationError("Not possible to get the number of the "
260 "build number.\nReason: {0}".format(err))
262 def _get_type_index(self, item_type):
263 """Get index of item type (environment, input, output, ...) in
264 specification YAML file.
266 :param item_type: Item type: Top level items in specification YAML file,
267 e.g.: environment, input, output.
269 :returns: Index of the given item type.
274 for item in self._cfg_yaml:
275 if item["type"] == item_type:
280 def _find_tag(self, text):
281 """Find the first tag in the given text. The tag is enclosed by the
282 TAG_OPENER and TAG_CLOSER.
284 :param text: Text to be searched.
286 :returns: The tag, or None if not found.
290 start = text.index(self.TAG_OPENER)
291 end = text.index(self.TAG_CLOSER, start + 1) + 1
292 return text[start:end]
296 def _replace_tags(self, data, src_data=None):
297 """Replace tag(s) in the data by their values.
299 :param data: The data where the tags will be replaced by their values.
300 :param src_data: Data where the tags are defined. It is dictionary where
301 the key is the tag and the value is the tag value. If not given, 'data'
303 :type data: str or dict
305 :returns: Data with the tags replaced.
307 :raises: PresentationError if it is not possible to replace the tag or
308 the data is not the supported data type (str, dict).
314 if isinstance(data, str):
315 tag = self._find_tag(data)
317 data = data.replace(tag, src_data[tag[1:-1]])
319 elif isinstance(data, dict):
321 for key, value in data.items():
322 tag = self._find_tag(value)
325 data[key] = value.replace(tag, src_data[tag[1:-1]])
328 raise PresentationError("Not possible to replace the "
329 "tag '{}'".format(tag))
331 self._replace_tags(data, src_data)
333 raise PresentationError("Replace tags: Not supported data type.")
337 def _parse_env(self):
338 """Parse environment specification in the specification YAML file.
341 logging.info("Parsing specification file: environment ...")
343 idx = self._get_type_index("environment")
348 self._specification["environment"]["configuration"] = \
349 self._cfg_yaml[idx]["configuration"]
351 self._specification["environment"]["configuration"] = None
354 self._specification["environment"]["paths"] = \
355 self._replace_tags(self._cfg_yaml[idx]["paths"])
357 self._specification["environment"]["paths"] = None
360 self._specification["environment"]["urls"] = \
361 self._replace_tags(self._cfg_yaml[idx]["urls"])
363 self._specification["environment"]["urls"] = None
366 self._specification["environment"]["make-dirs"] = \
367 self._cfg_yaml[idx]["make-dirs"]
369 self._specification["environment"]["make-dirs"] = None
372 self._specification["environment"]["remove-dirs"] = \
373 self._cfg_yaml[idx]["remove-dirs"]
375 self._specification["environment"]["remove-dirs"] = None
378 self._specification["environment"]["build-dirs"] = \
379 self._cfg_yaml[idx]["build-dirs"]
381 self._specification["environment"]["build-dirs"] = None
383 logging.info("Done.")
385 def _parse_configuration(self):
386 """Parse configuration of PAL in the specification YAML file.
389 logging.info("Parsing specification file: configuration ...")
391 idx = self._get_type_index("configuration")
393 logging.warning("No configuration information in the specification "
398 self._specification["configuration"] = self._cfg_yaml[idx]
401 raise PresentationError("No configuration defined.")
403 # Data sets: Replace ranges by lists
404 for set_name, data_set in self.configuration["data-sets"].items():
405 for job, builds in data_set.items():
407 if isinstance(builds, dict):
408 build_nr = builds.get("end", None)
410 build_nr = int(build_nr)
412 # defined as a range <start, build_type>
413 build_nr = self._get_build_number(job, build_nr)
414 builds = [x for x in range(builds["start"], build_nr+1)]
415 self.configuration["data-sets"][set_name][job] = builds
416 logging.info("Done.")
418 def _parse_input(self):
419 """Parse input specification in the specification YAML file.
421 :raises: PresentationError if there are no data to process.
424 logging.info("Parsing specification file: input ...")
426 idx = self._get_type_index("input")
428 raise PresentationError("No data to process.")
431 for key, value in self._cfg_yaml[idx]["general"].items():
432 self._specification["input"][key] = value
433 self._specification["input"]["builds"] = dict()
435 for job, builds in self._cfg_yaml[idx]["builds"].items():
437 if isinstance(builds, dict):
438 build_nr = builds.get("end", None)
440 build_nr = int(build_nr)
442 # defined as a range <start, build_type>
443 build_nr = self._get_build_number(job, build_nr)
444 builds = [x for x in range(builds["start"], build_nr+1)]
445 self._specification["input"]["builds"][job] = list()
447 self._specification["input"]["builds"][job]. \
448 append({"build": build, "status": None})
451 logging.warning("No build is defined for the job '{}'. "
452 "Trying to continue without it.".
455 raise PresentationError("No data to process.")
457 logging.info("Done.")
459 def _parse_output(self):
460 """Parse output specification in the specification YAML file.
462 :raises: PresentationError if there is no output defined.
465 logging.info("Parsing specification file: output ...")
467 idx = self._get_type_index("output")
469 raise PresentationError("No output defined.")
472 self._specification["output"] = self._cfg_yaml[idx]
473 except (KeyError, IndexError):
474 raise PresentationError("No output defined.")
476 logging.info("Done.")
478 def _parse_static(self):
479 """Parse specification of the static content in the specification YAML
483 logging.info("Parsing specification file: static content ...")
485 idx = self._get_type_index("static")
487 logging.warning("No static content specified.")
489 for key, value in self._cfg_yaml[idx].items():
490 if isinstance(value, str):
492 self._cfg_yaml[idx][key] = self._replace_tags(
493 value, self._specification["environment"]["paths"])
497 self._specification["static"] = self._cfg_yaml[idx]
499 logging.info("Done.")
501 def _parse_elements(self):
502 """Parse elements (tables, plots) specification in the specification
506 logging.info("Parsing specification file: elements ...")
509 for element in self._cfg_yaml:
511 element["output-file"] = self._replace_tags(
512 element["output-file"],
513 self._specification["environment"]["paths"])
518 element["input-file"] = self._replace_tags(
519 element["input-file"],
520 self._specification["environment"]["paths"])
524 # add data sets to the elements:
525 if isinstance(element.get("data", None), str):
526 data_set = element["data"]
528 element["data"] = self.configuration["data-sets"][data_set]
530 raise PresentationError("Data set {0} is not defined in "
531 "the configuration section.".
534 if element["type"] == "table":
535 logging.info(" {:3d} Processing a table ...".format(count))
537 element["template"] = self._replace_tags(
539 self._specification["environment"]["paths"])
542 self._specification["tables"].append(element)
545 elif element["type"] == "plot":
546 logging.info(" {:3d} Processing a plot ...".format(count))
548 # Add layout to the plots:
549 layout = element["layout"].get("layout", None)
550 if layout is not None:
551 element["layout"].pop("layout")
553 for key, val in (self.configuration["plot-layouts"]
555 element["layout"][key] = val
557 raise PresentationError("Layout {0} is not defined in "
558 "the configuration section.".
560 self._specification["plots"].append(element)
563 elif element["type"] == "file":
564 logging.info(" {:3d} Processing a file ...".format(count))
566 element["dir-tables"] = self._replace_tags(
567 element["dir-tables"],
568 self._specification["environment"]["paths"])
571 self._specification["files"].append(element)
574 elif element["type"] == "cpta":
575 logging.info(" {:3d} Processing Continuous Performance "
576 "Trending and Analysis ...".format(count))
578 for plot in element["plots"]:
579 # Add layout to the plots:
580 layout = plot.get("layout", None)
581 if layout is not None:
584 self.configuration["plot-layouts"][layout]
586 raise PresentationError(
587 "Layout {0} is not defined in the "
588 "configuration section.".format(layout))
590 if isinstance(plot.get("data", None), str):
591 data_set = plot["data"]
594 self.configuration["data-sets"][data_set]
596 raise PresentationError(
597 "Data set {0} is not defined in "
598 "the configuration section.".
600 self._specification["cpta"] = element
603 logging.info("Done.")
605 def read_specification(self):
606 """Parse specification in the specification YAML file.
608 :raises: PresentationError if an error occurred while parsing the
612 self._cfg_yaml = load(self._cfg_file)
613 except YAMLError as err:
614 raise PresentationError(msg="An error occurred while parsing the "
615 "specification file.",
619 self._parse_configuration()
623 self._parse_elements()
625 logging.debug("Specification: \n{}".
626 format(pformat(self._specification)))