resources/tools/presentation/configuration.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Configuration
  15
  16 Parsing of the configuration YAML file.
  17 """
  18
  19
  20 import logging
  21 from yaml import load, YAMLError
  22 from pprint import pformat
  23
  24 from errors import PresentationError
  25
  26
  27 class Configuration(object):
  28     """Configuration of Presentation and analytics layer.
  29
  30     - based on configuration specified in the configuration YAML file
  31     - presentation and analytics layer is model driven
  32     """
  33
  34     # Tags are used in configuration YAML file and replaced while the file is
  35     # parsed.
  36     TAG_OPENER = "{"
  37     TAG_CLOSER = "}"
  38
  39     def __init__(self, cfg_file):
  40         """Initialization.
  41
  42         :param cfg_file: File handler for the configuration YAML file.
  43         :type cfg_file: BinaryIO
  44         """
  45         self._cfg_file = cfg_file
  46         self._cfg_yaml = None
  47
  48         # The configuration is stored in this directory.
  49         self._configuration = {"environment": dict(),
  50                                "debug": dict(),
  51                                "input": dict(),
  52                                "output": dict(),
  53                                "tables": list(),
  54                                "plots": list()}
  55
  56     @property
  57     def configuration(self):
  58         """Getter - configuration.
  59
  60         :returns: Configuration.
  61         :rtype: dict
  62         """
  63         return self._configuration
  64
  65     @property
  66     def environment(self):
  67         """Getter - environment.
  68
  69         :returns: Environment configuration.
  70         :rtype: dict
  71         """
  72         return self._configuration["environment"]
  73
  74     @property
  75     def debug(self):
  76         """Getter - debug
  77
  78         :returns: Debug configuration
  79         :rtype: dict
  80         """
  81         return self._configuration["debug"]
  82
  83     @property
  84     def is_debug(self):
  85         """Getter - debug mode
  86
  87         :returns: True if debug mode is on, otherwise False.
  88         :rtype: bool
  89         """
  90
  91         try:
  92             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
  93                 return True
  94             else:
  95                 return False
  96         except KeyError:
  97             return False
  98
  99     @property
 100     def input(self):
 101         """Getter - configuration - inputs.
 102         - jobs and builds.
 103
 104         :returns: Inputs.
 105         :rtype: dict
 106         """
 107         return self._configuration["input"]
 108
 109     @property
 110     def builds(self):
 111         """Getter - builds defined in configuration.
 112
 113         :returns: Builds defined in the configuration.
 114         :rtype: dict
 115         """
 116         return self.input["builds"]
 117
 118     @property
 119     def output(self):
 120         """Getter - configuration - output formats and versions to be generated.
 121         - formats: html, pdf
 122         - versions: full, ...
 123
 124         :returns: Outputs to be generated.
 125         :rtype: dict
 126         """
 127         return self._configuration["output"]
 128
 129     @property
 130     def tables(self):
 131         """Getter - tables to be generated.
 132
 133         :returns: List of specifications of tables to be generated.
 134         :rtype: list
 135         """
 136         return self._configuration["tables"]
 137
 138     @property
 139     def plots(self):
 140         """Getter - plots to be generated.
 141
 142         :returns: List of specifications of plots to be generated.
 143         :rtype: list
 144         """
 145         return self._configuration["plots"]
 146
 147     def set_input_state(self, job, build_nr, state):
 148         """Set the state of input
 149
 150         :param job:
 151         :param build_nr:
 152         :param state:
 153         :return:
 154         """
 155
 156         try:
 157             for build in self._configuration["input"]["builds"][job]:
 158                 if build["build"] == build_nr:
 159                     build["status"] = state
 160                     break
 161             else:
 162                 raise PresentationError("Build '{}' is not defined for job '{}'"
 163                                         " in configuration file.".
 164                                         format(build_nr, job))
 165         except KeyError:
 166             raise PresentationError("Job '{}' and build '{}' is not defined in "
 167                                     "configuration file.".format(job, build_nr))
 168
 169     def set_input_file_name(self, job, build_nr, file_name):
 170         """Set the state of input
 171
 172         :param job:
 173         :param build_nr:
 174         :param file_name:
 175         :return:
 176         """
 177
 178         try:
 179             for build in self._configuration["input"]["builds"][job]:
 180                 if build["build"] == build_nr:
 181                     build["file-name"] = file_name
 182                     break
 183             else:
 184                 raise PresentationError("Build '{}' is not defined for job '{}'"
 185                                         " in configuration file.".
 186                                         format(build_nr, job))
 187         except KeyError:
 188             raise PresentationError("Job '{}' and build '{}' is not defined in "
 189                                     "configuration file.".format(job, build_nr))
 190
 191     def _get_type_index(self, item_type):
 192         """Get index of item type (environment, input, output, ...) in
 193         configuration YAML file.
 194
 195         :param item_type: Item type: Top level items in configuration YAML file,
 196         e.g.: environment, input, output.
 197         :type item_type: str
 198         :returns: Index of the given item type.
 199         :rtype: int
 200         """
 201
 202         index = 0
 203         for item in self._cfg_yaml:
 204             if item["type"] == item_type:
 205                 return index
 206             index += 1
 207         return None
 208
 209     def _find_tag(self, text):
 210         """Find the first tag in the given text. The tag is enclosed by the
 211         TAG_OPENER and TAG_CLOSER.
 212
 213         :param text: Text to be searched.
 214         :type text: str
 215         :returns: The tag, or None if not found.
 216         :rtype: str
 217         """
 218         try:
 219             start = text.index(self.TAG_OPENER)
 220             end = text.index(self.TAG_CLOSER, start + 1) + 1
 221             return text[start:end]
 222         except ValueError:
 223             return None
 224
 225     def _replace_tags(self, data, src_data=None):
 226         """Replace tag(s) in the data by their values.
 227
 228         :param data: The data where the tags will be replaced by their values.
 229         :param src_data: Data where the tags are defined. It is dictionary where
 230         the key is the tag and the value is the tag value. If not given, 'data'
 231         is used instead.
 232         :type data: str or dict
 233         :type src_data: dict
 234         :returns: Data with the tags replaced.
 235         :rtype: str or dict
 236         :raises: PresentationError if it is not possible to replace the tag or
 237         the data is not the supported data type (str, dict).
 238         """
 239
 240         if src_data is None:
 241             src_data = data
 242
 243         if isinstance(data, str):
 244             tag = self._find_tag(data)
 245             if tag is not None:
 246                 data = data.replace(tag, src_data[tag[1:-1]])
 247
 248         elif isinstance(data, dict):
 249             counter = 0
 250             for key, value in data.items():
 251                 tag = self._find_tag(value)
 252                 if tag is not None:
 253                     try:
 254                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 255                         counter += 1
 256                     except KeyError:
 257                         raise PresentationError("Not possible to replace the "
 258                                                 "tag '{}'".format(tag))
 259             if counter:
 260                 self._replace_tags(data, src_data)
 261         else:
 262             raise PresentationError("Replace tags: Not supported data type.")
 263
 264         return data
 265
 266     def _parse_env(self):
 267         """Parse environment configuration in the configuration YAML file.
 268         """
 269
 270         logging.info("Parsing configuration file: environment ...")
 271
 272         idx = self._get_type_index("environment")
 273         if idx is None:
 274             return None
 275
 276         try:
 277             self._configuration["environment"]["configuration"] = \
 278                 self._cfg_yaml[idx]["configuration"]
 279         except KeyError:
 280             self._configuration["environment"]["configuration"] = None
 281
 282         try:
 283             self._configuration["environment"]["paths"] = \
 284                 self._replace_tags(self._cfg_yaml[idx]["paths"])
 285         except KeyError:
 286             self._configuration["environment"]["paths"] = None
 287
 288         try:
 289             self._configuration["environment"]["urls"] = \
 290                 self._replace_tags(self._cfg_yaml[idx]["urls"])
 291         except KeyError:
 292             self._configuration["environment"]["urls"] = None
 293
 294         try:
 295             self._configuration["environment"]["make-dirs"] = \
 296                 self._cfg_yaml[idx]["make-dirs"]
 297         except KeyError:
 298             self._configuration["environment"]["make-dirs"] = None
 299
 300         try:
 301             self._configuration["environment"]["remove-dirs"] = \
 302                 self._cfg_yaml[idx]["remove-dirs"]
 303         except KeyError:
 304             self._configuration["environment"]["remove-dirs"] = None
 305
 306         try:
 307             self._configuration["environment"]["build-dirs"] = \
 308                 self._cfg_yaml[idx]["build-dirs"]
 309         except KeyError:
 310             self._configuration["environment"]["build-dirs"] = None
 311
 312         logging.info("Done.")
 313
 314     def _parse_debug(self):
 315         """Parse debug configuration in the configuration YAML file.
 316         """
 317
 318         logging.info("Parsing configuration file: debug ...")
 319
 320         idx = self._get_type_index("debug")
 321         if idx is None:
 322             self.environment["configuration"]["CFG[DEBUG]"] = 0
 323             return None
 324
 325         try:
 326             for key, value in self._cfg_yaml[idx]["general"].items():
 327                 self._configuration["debug"][key] = value
 328
 329             self._configuration["input"]["builds"] = dict()
 330             for job, builds in self._cfg_yaml[idx]["builds"].items():
 331                 if builds:
 332                     self._configuration["input"]["builds"][job] = list()
 333                     for build in builds:
 334                         self._configuration["input"]["builds"][job].\
 335                             append({"build": build["build"],
 336                                     "status": "downloaded",
 337                                     "file-name": self._replace_tags(
 338                                         build["file"],
 339                                         self.environment["paths"])})
 340                 else:
 341                     logging.warning("No build is defined for the job '{}'. "
 342                                     "Trying to continue without it.".
 343                                     format(job))
 344
 345         except KeyError:
 346             raise PresentationError("No data to process.")
 347
 348     def _parse_input(self):
 349         """Parse input configuration in the configuration YAML file.
 350
 351         :raises: PresentationError if there are no data to process.
 352         """
 353
 354         logging.info("Parsing configuration file: input ...")
 355
 356         idx = self._get_type_index("input")
 357         if idx is None:
 358             raise PresentationError("No data to process.")
 359
 360         try:
 361             for key, value in self._cfg_yaml[idx]["general"].items():
 362                 self._configuration["input"][key] = value
 363             self._configuration["input"]["builds"] = dict()
 364             for job, builds in self._cfg_yaml[idx]["builds"].items():
 365                 if builds:
 366                     self._configuration["input"]["builds"][job] = list()
 367                     for build in builds:
 368                         self._configuration["input"]["builds"][job].\
 369                             append({"build": build, "status": None})
 370                 else:
 371                     logging.warning("No build is defined for the job '{}'. "
 372                                     "Trying to continue without it.".
 373                                     format(job))
 374         except KeyError:
 375             raise PresentationError("No data to process.")
 376
 377         logging.info("Done.")
 378
 379     def _parse_output(self):
 380         """Parse output configuration in the configuration YAML file.
 381
 382         :raises: PresentationError if there is no output defined.
 383         """
 384
 385         logging.info("Parsing configuration file: output ...")
 386
 387         idx = self._get_type_index("output")
 388         if idx is None:
 389             raise PresentationError("No output defined.")
 390
 391         try:
 392             self._configuration["output"] = self._cfg_yaml[idx]["format"]
 393         except KeyError:
 394             raise PresentationError("No output defined.")
 395
 396         logging.info("Done.")
 397
 398     def _parse_elements(self):
 399         """Parse elements (tables, plots) configuration in the configuration
 400         YAML file.
 401         """
 402
 403         logging.info("Parsing configuration file: elements ...")
 404
 405         count = 1
 406         for element in self._cfg_yaml:
 407             try:
 408                 element["output-file"] = self._replace_tags(
 409                     element["output-file"],
 410                     self._configuration["environment"]["paths"])
 411             except KeyError:
 412                 pass
 413             if element["type"] == "table":
 414                 logging.info("  {:3d} Processing a table ...".format(count))
 415                 self._configuration["tables"].append(element)
 416                 count += 1
 417             elif element["type"] == "plot":
 418                 logging.info("  {:3d} Processing a plot ...".format(count))
 419                 self._configuration["plots"].append(element)
 420                 count += 1
 421
 422         logging.info("Done.")
 423
 424     def parse_cfg(self):
 425         """Parse configuration in the configuration YAML file.
 426
 427         :raises: PresentationError if An error occurred while parsing the
 428         configuration file.
 429         """
 430         try:
 431             self._cfg_yaml = load(self._cfg_file)
 432         except YAMLError as err:
 433             raise PresentationError(msg="An error occurred while parsing the "
 434                                         "configuration file.",
 435                                     details=str(err))
 436
 437         self._parse_env()
 438         self._parse_debug()
 439         if not self.debug:
 440             self._parse_input()
 441         self._parse_output()
 442         self._parse_elements()
 443
 444         logging.debug("Configuration: \n{}".
 445                       format(pformat(self._configuration)))