resources/tools/presentation/specification_parser.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 import logging
  21 from yaml import load, YAMLError
  22 from pprint import pformat
  23
  24 from errors import PresentationError
  25
  26
  27 class Specification(object):
  28     """Specification of Presentation and analytics layer.
  29
  30     - based on specification specified in the specification YAML file
  31     - presentation and analytics layer is model driven
  32     """
  33
  34     # Tags are used in specification YAML file and replaced while the file is
  35     # parsed.
  36     TAG_OPENER = "{"
  37     TAG_CLOSER = "}"
  38
  39     def __init__(self, cfg_file):
  40         """Initialization.
  41
  42         :param cfg_file: File handler for the specification YAML file.
  43         :type cfg_file: BinaryIO
  44         """
  45         self._cfg_file = cfg_file
  46         self._cfg_yaml = None
  47
  48         self._specification = {"environment": dict(),
  49                                "debug": dict(),
  50                                "static": dict(),
  51                                "input": dict(),
  52                                "output": dict(),
  53                                "tables": list(),
  54                                "plots": list(),
  55                                "files": list()}
  56
  57     @property
  58     def specification(self):
  59         """Getter - specification.
  60
  61         :returns: Specification.
  62         :rtype: dict
  63         """
  64         return self._specification
  65
  66     @property
  67     def environment(self):
  68         """Getter - environment.
  69
  70         :returns: Environment specification.
  71         :rtype: dict
  72         """
  73         return self._specification["environment"]
  74
  75     @property
  76     def static(self):
  77         """Getter - static content.
  78
  79         :returns: Static content specification.
  80         :rtype: dict
  81         """
  82         return self._specification["static"]
  83
  84     @property
  85     def debug(self):
  86         """Getter - debug
  87
  88         :returns: Debug specification
  89         :rtype: dict
  90         """
  91         return self._specification["debug"]
  92
  93     @property
  94     def is_debug(self):
  95         """Getter - debug mode
  96
  97         :returns: True if debug mode is on, otherwise False.
  98         :rtype: bool
  99         """
 100
 101         try:
 102             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
 103                 return True
 104             else:
 105                 return False
 106         except KeyError:
 107             return False
 108
 109     @property
 110     def input(self):
 111         """Getter - specification - inputs.
 112         - jobs and builds.
 113
 114         :returns: Inputs.
 115         :rtype: dict
 116         """
 117         return self._specification["input"]
 118
 119     @property
 120     def builds(self):
 121         """Getter - builds defined in specification.
 122
 123         :returns: Builds defined in the specification.
 124         :rtype: dict
 125         """
 126         return self.input["builds"]
 127
 128     @property
 129     def output(self):
 130         """Getter - specification - output formats and versions to be generated.
 131         - formats: html, pdf
 132         - versions: full, ...
 133
 134         :returns: Outputs to be generated.
 135         :rtype: dict
 136         """
 137         return self._specification["output"]
 138
 139     @property
 140     def tables(self):
 141         """Getter - tables to be generated.
 142
 143         :returns: List of specifications of tables to be generated.
 144         :rtype: list
 145         """
 146         return self._specification["tables"]
 147
 148     @property
 149     def plots(self):
 150         """Getter - plots to be generated.
 151
 152         :returns: List of specifications of plots to be generated.
 153         :rtype: list
 154         """
 155         return self._specification["plots"]
 156
 157     @property
 158     def files(self):
 159         """Getter - files to be generated.
 160
 161         :returns: List of specifications of files to be generated.
 162         :rtype: list
 163         """
 164         return self._specification["files"]
 165
 166     def set_input_state(self, job, build_nr, state):
 167         """Set the state of input
 168
 169         :param job:
 170         :param build_nr:
 171         :param state:
 172         :return:
 173         """
 174
 175         try:
 176             for build in self._specification["input"]["builds"][job]:
 177                 if build["build"] == build_nr:
 178                     build["status"] = state
 179                     break
 180             else:
 181                 raise PresentationError("Build '{}' is not defined for job '{}'"
 182                                         " in specification file.".
 183                                         format(build_nr, job))
 184         except KeyError:
 185             raise PresentationError("Job '{}' and build '{}' is not defined in "
 186                                     "specification file.".format(job, build_nr))
 187
 188     def set_input_file_name(self, job, build_nr, file_name):
 189         """Set the state of input
 190
 191         :param job:
 192         :param build_nr:
 193         :param file_name:
 194         :return:
 195         """
 196
 197         try:
 198             for build in self._specification["input"]["builds"][job]:
 199                 if build["build"] == build_nr:
 200                     build["file-name"] = file_name
 201                     break
 202             else:
 203                 raise PresentationError("Build '{}' is not defined for job '{}'"
 204                                         " in specification file.".
 205                                         format(build_nr, job))
 206         except KeyError:
 207             raise PresentationError("Job '{}' and build '{}' is not defined in "
 208                                     "specification file.".format(job, build_nr))
 209
 210     def _get_type_index(self, item_type):
 211         """Get index of item type (environment, input, output, ...) in
 212         specification YAML file.
 213
 214         :param item_type: Item type: Top level items in specification YAML file,
 215         e.g.: environment, input, output.
 216         :type item_type: str
 217         :returns: Index of the given item type.
 218         :rtype: int
 219         """
 220
 221         index = 0
 222         for item in self._cfg_yaml:
 223             if item["type"] == item_type:
 224                 return index
 225             index += 1
 226         return None
 227
 228     def _find_tag(self, text):
 229         """Find the first tag in the given text. The tag is enclosed by the
 230         TAG_OPENER and TAG_CLOSER.
 231
 232         :param text: Text to be searched.
 233         :type text: str
 234         :returns: The tag, or None if not found.
 235         :rtype: str
 236         """
 237         try:
 238             start = text.index(self.TAG_OPENER)
 239             end = text.index(self.TAG_CLOSER, start + 1) + 1
 240             return text[start:end]
 241         except ValueError:
 242             return None
 243
 244     def _replace_tags(self, data, src_data=None):
 245         """Replace tag(s) in the data by their values.
 246
 247         :param data: The data where the tags will be replaced by their values.
 248         :param src_data: Data where the tags are defined. It is dictionary where
 249         the key is the tag and the value is the tag value. If not given, 'data'
 250         is used instead.
 251         :type data: str or dict
 252         :type src_data: dict
 253         :returns: Data with the tags replaced.
 254         :rtype: str or dict
 255         :raises: PresentationError if it is not possible to replace the tag or
 256         the data is not the supported data type (str, dict).
 257         """
 258
 259         if src_data is None:
 260             src_data = data
 261
 262         if isinstance(data, str):
 263             tag = self._find_tag(data)
 264             if tag is not None:
 265                 data = data.replace(tag, src_data[tag[1:-1]])
 266
 267         elif isinstance(data, dict):
 268             counter = 0
 269             for key, value in data.items():
 270                 tag = self._find_tag(value)
 271                 if tag is not None:
 272                     try:
 273                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 274                         counter += 1
 275                     except KeyError:
 276                         raise PresentationError("Not possible to replace the "
 277                                                 "tag '{}'".format(tag))
 278             if counter:
 279                 self._replace_tags(data, src_data)
 280         else:
 281             raise PresentationError("Replace tags: Not supported data type.")
 282
 283         return data
 284
 285     def _parse_env(self):
 286         """Parse environment specification in the specification YAML file.
 287         """
 288
 289         logging.info("Parsing specification file: environment ...")
 290
 291         idx = self._get_type_index("environment")
 292         if idx is None:
 293             return None
 294
 295         try:
 296             self._specification["environment"]["configuration"] = \
 297                 self._cfg_yaml[idx]["configuration"]
 298         except KeyError:
 299             self._specification["environment"]["configuration"] = None
 300
 301         try:
 302             self._specification["environment"]["paths"] = \
 303                 self._replace_tags(self._cfg_yaml[idx]["paths"])
 304         except KeyError:
 305             self._specification["environment"]["paths"] = None
 306
 307         try:
 308             self._specification["environment"]["urls"] = \
 309                 self._replace_tags(self._cfg_yaml[idx]["urls"])
 310         except KeyError:
 311             self._specification["environment"]["urls"] = None
 312
 313         try:
 314             self._specification["environment"]["make-dirs"] = \
 315                 self._cfg_yaml[idx]["make-dirs"]
 316         except KeyError:
 317             self._specification["environment"]["make-dirs"] = None
 318
 319         try:
 320             self._specification["environment"]["remove-dirs"] = \
 321                 self._cfg_yaml[idx]["remove-dirs"]
 322         except KeyError:
 323             self._specification["environment"]["remove-dirs"] = None
 324
 325         try:
 326             self._specification["environment"]["build-dirs"] = \
 327                 self._cfg_yaml[idx]["build-dirs"]
 328         except KeyError:
 329             self._specification["environment"]["build-dirs"] = None
 330
 331         logging.info("Done.")
 332
 333     def _parse_debug(self):
 334         """Parse debug specification in the specification YAML file.
 335         """
 336
 337         if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
 338             return None
 339
 340         logging.info("Parsing specification file: debug ...")
 341
 342         idx = self._get_type_index("debug")
 343         if idx is None:
 344             self.environment["configuration"]["CFG[DEBUG]"] = 0
 345             return None
 346
 347         try:
 348             for key, value in self._cfg_yaml[idx]["general"].items():
 349                 self._specification["debug"][key] = value
 350
 351             self._specification["input"]["builds"] = dict()
 352             for job, builds in self._cfg_yaml[idx]["builds"].items():
 353                 if builds:
 354                     self._specification["input"]["builds"][job] = list()
 355                     for build in builds:
 356                         self._specification["input"]["builds"][job].\
 357                             append({"build": build["build"],
 358                                     "status": "downloaded",
 359                                     "file-name": self._replace_tags(
 360                                         build["file"],
 361                                         self.environment["paths"])})
 362                 else:
 363                     logging.warning("No build is defined for the job '{}'. "
 364                                     "Trying to continue without it.".
 365                                     format(job))
 366
 367         except KeyError:
 368             raise PresentationError("No data to process.")
 369
 370     def _parse_input(self):
 371         """Parse input specification in the specification YAML file.
 372
 373         :raises: PresentationError if there are no data to process.
 374         """
 375
 376         logging.info("Parsing specification file: input ...")
 377
 378         idx = self._get_type_index("input")
 379         if idx is None:
 380             raise PresentationError("No data to process.")
 381
 382         try:
 383             for key, value in self._cfg_yaml[idx]["general"].items():
 384                 self._specification["input"][key] = value
 385             self._specification["input"]["builds"] = dict()
 386             for job, builds in self._cfg_yaml[idx]["builds"].items():
 387                 if builds:
 388                     self._specification["input"]["builds"][job] = list()
 389                     for build in builds:
 390                         self._specification["input"]["builds"][job].\
 391                             append({"build": build, "status": None})
 392                 else:
 393                     logging.warning("No build is defined for the job '{}'. "
 394                                     "Trying to continue without it.".
 395                                     format(job))
 396         except KeyError:
 397             raise PresentationError("No data to process.")
 398
 399         logging.info("Done.")
 400
 401     def _parse_output(self):
 402         """Parse output specification in the specification YAML file.
 403
 404         :raises: PresentationError if there is no output defined.
 405         """
 406
 407         logging.info("Parsing specification file: output ...")
 408
 409         idx = self._get_type_index("output")
 410         if idx is None:
 411             raise PresentationError("No output defined.")
 412
 413         try:
 414             self._specification["output"] = self._cfg_yaml[idx]["format"]
 415         except KeyError:
 416             raise PresentationError("No output defined.")
 417
 418         logging.info("Done.")
 419
 420     def _parse_static(self):
 421         """Parse specification of the static content in the specification YAML
 422         file.
 423         """
 424
 425         logging.info("Parsing specification file: static content ...")
 426
 427         idx = self._get_type_index("static")
 428         if idx is None:
 429             logging.warning("No static content specified.")
 430
 431         for key, value in self._cfg_yaml[idx].items():
 432             if isinstance(value, str):
 433                 try:
 434                     self._cfg_yaml[idx][key] = self._replace_tags(
 435                         value, self._specification["environment"]["paths"])
 436                 except KeyError:
 437                     pass
 438
 439         self._specification["static"] = self._cfg_yaml[idx]
 440
 441         logging.info("Done.")
 442
 443     def _parse_elements(self):
 444         """Parse elements (tables, plots) specification in the specification
 445         YAML file.
 446         """
 447
 448         logging.info("Parsing specification file: elements ...")
 449
 450         count = 1
 451         for element in self._cfg_yaml:
 452             try:
 453                 element["output-file"] = self._replace_tags(
 454                     element["output-file"],
 455                     self._specification["environment"]["paths"])
 456             except KeyError:
 457                 pass
 458             if element["type"] == "table":
 459                 logging.info("  {:3d} Processing a table ...".format(count))
 460                 try:
 461                     element["template"] = self._replace_tags(
 462                         element["template"],
 463                         self._specification["environment"]["paths"])
 464                 except KeyError:
 465                     pass
 466                 self._specification["tables"].append(element)
 467                 count += 1
 468             elif element["type"] == "plot":
 469                 logging.info("  {:3d} Processing a plot ...".format(count))
 470                 self._specification["plots"].append(element)
 471                 count += 1
 472             elif element["type"] == "file":
 473                 logging.info("  {:3d} Processing a file ...".format(count))
 474                 try:
 475                     element["dir-tables"] = self._replace_tags(
 476                         element["dir-tables"],
 477                         self._specification["environment"]["paths"])
 478                 except KeyError:
 479                     pass
 480                 self._specification["files"].append(element)
 481                 count += 1
 482
 483         logging.info("Done.")
 484
 485     def read_specification(self):
 486         """Parse specification in the specification YAML file.
 487
 488         :raises: PresentationError if an error occurred while parsing the
 489         specification file.
 490         """
 491         try:
 492             self._cfg_yaml = load(self._cfg_file)
 493         except YAMLError as err:
 494             raise PresentationError(msg="An error occurred while parsing the "
 495                                         "specification file.",
 496                                     details=str(err))
 497
 498         self._parse_env()
 499         self._parse_debug()
 500         if not self.debug:
 501             self._parse_input()
 502         self._parse_output()
 503         self._parse_static()
 504         self._parse_elements()
 505
 506         logging.debug("Specification: \n{}".
 507                       format(pformat(self._specification)))