resources/tools/presentation/specification_parser.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 import logging
  21 from yaml import load, YAMLError
  22 from pprint import pformat
  23
  24 from errors import PresentationError
  25
  26
  27 class Specification(object):
  28     """Specification of Presentation and analytics layer.
  29
  30     - based on specification specified in the specification YAML file
  31     - presentation and analytics layer is model driven
  32     """
  33
  34     # Tags are used in specification YAML file and replaced while the file is
  35     # parsed.
  36     TAG_OPENER = "{"
  37     TAG_CLOSER = "}"
  38
  39     def __init__(self, cfg_file):
  40         """Initialization.
  41
  42         :param cfg_file: File handler for the specification YAML file.
  43         :type cfg_file: BinaryIO
  44         """
  45         self._cfg_file = cfg_file
  46         self._cfg_yaml = None
  47
  48         self._specification = {"environment": dict(),
  49                                "configuration": dict(),
  50                                "debug": dict(),
  51                                "static": dict(),
  52                                "input": dict(),
  53                                "output": dict(),
  54                                "tables": list(),
  55                                "plots": list(),
  56                                "files": list()}
  57
  58     @property
  59     def specification(self):
  60         """Getter - specification.
  61
  62         :returns: Specification.
  63         :rtype: dict
  64         """
  65         return self._specification
  66
  67     @property
  68     def environment(self):
  69         """Getter - environment.
  70
  71         :returns: Environment specification.
  72         :rtype: dict
  73         """
  74         return self._specification["environment"]
  75
  76     @property
  77     def configuration(self):
  78         """Getter - configuration.
  79
  80         :returns: Configuration of PAL.
  81         :rtype: dict
  82         """
  83         return self._specification["configuration"]
  84
  85     @property
  86     def static(self):
  87         """Getter - static content.
  88
  89         :returns: Static content specification.
  90         :rtype: dict
  91         """
  92         return self._specification["static"]
  93
  94     @property
  95     def debug(self):
  96         """Getter - debug
  97
  98         :returns: Debug specification
  99         :rtype: dict
 100         """
 101         return self._specification["debug"]
 102
 103     @property
 104     def is_debug(self):
 105         """Getter - debug mode
 106
 107         :returns: True if debug mode is on, otherwise False.
 108         :rtype: bool
 109         """
 110
 111         try:
 112             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
 113                 return True
 114             else:
 115                 return False
 116         except KeyError:
 117             return False
 118
 119     @property
 120     def input(self):
 121         """Getter - specification - inputs.
 122         - jobs and builds.
 123
 124         :returns: Inputs.
 125         :rtype: dict
 126         """
 127         return self._specification["input"]
 128
 129     @property
 130     def builds(self):
 131         """Getter - builds defined in specification.
 132
 133         :returns: Builds defined in the specification.
 134         :rtype: dict
 135         """
 136         return self.input["builds"]
 137
 138     @property
 139     def output(self):
 140         """Getter - specification - output formats and versions to be generated.
 141         - formats: html, pdf
 142         - versions: full, ...
 143
 144         :returns: Outputs to be generated.
 145         :rtype: dict
 146         """
 147         return self._specification["output"]
 148
 149     @property
 150     def tables(self):
 151         """Getter - tables to be generated.
 152
 153         :returns: List of specifications of tables to be generated.
 154         :rtype: list
 155         """
 156         return self._specification["tables"]
 157
 158     @property
 159     def plots(self):
 160         """Getter - plots to be generated.
 161
 162         :returns: List of specifications of plots to be generated.
 163         :rtype: list
 164         """
 165         return self._specification["plots"]
 166
 167     @property
 168     def files(self):
 169         """Getter - files to be generated.
 170
 171         :returns: List of specifications of files to be generated.
 172         :rtype: list
 173         """
 174         return self._specification["files"]
 175
 176     def set_input_state(self, job, build_nr, state):
 177         """Set the state of input
 178
 179         :param job:
 180         :param build_nr:
 181         :param state:
 182         :return:
 183         """
 184
 185         try:
 186             for build in self._specification["input"]["builds"][job]:
 187                 if build["build"] == build_nr:
 188                     build["status"] = state
 189                     break
 190             else:
 191                 raise PresentationError("Build '{}' is not defined for job '{}'"
 192                                         " in specification file.".
 193                                         format(build_nr, job))
 194         except KeyError:
 195             raise PresentationError("Job '{}' and build '{}' is not defined in "
 196                                     "specification file.".format(job, build_nr))
 197
 198     def set_input_file_name(self, job, build_nr, file_name):
 199         """Set the state of input
 200
 201         :param job:
 202         :param build_nr:
 203         :param file_name:
 204         :return:
 205         """
 206
 207         try:
 208             for build in self._specification["input"]["builds"][job]:
 209                 if build["build"] == build_nr:
 210                     build["file-name"] = file_name
 211                     break
 212             else:
 213                 raise PresentationError("Build '{}' is not defined for job '{}'"
 214                                         " in specification file.".
 215                                         format(build_nr, job))
 216         except KeyError:
 217             raise PresentationError("Job '{}' and build '{}' is not defined in "
 218                                     "specification file.".format(job, build_nr))
 219
 220     def _get_type_index(self, item_type):
 221         """Get index of item type (environment, input, output, ...) in
 222         specification YAML file.
 223
 224         :param item_type: Item type: Top level items in specification YAML file,
 225         e.g.: environment, input, output.
 226         :type item_type: str
 227         :returns: Index of the given item type.
 228         :rtype: int
 229         """
 230
 231         index = 0
 232         for item in self._cfg_yaml:
 233             if item["type"] == item_type:
 234                 return index
 235             index += 1
 236         return None
 237
 238     def _find_tag(self, text):
 239         """Find the first tag in the given text. The tag is enclosed by the
 240         TAG_OPENER and TAG_CLOSER.
 241
 242         :param text: Text to be searched.
 243         :type text: str
 244         :returns: The tag, or None if not found.
 245         :rtype: str
 246         """
 247         try:
 248             start = text.index(self.TAG_OPENER)
 249             end = text.index(self.TAG_CLOSER, start + 1) + 1
 250             return text[start:end]
 251         except ValueError:
 252             return None
 253
 254     def _replace_tags(self, data, src_data=None):
 255         """Replace tag(s) in the data by their values.
 256
 257         :param data: The data where the tags will be replaced by their values.
 258         :param src_data: Data where the tags are defined. It is dictionary where
 259         the key is the tag and the value is the tag value. If not given, 'data'
 260         is used instead.
 261         :type data: str or dict
 262         :type src_data: dict
 263         :returns: Data with the tags replaced.
 264         :rtype: str or dict
 265         :raises: PresentationError if it is not possible to replace the tag or
 266         the data is not the supported data type (str, dict).
 267         """
 268
 269         if src_data is None:
 270             src_data = data
 271
 272         if isinstance(data, str):
 273             tag = self._find_tag(data)
 274             if tag is not None:
 275                 data = data.replace(tag, src_data[tag[1:-1]])
 276
 277         elif isinstance(data, dict):
 278             counter = 0
 279             for key, value in data.items():
 280                 tag = self._find_tag(value)
 281                 if tag is not None:
 282                     try:
 283                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 284                         counter += 1
 285                     except KeyError:
 286                         raise PresentationError("Not possible to replace the "
 287                                                 "tag '{}'".format(tag))
 288             if counter:
 289                 self._replace_tags(data, src_data)
 290         else:
 291             raise PresentationError("Replace tags: Not supported data type.")
 292
 293         return data
 294
 295     def _parse_env(self):
 296         """Parse environment specification in the specification YAML file.
 297         """
 298
 299         logging.info("Parsing specification file: environment ...")
 300
 301         idx = self._get_type_index("environment")
 302         if idx is None:
 303             return None
 304
 305         try:
 306             self._specification["environment"]["configuration"] = \
 307                 self._cfg_yaml[idx]["configuration"]
 308         except KeyError:
 309             self._specification["environment"]["configuration"] = None
 310
 311         try:
 312             self._specification["environment"]["paths"] = \
 313                 self._replace_tags(self._cfg_yaml[idx]["paths"])
 314         except KeyError:
 315             self._specification["environment"]["paths"] = None
 316
 317         try:
 318             self._specification["environment"]["urls"] = \
 319                 self._replace_tags(self._cfg_yaml[idx]["urls"])
 320         except KeyError:
 321             self._specification["environment"]["urls"] = None
 322
 323         try:
 324             self._specification["environment"]["make-dirs"] = \
 325                 self._cfg_yaml[idx]["make-dirs"]
 326         except KeyError:
 327             self._specification["environment"]["make-dirs"] = None
 328
 329         try:
 330             self._specification["environment"]["remove-dirs"] = \
 331                 self._cfg_yaml[idx]["remove-dirs"]
 332         except KeyError:
 333             self._specification["environment"]["remove-dirs"] = None
 334
 335         try:
 336             self._specification["environment"]["build-dirs"] = \
 337                 self._cfg_yaml[idx]["build-dirs"]
 338         except KeyError:
 339             self._specification["environment"]["build-dirs"] = None
 340
 341         logging.info("Done.")
 342
 343     def _parse_configuration(self):
 344         """Parse configuration of PAL in the specification YAML file.
 345         """
 346
 347         logging.info("Parsing specification file: configuration ...")
 348
 349         idx = self._get_type_index("configuration")
 350         if idx is None:
 351             logging.warning("No configuration information in the specification "
 352                             "file.")
 353             return None
 354
 355         try:
 356             self._specification["configuration"] = self._cfg_yaml[idx]
 357         except KeyError:
 358             raise PresentationError("No configuration defined.")
 359
 360         logging.info("Done.")
 361
 362     def _parse_debug(self):
 363         """Parse debug specification in the specification YAML file.
 364         """
 365
 366         if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
 367             return None
 368
 369         logging.info("Parsing specification file: debug ...")
 370
 371         idx = self._get_type_index("debug")
 372         if idx is None:
 373             self.environment["configuration"]["CFG[DEBUG]"] = 0
 374             return None
 375
 376         try:
 377             for key, value in self._cfg_yaml[idx]["general"].items():
 378                 self._specification["debug"][key] = value
 379
 380             self._specification["input"]["builds"] = dict()
 381             for job, builds in self._cfg_yaml[idx]["builds"].items():
 382                 if builds:
 383                     self._specification["input"]["builds"][job] = list()
 384                     for build in builds:
 385                         self._specification["input"]["builds"][job].\
 386                             append({"build": build["build"],
 387                                     "status": "downloaded",
 388                                     "file-name": self._replace_tags(
 389                                         build["file"],
 390                                         self.environment["paths"])})
 391                 else:
 392                     logging.warning("No build is defined for the job '{}'. "
 393                                     "Trying to continue without it.".
 394                                     format(job))
 395
 396         except KeyError:
 397             raise PresentationError("No data to process.")
 398
 399     def _parse_input(self):
 400         """Parse input specification in the specification YAML file.
 401
 402         :raises: PresentationError if there are no data to process.
 403         """
 404
 405         logging.info("Parsing specification file: input ...")
 406
 407         idx = self._get_type_index("input")
 408         if idx is None:
 409             raise PresentationError("No data to process.")
 410
 411         try:
 412             for key, value in self._cfg_yaml[idx]["general"].items():
 413                 self._specification["input"][key] = value
 414             self._specification["input"]["builds"] = dict()
 415             for job, builds in self._cfg_yaml[idx]["builds"].items():
 416                 if builds:
 417                     self._specification["input"]["builds"][job] = list()
 418                     for build in builds:
 419                         self._specification["input"]["builds"][job].\
 420                             append({"build": build, "status": None})
 421                 else:
 422                     logging.warning("No build is defined for the job '{}'. "
 423                                     "Trying to continue without it.".
 424                                     format(job))
 425         except KeyError:
 426             raise PresentationError("No data to process.")
 427
 428         logging.info("Done.")
 429
 430     def _parse_output(self):
 431         """Parse output specification in the specification YAML file.
 432
 433         :raises: PresentationError if there is no output defined.
 434         """
 435
 436         logging.info("Parsing specification file: output ...")
 437
 438         idx = self._get_type_index("output")
 439         if idx is None:
 440             raise PresentationError("No output defined.")
 441
 442         try:
 443             self._specification["output"] = self._cfg_yaml[idx]["format"]
 444         except KeyError:
 445             raise PresentationError("No output defined.")
 446
 447         logging.info("Done.")
 448
 449     def _parse_static(self):
 450         """Parse specification of the static content in the specification YAML
 451         file.
 452         """
 453
 454         logging.info("Parsing specification file: static content ...")
 455
 456         idx = self._get_type_index("static")
 457         if idx is None:
 458             logging.warning("No static content specified.")
 459
 460         for key, value in self._cfg_yaml[idx].items():
 461             if isinstance(value, str):
 462                 try:
 463                     self._cfg_yaml[idx][key] = self._replace_tags(
 464                         value, self._specification["environment"]["paths"])
 465                 except KeyError:
 466                     pass
 467
 468         self._specification["static"] = self._cfg_yaml[idx]
 469
 470         logging.info("Done.")
 471
 472     def _parse_elements(self):
 473         """Parse elements (tables, plots) specification in the specification
 474         YAML file.
 475         """
 476
 477         logging.info("Parsing specification file: elements ...")
 478
 479         count = 1
 480         for element in self._cfg_yaml:
 481             try:
 482                 element["output-file"] = self._replace_tags(
 483                     element["output-file"],
 484                     self._specification["environment"]["paths"])
 485             except KeyError:
 486                 pass
 487
 488             # add data sets to the elements:
 489             if isinstance(element.get("data", None), str):
 490                 data_set = element["data"]
 491                 try:
 492                     element["data"] = self.configuration["data-sets"][data_set]
 493                 except KeyError:
 494                     raise PresentationError("Data set {0} is not defined in "
 495                                             "the configuration section.".
 496                                             format(data_set))
 497
 498             if element["type"] == "table":
 499                 logging.info("  {:3d} Processing a table ...".format(count))
 500                 try:
 501                     element["template"] = self._replace_tags(
 502                         element["template"],
 503                         self._specification["environment"]["paths"])
 504                 except KeyError:
 505                     pass
 506                 self._specification["tables"].append(element)
 507                 count += 1
 508
 509             elif element["type"] == "plot":
 510                 logging.info("  {:3d} Processing a plot ...".format(count))
 511
 512                 # Add layout to the plots:
 513                 layout = element["layout"].get("layout", None)
 514                 if layout is not None:
 515                     element["layout"].pop("layout")
 516                     try:
 517                         for key, val in (self.configuration["plot-layouts"]
 518                                          [layout].items()):
 519                             element["layout"][key] = val
 520                     except KeyError:
 521                         raise PresentationError("Layout {0} is not defined in "
 522                                                 "the configuration section.".
 523                                                 format(layout))
 524                 self._specification["plots"].append(element)
 525                 count += 1
 526
 527             elif element["type"] == "file":
 528                 logging.info("  {:3d} Processing a file ...".format(count))
 529                 try:
 530                     element["dir-tables"] = self._replace_tags(
 531                         element["dir-tables"],
 532                         self._specification["environment"]["paths"])
 533                 except KeyError:
 534                     pass
 535                 self._specification["files"].append(element)
 536                 count += 1
 537
 538         logging.info("Done.")
 539
 540     def read_specification(self):
 541         """Parse specification in the specification YAML file.
 542
 543         :raises: PresentationError if an error occurred while parsing the
 544         specification file.
 545         """
 546         try:
 547             self._cfg_yaml = load(self._cfg_file)
 548         except YAMLError as err:
 549             raise PresentationError(msg="An error occurred while parsing the "
 550                                         "specification file.",
 551                                     details=str(err))
 552
 553         self._parse_env()
 554         self._parse_configuration()
 555         self._parse_debug()
 556         if not self.debug:
 557             self._parse_input()
 558         self._parse_output()
 559         self._parse_static()
 560         self._parse_elements()
 561
 562         logging.debug("Specification: \n{}".
 563                       format(pformat(self._specification)))