resources/tools/presentation/specification_parser.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 import logging
  21 from yaml import load, YAMLError
  22 from pprint import pformat
  23
  24 from errors import PresentationError
  25 from utils import get_last_build_number
  26
  27
  28 class Specification(object):
  29     """Specification of Presentation and analytics layer.
  30
  31     - based on specification specified in the specification YAML file
  32     - presentation and analytics layer is model driven
  33     """
  34
  35     # Tags are used in specification YAML file and replaced while the file is
  36     # parsed.
  37     TAG_OPENER = "{"
  38     TAG_CLOSER = "}"
  39
  40     def __init__(self, cfg_file):
  41         """Initialization.
  42
  43         :param cfg_file: File handler for the specification YAML file.
  44         :type cfg_file: BinaryIO
  45         """
  46         self._cfg_file = cfg_file
  47         self._cfg_yaml = None
  48
  49         self._specification = {"environment": dict(),
  50                                "configuration": dict(),
  51                                "debug": dict(),
  52                                "static": dict(),
  53                                "input": dict(),
  54                                "output": dict(),
  55                                "tables": list(),
  56                                "plots": list(),
  57                                "files": list(),
  58                                "cpta": dict()}
  59
  60     @property
  61     def specification(self):
  62         """Getter - specification.
  63
  64         :returns: Specification.
  65         :rtype: dict
  66         """
  67         return self._specification
  68
  69     @property
  70     def environment(self):
  71         """Getter - environment.
  72
  73         :returns: Environment specification.
  74         :rtype: dict
  75         """
  76         return self._specification["environment"]
  77
  78     @property
  79     def configuration(self):
  80         """Getter - configuration.
  81
  82         :returns: Configuration of PAL.
  83         :rtype: dict
  84         """
  85         return self._specification["configuration"]
  86
  87     @property
  88     def static(self):
  89         """Getter - static content.
  90
  91         :returns: Static content specification.
  92         :rtype: dict
  93         """
  94         return self._specification["static"]
  95
  96     @property
  97     def debug(self):
  98         """Getter - debug
  99
 100         :returns: Debug specification
 101         :rtype: dict
 102         """
 103         return self._specification["debug"]
 104
 105     @property
 106     def is_debug(self):
 107         """Getter - debug mode
 108
 109         :returns: True if debug mode is on, otherwise False.
 110         :rtype: bool
 111         """
 112
 113         try:
 114             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
 115                 return True
 116             else:
 117                 return False
 118         except KeyError:
 119             return False
 120
 121     @property
 122     def input(self):
 123         """Getter - specification - inputs.
 124         - jobs and builds.
 125
 126         :returns: Inputs.
 127         :rtype: dict
 128         """
 129         return self._specification["input"]
 130
 131     @property
 132     def builds(self):
 133         """Getter - builds defined in specification.
 134
 135         :returns: Builds defined in the specification.
 136         :rtype: dict
 137         """
 138         return self.input["builds"]
 139
 140     @property
 141     def output(self):
 142         """Getter - specification - output formats and versions to be generated.
 143         - formats: html, pdf
 144         - versions: full, ...
 145
 146         :returns: Outputs to be generated.
 147         :rtype: dict
 148         """
 149         return self._specification["output"]
 150
 151     @property
 152     def tables(self):
 153         """Getter - tables to be generated.
 154
 155         :returns: List of specifications of tables to be generated.
 156         :rtype: list
 157         """
 158         return self._specification["tables"]
 159
 160     @property
 161     def plots(self):
 162         """Getter - plots to be generated.
 163
 164         :returns: List of specifications of plots to be generated.
 165         :rtype: list
 166         """
 167         return self._specification["plots"]
 168
 169     @property
 170     def files(self):
 171         """Getter - files to be generated.
 172
 173         :returns: List of specifications of files to be generated.
 174         :rtype: list
 175         """
 176         return self._specification["files"]
 177
 178     @property
 179     def cpta(self):
 180         """Getter - Continuous Performance Trending and Analysis to be
 181         generated.
 182
 183         :returns: List of specifications of Continuous Performance Trending and
 184         Analysis to be generated.
 185         :rtype: list
 186         """
 187         return self._specification["cpta"]
 188
 189     def set_input_state(self, job, build_nr, state):
 190         """Set the state of input
 191
 192         :param job:
 193         :param build_nr:
 194         :param state:
 195         :return:
 196         """
 197
 198         try:
 199             for build in self._specification["input"]["builds"][job]:
 200                 if build["build"] == build_nr:
 201                     build["status"] = state
 202                     break
 203             else:
 204                 raise PresentationError("Build '{}' is not defined for job '{}'"
 205                                         " in specification file.".
 206                                         format(build_nr, job))
 207         except KeyError:
 208             raise PresentationError("Job '{}' and build '{}' is not defined in "
 209                                     "specification file.".format(job, build_nr))
 210
 211     def set_input_file_name(self, job, build_nr, file_name):
 212         """Set the state of input
 213
 214         :param job:
 215         :param build_nr:
 216         :param file_name:
 217         :return:
 218         """
 219
 220         try:
 221             for build in self._specification["input"]["builds"][job]:
 222                 if build["build"] == build_nr:
 223                     build["file-name"] = file_name
 224                     break
 225             else:
 226                 raise PresentationError("Build '{}' is not defined for job '{}'"
 227                                         " in specification file.".
 228                                         format(build_nr, job))
 229         except KeyError:
 230             raise PresentationError("Job '{}' and build '{}' is not defined in "
 231                                     "specification file.".format(job, build_nr))
 232
 233     def _get_type_index(self, item_type):
 234         """Get index of item type (environment, input, output, ...) in
 235         specification YAML file.
 236
 237         :param item_type: Item type: Top level items in specification YAML file,
 238         e.g.: environment, input, output.
 239         :type item_type: str
 240         :returns: Index of the given item type.
 241         :rtype: int
 242         """
 243
 244         index = 0
 245         for item in self._cfg_yaml:
 246             if item["type"] == item_type:
 247                 return index
 248             index += 1
 249         return None
 250
 251     def _find_tag(self, text):
 252         """Find the first tag in the given text. The tag is enclosed by the
 253         TAG_OPENER and TAG_CLOSER.
 254
 255         :param text: Text to be searched.
 256         :type text: str
 257         :returns: The tag, or None if not found.
 258         :rtype: str
 259         """
 260         try:
 261             start = text.index(self.TAG_OPENER)
 262             end = text.index(self.TAG_CLOSER, start + 1) + 1
 263             return text[start:end]
 264         except ValueError:
 265             return None
 266
 267     def _replace_tags(self, data, src_data=None):
 268         """Replace tag(s) in the data by their values.
 269
 270         :param data: The data where the tags will be replaced by their values.
 271         :param src_data: Data where the tags are defined. It is dictionary where
 272         the key is the tag and the value is the tag value. If not given, 'data'
 273         is used instead.
 274         :type data: str or dict
 275         :type src_data: dict
 276         :returns: Data with the tags replaced.
 277         :rtype: str or dict
 278         :raises: PresentationError if it is not possible to replace the tag or
 279         the data is not the supported data type (str, dict).
 280         """
 281
 282         if src_data is None:
 283             src_data = data
 284
 285         if isinstance(data, str):
 286             tag = self._find_tag(data)
 287             if tag is not None:
 288                 data = data.replace(tag, src_data[tag[1:-1]])
 289
 290         elif isinstance(data, dict):
 291             counter = 0
 292             for key, value in data.items():
 293                 tag = self._find_tag(value)
 294                 if tag is not None:
 295                     try:
 296                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 297                         counter += 1
 298                     except KeyError:
 299                         raise PresentationError("Not possible to replace the "
 300                                                 "tag '{}'".format(tag))
 301             if counter:
 302                 self._replace_tags(data, src_data)
 303         else:
 304             raise PresentationError("Replace tags: Not supported data type.")
 305
 306         return data
 307
 308     def _parse_env(self):
 309         """Parse environment specification in the specification YAML file.
 310         """
 311
 312         logging.info("Parsing specification file: environment ...")
 313
 314         idx = self._get_type_index("environment")
 315         if idx is None:
 316             return None
 317
 318         try:
 319             self._specification["environment"]["configuration"] = \
 320                 self._cfg_yaml[idx]["configuration"]
 321         except KeyError:
 322             self._specification["environment"]["configuration"] = None
 323
 324         try:
 325             self._specification["environment"]["paths"] = \
 326                 self._replace_tags(self._cfg_yaml[idx]["paths"])
 327         except KeyError:
 328             self._specification["environment"]["paths"] = None
 329
 330         try:
 331             self._specification["environment"]["urls"] = \
 332                 self._replace_tags(self._cfg_yaml[idx]["urls"])
 333         except KeyError:
 334             self._specification["environment"]["urls"] = None
 335
 336         try:
 337             self._specification["environment"]["make-dirs"] = \
 338                 self._cfg_yaml[idx]["make-dirs"]
 339         except KeyError:
 340             self._specification["environment"]["make-dirs"] = None
 341
 342         try:
 343             self._specification["environment"]["remove-dirs"] = \
 344                 self._cfg_yaml[idx]["remove-dirs"]
 345         except KeyError:
 346             self._specification["environment"]["remove-dirs"] = None
 347
 348         try:
 349             self._specification["environment"]["build-dirs"] = \
 350                 self._cfg_yaml[idx]["build-dirs"]
 351         except KeyError:
 352             self._specification["environment"]["build-dirs"] = None
 353
 354         logging.info("Done.")
 355
 356     def _parse_configuration(self):
 357         """Parse configuration of PAL in the specification YAML file.
 358         """
 359
 360         logging.info("Parsing specification file: configuration ...")
 361
 362         idx = self._get_type_index("configuration")
 363         if idx is None:
 364             logging.warning("No configuration information in the specification "
 365                             "file.")
 366             return None
 367
 368         try:
 369             self._specification["configuration"] = self._cfg_yaml[idx]
 370
 371         except KeyError:
 372             raise PresentationError("No configuration defined.")
 373
 374         # Data sets: Replace ranges by lists
 375         for set_name, data_set in self.configuration["data-sets"].items():
 376             for job, builds in data_set.items():
 377                 if builds:
 378                     if isinstance(builds, dict):
 379                         # defined as a range <start, end>
 380                         if builds.get("end", None) == "lastSuccessfulBuild":
 381                             # defined as a range <start, lastSuccessfulBuild>
 382                             ret_code, build_nr, _ = get_last_build_number(
 383                                 self.environment["urls"]["URL[JENKINS,CSIT]"],
 384                                 job)
 385                             if ret_code != 0:
 386                                 raise PresentationError(
 387                                     "Not possible to get the number of the "
 388                                     "last successful  build.")
 389                         else:
 390                             # defined as a range <start, end (build number)>
 391                             build_nr = builds.get("end", None)
 392                         builds = [x for x in range(1, int(build_nr)+1)]
 393                         self.configuration["data-sets"][set_name][job] = builds
 394
 395         logging.info("Done.")
 396
 397     def _parse_debug(self):
 398         """Parse debug specification in the specification YAML file.
 399         """
 400
 401         if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
 402             return None
 403
 404         logging.info("Parsing specification file: debug ...")
 405
 406         idx = self._get_type_index("debug")
 407         if idx is None:
 408             self.environment["configuration"]["CFG[DEBUG]"] = 0
 409             return None
 410
 411         try:
 412             for key, value in self._cfg_yaml[idx]["general"].items():
 413                 self._specification["debug"][key] = value
 414
 415             self._specification["input"]["builds"] = dict()
 416             for job, builds in self._cfg_yaml[idx]["builds"].items():
 417                 if builds:
 418                     self._specification["input"]["builds"][job] = list()
 419                     for build in builds:
 420                         self._specification["input"]["builds"][job].\
 421                             append({"build": build["build"],
 422                                     "status": "downloaded",
 423                                     "file-name": self._replace_tags(
 424                                         build["file"],
 425                                         self.environment["paths"])})
 426                 else:
 427                     logging.warning("No build is defined for the job '{}'. "
 428                                     "Trying to continue without it.".
 429                                     format(job))
 430
 431         except KeyError:
 432             raise PresentationError("No data to process.")
 433
 434     def _parse_input(self):
 435         """Parse input specification in the specification YAML file.
 436
 437         :raises: PresentationError if there are no data to process.
 438         """
 439
 440         logging.info("Parsing specification file: input ...")
 441
 442         idx = self._get_type_index("input")
 443         if idx is None:
 444             raise PresentationError("No data to process.")
 445
 446         try:
 447             for key, value in self._cfg_yaml[idx]["general"].items():
 448                 self._specification["input"][key] = value
 449             self._specification["input"]["builds"] = dict()
 450
 451             for job, builds in self._cfg_yaml[idx]["builds"].items():
 452                 if builds:
 453                     if isinstance(builds, dict):
 454                         # defined as a range <start, end>
 455                         if builds.get("end", None) == "lastSuccessfulBuild":
 456                             # defined as a range <start, lastSuccessfulBuild>
 457                             ret_code, build_nr, _ = get_last_build_number(
 458                                 self.environment["urls"]["URL[JENKINS,CSIT]"],
 459                                 job)
 460                             if ret_code != 0:
 461                                 raise PresentationError(
 462                                     "Not possible to get the number of the "
 463                                     "last successful  build.")
 464                         else:
 465                             # defined as a range <start, end (build number)>
 466                             build_nr = builds.get("end", None)
 467                         builds = [x for x in range(builds["start"],
 468                                                    int(build_nr) + 1)]
 469                     self._specification["input"]["builds"][job] = list()
 470                     for build in builds:
 471                         self._specification["input"]["builds"][job].\
 472                             append({"build": build, "status": None})
 473                 else:
 474                     logging.warning("No build is defined for the job '{}'. "
 475                                     "Trying to continue without it.".
 476                                     format(job))
 477         except KeyError:
 478             raise PresentationError("No data to process.")
 479
 480         logging.info("Done.")
 481
 482     def _parse_output(self):
 483         """Parse output specification in the specification YAML file.
 484
 485         :raises: PresentationError if there is no output defined.
 486         """
 487
 488         logging.info("Parsing specification file: output ...")
 489
 490         idx = self._get_type_index("output")
 491         if idx is None:
 492             raise PresentationError("No output defined.")
 493
 494         try:
 495             self._specification["output"] = self._cfg_yaml[idx]
 496         except (KeyError, IndexError):
 497             raise PresentationError("No output defined.")
 498
 499         logging.info("Done.")
 500
 501     def _parse_static(self):
 502         """Parse specification of the static content in the specification YAML
 503         file.
 504         """
 505
 506         logging.info("Parsing specification file: static content ...")
 507
 508         idx = self._get_type_index("static")
 509         if idx is None:
 510             logging.warning("No static content specified.")
 511
 512         for key, value in self._cfg_yaml[idx].items():
 513             if isinstance(value, str):
 514                 try:
 515                     self._cfg_yaml[idx][key] = self._replace_tags(
 516                         value, self._specification["environment"]["paths"])
 517                 except KeyError:
 518                     pass
 519
 520         self._specification["static"] = self._cfg_yaml[idx]
 521
 522         logging.info("Done.")
 523
 524     def _parse_elements(self):
 525         """Parse elements (tables, plots) specification in the specification
 526         YAML file.
 527         """
 528
 529         logging.info("Parsing specification file: elements ...")
 530
 531         count = 1
 532         for element in self._cfg_yaml:
 533             try:
 534                 element["output-file"] = self._replace_tags(
 535                     element["output-file"],
 536                     self._specification["environment"]["paths"])
 537             except KeyError:
 538                 pass
 539
 540             # add data sets to the elements:
 541             if isinstance(element.get("data", None), str):
 542                 data_set = element["data"]
 543                 try:
 544                     element["data"] = self.configuration["data-sets"][data_set]
 545                 except KeyError:
 546                     raise PresentationError("Data set {0} is not defined in "
 547                                             "the configuration section.".
 548                                             format(data_set))
 549
 550             if element["type"] == "table":
 551                 logging.info("  {:3d} Processing a table ...".format(count))
 552                 try:
 553                     element["template"] = self._replace_tags(
 554                         element["template"],
 555                         self._specification["environment"]["paths"])
 556                 except KeyError:
 557                     pass
 558                 self._specification["tables"].append(element)
 559                 count += 1
 560
 561             elif element["type"] == "plot":
 562                 logging.info("  {:3d} Processing a plot ...".format(count))
 563
 564                 # Add layout to the plots:
 565                 layout = element["layout"].get("layout", None)
 566                 if layout is not None:
 567                     element["layout"].pop("layout")
 568                     try:
 569                         for key, val in (self.configuration["plot-layouts"]
 570                                          [layout].items()):
 571                             element["layout"][key] = val
 572                     except KeyError:
 573                         raise PresentationError("Layout {0} is not defined in "
 574                                                 "the configuration section.".
 575                                                 format(layout))
 576                 self._specification["plots"].append(element)
 577                 count += 1
 578
 579             elif element["type"] == "file":
 580                 logging.info("  {:3d} Processing a file ...".format(count))
 581                 try:
 582                     element["dir-tables"] = self._replace_tags(
 583                         element["dir-tables"],
 584                         self._specification["environment"]["paths"])
 585                 except KeyError:
 586                     pass
 587                 self._specification["files"].append(element)
 588                 count += 1
 589
 590             elif element["type"] == "cpta":
 591                 logging.info("  {:3d} Processing Continuous Performance "
 592                              "Trending and Analysis ...".format(count))
 593
 594                 for plot in element["plots"]:
 595                     # Add layout to the plots:
 596                     layout = plot.get("layout", None)
 597                     if layout is not None:
 598                         try:
 599                             plot["layout"] = \
 600                                 self.configuration["plot-layouts"][layout]
 601                         except KeyError:
 602                             raise PresentationError(
 603                                 "Layout {0} is not defined in the "
 604                                 "configuration section.".format(layout))
 605                     # Add data sets:
 606                     if isinstance(plot.get("data", None), str):
 607                         data_set = plot["data"]
 608                         try:
 609                             plot["data"] = \
 610                                 self.configuration["data-sets"][data_set]
 611                         except KeyError:
 612                             raise PresentationError(
 613                                 "Data set {0} is not defined in "
 614                                 "the configuration section.".
 615                                 format(data_set))
 616                 self._specification["cpta"] = element
 617                 count += 1
 618
 619         logging.info("Done.")
 620
 621     def read_specification(self):
 622         """Parse specification in the specification YAML file.
 623
 624         :raises: PresentationError if an error occurred while parsing the
 625         specification file.
 626         """
 627         try:
 628             self._cfg_yaml = load(self._cfg_file)
 629         except YAMLError as err:
 630             raise PresentationError(msg="An error occurred while parsing the "
 631                                         "specification file.",
 632                                     details=str(err))
 633
 634         self._parse_env()
 635         self._parse_configuration()
 636         self._parse_debug()
 637         if not self.debug:
 638             self._parse_input()
 639         self._parse_output()
 640         self._parse_static()
 641         self._parse_elements()
 642
 643         logging.debug("Specification: \n{}".
 644                       format(pformat(self._specification)))