resources/tools/presentation/specification_parser.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 import logging
  21 from yaml import load, YAMLError
  22 from pprint import pformat
  23
  24 from errors import PresentationError
  25 from utils import get_last_successful_build_number
  26 from utils import get_last_completed_build_number
  27
  28
  29 class Specification(object):
  30     """Specification of Presentation and analytics layer.
  31
  32     - based on specification specified in the specification YAML file
  33     - presentation and analytics layer is model driven
  34     """
  35
  36     # Tags are used in specification YAML file and replaced while the file is
  37     # parsed.
  38     TAG_OPENER = "{"
  39     TAG_CLOSER = "}"
  40
  41     def __init__(self, cfg_file):
  42         """Initialization.
  43
  44         :param cfg_file: File handler for the specification YAML file.
  45         :type cfg_file: BinaryIO
  46         """
  47         self._cfg_file = cfg_file
  48         self._cfg_yaml = None
  49
  50         self._specification = {"environment": dict(),
  51                                "configuration": dict(),
  52                                "debug": dict(),
  53                                "static": dict(),
  54                                "input": dict(),
  55                                "output": dict(),
  56                                "tables": list(),
  57                                "plots": list(),
  58                                "files": list(),
  59                                "cpta": dict()}
  60
  61     @property
  62     def specification(self):
  63         """Getter - specification.
  64
  65         :returns: Specification.
  66         :rtype: dict
  67         """
  68         return self._specification
  69
  70     @property
  71     def environment(self):
  72         """Getter - environment.
  73
  74         :returns: Environment specification.
  75         :rtype: dict
  76         """
  77         return self._specification["environment"]
  78
  79     @property
  80     def configuration(self):
  81         """Getter - configuration.
  82
  83         :returns: Configuration of PAL.
  84         :rtype: dict
  85         """
  86         return self._specification["configuration"]
  87
  88     @property
  89     def static(self):
  90         """Getter - static content.
  91
  92         :returns: Static content specification.
  93         :rtype: dict
  94         """
  95         return self._specification["static"]
  96
  97     @property
  98     def debug(self):
  99         """Getter - debug
 100
 101         :returns: Debug specification
 102         :rtype: dict
 103         """
 104         return self._specification["debug"]
 105
 106     @property
 107     def is_debug(self):
 108         """Getter - debug mode
 109
 110         :returns: True if debug mode is on, otherwise False.
 111         :rtype: bool
 112         """
 113
 114         try:
 115             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
 116                 return True
 117             else:
 118                 return False
 119         except KeyError:
 120             return False
 121
 122     @property
 123     def input(self):
 124         """Getter - specification - inputs.
 125         - jobs and builds.
 126
 127         :returns: Inputs.
 128         :rtype: dict
 129         """
 130         return self._specification["input"]
 131
 132     @property
 133     def builds(self):
 134         """Getter - builds defined in specification.
 135
 136         :returns: Builds defined in the specification.
 137         :rtype: dict
 138         """
 139         return self.input["builds"]
 140
 141     @property
 142     def output(self):
 143         """Getter - specification - output formats and versions to be generated.
 144         - formats: html, pdf
 145         - versions: full, ...
 146
 147         :returns: Outputs to be generated.
 148         :rtype: dict
 149         """
 150         return self._specification["output"]
 151
 152     @property
 153     def tables(self):
 154         """Getter - tables to be generated.
 155
 156         :returns: List of specifications of tables to be generated.
 157         :rtype: list
 158         """
 159         return self._specification["tables"]
 160
 161     @property
 162     def plots(self):
 163         """Getter - plots to be generated.
 164
 165         :returns: List of specifications of plots to be generated.
 166         :rtype: list
 167         """
 168         return self._specification["plots"]
 169
 170     @property
 171     def files(self):
 172         """Getter - files to be generated.
 173
 174         :returns: List of specifications of files to be generated.
 175         :rtype: list
 176         """
 177         return self._specification["files"]
 178
 179     @property
 180     def cpta(self):
 181         """Getter - Continuous Performance Trending and Analysis to be
 182         generated.
 183
 184         :returns: List of specifications of Continuous Performance Trending and
 185         Analysis to be generated.
 186         :rtype: list
 187         """
 188         return self._specification["cpta"]
 189
 190     def set_input_state(self, job, build_nr, state):
 191         """Set the state of input
 192
 193         :param job:
 194         :param build_nr:
 195         :param state:
 196         :return:
 197         """
 198
 199         try:
 200             for build in self._specification["input"]["builds"][job]:
 201                 if build["build"] == build_nr:
 202                     build["status"] = state
 203                     break
 204             else:
 205                 raise PresentationError("Build '{}' is not defined for job '{}'"
 206                                         " in specification file.".
 207                                         format(build_nr, job))
 208         except KeyError:
 209             raise PresentationError("Job '{}' and build '{}' is not defined in "
 210                                     "specification file.".format(job, build_nr))
 211
 212     def set_input_file_name(self, job, build_nr, file_name):
 213         """Set the state of input
 214
 215         :param job:
 216         :param build_nr:
 217         :param file_name:
 218         :return:
 219         """
 220
 221         try:
 222             for build in self._specification["input"]["builds"][job]:
 223                 if build["build"] == build_nr:
 224                     build["file-name"] = file_name
 225                     break
 226             else:
 227                 raise PresentationError("Build '{}' is not defined for job '{}'"
 228                                         " in specification file.".
 229                                         format(build_nr, job))
 230         except KeyError:
 231             raise PresentationError("Job '{}' and build '{}' is not defined in "
 232                                     "specification file.".format(job, build_nr))
 233
 234     def _get_build_number(self, job, build_type):
 235         """Get the number of the job defined by its name:
 236          - lastSuccessfulBuild
 237          - lastCompletedBuild
 238
 239         :param job: Job name.
 240         :param build_type: Build type:
 241          - lastSuccessfulBuild
 242          - lastCompletedBuild
 243         :type job" str
 244         :raises PresentationError: If it is not possible to get the build
 245         number.
 246         :returns: The build number.
 247         :rtype: int
 248         """
 249
 250         # defined as a range <start, end>
 251         if build_type == "lastSuccessfulBuild":
 252             # defined as a range <start, lastSuccessfulBuild>
 253             ret_code, build_nr, _ = get_last_successful_build_number(
 254                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
 255         elif build_type == "lastCompletedBuild":
 256             # defined as a range <start, lastCompletedBuild>
 257             ret_code, build_nr, _ = get_last_completed_build_number(
 258                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
 259         else:
 260             raise PresentationError("Not supported build type: '{0}'".
 261                                     format(build_type))
 262         if ret_code != 0:
 263             raise PresentationError("Not possible to get the number of the "
 264                                     "build number.")
 265         try:
 266             build_nr = int(build_nr)
 267             return build_nr
 268         except ValueError as err:
 269             raise PresentationError("Not possible to get the number of the "
 270                                     "build number.\nReason: {0}".format(err))
 271
 272     def _get_type_index(self, item_type):
 273         """Get index of item type (environment, input, output, ...) in
 274         specification YAML file.
 275
 276         :param item_type: Item type: Top level items in specification YAML file,
 277         e.g.: environment, input, output.
 278         :type item_type: str
 279         :returns: Index of the given item type.
 280         :rtype: int
 281         """
 282
 283         index = 0
 284         for item in self._cfg_yaml:
 285             if item["type"] == item_type:
 286                 return index
 287             index += 1
 288         return None
 289
 290     def _find_tag(self, text):
 291         """Find the first tag in the given text. The tag is enclosed by the
 292         TAG_OPENER and TAG_CLOSER.
 293
 294         :param text: Text to be searched.
 295         :type text: str
 296         :returns: The tag, or None if not found.
 297         :rtype: str
 298         """
 299         try:
 300             start = text.index(self.TAG_OPENER)
 301             end = text.index(self.TAG_CLOSER, start + 1) + 1
 302             return text[start:end]
 303         except ValueError:
 304             return None
 305
 306     def _replace_tags(self, data, src_data=None):
 307         """Replace tag(s) in the data by their values.
 308
 309         :param data: The data where the tags will be replaced by their values.
 310         :param src_data: Data where the tags are defined. It is dictionary where
 311         the key is the tag and the value is the tag value. If not given, 'data'
 312         is used instead.
 313         :type data: str or dict
 314         :type src_data: dict
 315         :returns: Data with the tags replaced.
 316         :rtype: str or dict
 317         :raises: PresentationError if it is not possible to replace the tag or
 318         the data is not the supported data type (str, dict).
 319         """
 320
 321         if src_data is None:
 322             src_data = data
 323
 324         if isinstance(data, str):
 325             tag = self._find_tag(data)
 326             if tag is not None:
 327                 data = data.replace(tag, src_data[tag[1:-1]])
 328
 329         elif isinstance(data, dict):
 330             counter = 0
 331             for key, value in data.items():
 332                 tag = self._find_tag(value)
 333                 if tag is not None:
 334                     try:
 335                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 336                         counter += 1
 337                     except KeyError:
 338                         raise PresentationError("Not possible to replace the "
 339                                                 "tag '{}'".format(tag))
 340             if counter:
 341                 self._replace_tags(data, src_data)
 342         else:
 343             raise PresentationError("Replace tags: Not supported data type.")
 344
 345         return data
 346
 347     def _parse_env(self):
 348         """Parse environment specification in the specification YAML file.
 349         """
 350
 351         logging.info("Parsing specification file: environment ...")
 352
 353         idx = self._get_type_index("environment")
 354         if idx is None:
 355             return None
 356
 357         try:
 358             self._specification["environment"]["configuration"] = \
 359                 self._cfg_yaml[idx]["configuration"]
 360         except KeyError:
 361             self._specification["environment"]["configuration"] = None
 362
 363         try:
 364             self._specification["environment"]["paths"] = \
 365                 self._replace_tags(self._cfg_yaml[idx]["paths"])
 366         except KeyError:
 367             self._specification["environment"]["paths"] = None
 368
 369         try:
 370             self._specification["environment"]["urls"] = \
 371                 self._replace_tags(self._cfg_yaml[idx]["urls"])
 372         except KeyError:
 373             self._specification["environment"]["urls"] = None
 374
 375         try:
 376             self._specification["environment"]["make-dirs"] = \
 377                 self._cfg_yaml[idx]["make-dirs"]
 378         except KeyError:
 379             self._specification["environment"]["make-dirs"] = None
 380
 381         try:
 382             self._specification["environment"]["remove-dirs"] = \
 383                 self._cfg_yaml[idx]["remove-dirs"]
 384         except KeyError:
 385             self._specification["environment"]["remove-dirs"] = None
 386
 387         try:
 388             self._specification["environment"]["build-dirs"] = \
 389                 self._cfg_yaml[idx]["build-dirs"]
 390         except KeyError:
 391             self._specification["environment"]["build-dirs"] = None
 392
 393         logging.info("Done.")
 394
 395     def _parse_configuration(self):
 396         """Parse configuration of PAL in the specification YAML file.
 397         """
 398
 399         logging.info("Parsing specification file: configuration ...")
 400
 401         idx = self._get_type_index("configuration")
 402         if idx is None:
 403             logging.warning("No configuration information in the specification "
 404                             "file.")
 405             return None
 406
 407         try:
 408             self._specification["configuration"] = self._cfg_yaml[idx]
 409
 410         except KeyError:
 411             raise PresentationError("No configuration defined.")
 412
 413         # Data sets: Replace ranges by lists
 414         for set_name, data_set in self.configuration["data-sets"].items():
 415             for job, builds in data_set.items():
 416                 if builds:
 417                     if isinstance(builds, dict):
 418                         build_nr = builds.get("end", None)
 419                         try:
 420                             build_nr = int(build_nr)
 421                         except ValueError:
 422                             # defined as a range <start, build_type>
 423                             build_nr = self._get_build_number(job, build_nr)
 424                         builds = [x for x in range(builds["start"], build_nr+1)]
 425                         self.configuration["data-sets"][set_name][job] = builds
 426         logging.info("Done.")
 427
 428     def _parse_debug(self):
 429         """Parse debug specification in the specification YAML file.
 430         """
 431
 432         if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
 433             return None
 434
 435         logging.info("Parsing specification file: debug ...")
 436
 437         idx = self._get_type_index("debug")
 438         if idx is None:
 439             self.environment["configuration"]["CFG[DEBUG]"] = 0
 440             return None
 441
 442         try:
 443             for key, value in self._cfg_yaml[idx]["general"].items():
 444                 self._specification["debug"][key] = value
 445
 446             self._specification["input"]["builds"] = dict()
 447             for job, builds in self._cfg_yaml[idx]["builds"].items():
 448                 if builds:
 449                     self._specification["input"]["builds"][job] = list()
 450                     for build in builds:
 451                         self._specification["input"]["builds"][job].\
 452                             append({"build": build["build"],
 453                                     "status": "downloaded",
 454                                     "file-name": self._replace_tags(
 455                                         build["file"],
 456                                         self.environment["paths"])})
 457                 else:
 458                     logging.warning("No build is defined for the job '{}'. "
 459                                     "Trying to continue without it.".
 460                                     format(job))
 461
 462         except KeyError:
 463             raise PresentationError("No data to process.")
 464
 465     def _parse_input(self):
 466         """Parse input specification in the specification YAML file.
 467
 468         :raises: PresentationError if there are no data to process.
 469         """
 470
 471         logging.info("Parsing specification file: input ...")
 472
 473         idx = self._get_type_index("input")
 474         if idx is None:
 475             raise PresentationError("No data to process.")
 476
 477         try:
 478             for key, value in self._cfg_yaml[idx]["general"].items():
 479                 self._specification["input"][key] = value
 480             self._specification["input"]["builds"] = dict()
 481
 482             for job, builds in self._cfg_yaml[idx]["builds"].items():
 483                 if builds:
 484                     if isinstance(builds, dict):
 485                         build_nr = builds.get("end", None)
 486                         try:
 487                             build_nr = int(build_nr)
 488                         except ValueError:
 489                             # defined as a range <start, build_type>
 490                             build_nr = self._get_build_number(job, build_nr)
 491                         builds = [x for x in range(builds["start"], build_nr+1)]
 492                     self._specification["input"]["builds"][job] = list()
 493                     for build in builds:
 494                         self._specification["input"]["builds"][job]. \
 495                             append({"build": build, "status": None})
 496
 497                 else:
 498                     logging.warning("No build is defined for the job '{}'. "
 499                                     "Trying to continue without it.".
 500                                     format(job))
 501         except KeyError:
 502             raise PresentationError("No data to process.")
 503
 504         logging.info("Done.")
 505
 506     def _parse_output(self):
 507         """Parse output specification in the specification YAML file.
 508
 509         :raises: PresentationError if there is no output defined.
 510         """
 511
 512         logging.info("Parsing specification file: output ...")
 513
 514         idx = self._get_type_index("output")
 515         if idx is None:
 516             raise PresentationError("No output defined.")
 517
 518         try:
 519             self._specification["output"] = self._cfg_yaml[idx]
 520         except (KeyError, IndexError):
 521             raise PresentationError("No output defined.")
 522
 523         logging.info("Done.")
 524
 525     def _parse_static(self):
 526         """Parse specification of the static content in the specification YAML
 527         file.
 528         """
 529
 530         logging.info("Parsing specification file: static content ...")
 531
 532         idx = self._get_type_index("static")
 533         if idx is None:
 534             logging.warning("No static content specified.")
 535
 536         for key, value in self._cfg_yaml[idx].items():
 537             if isinstance(value, str):
 538                 try:
 539                     self._cfg_yaml[idx][key] = self._replace_tags(
 540                         value, self._specification["environment"]["paths"])
 541                 except KeyError:
 542                     pass
 543
 544         self._specification["static"] = self._cfg_yaml[idx]
 545
 546         logging.info("Done.")
 547
 548     def _parse_elements(self):
 549         """Parse elements (tables, plots) specification in the specification
 550         YAML file.
 551         """
 552
 553         logging.info("Parsing specification file: elements ...")
 554
 555         count = 1
 556         for element in self._cfg_yaml:
 557             try:
 558                 element["output-file"] = self._replace_tags(
 559                     element["output-file"],
 560                     self._specification["environment"]["paths"])
 561             except KeyError:
 562                 pass
 563
 564             # add data sets to the elements:
 565             if isinstance(element.get("data", None), str):
 566                 data_set = element["data"]
 567                 try:
 568                     element["data"] = self.configuration["data-sets"][data_set]
 569                 except KeyError:
 570                     raise PresentationError("Data set {0} is not defined in "
 571                                             "the configuration section.".
 572                                             format(data_set))
 573
 574             if element["type"] == "table":
 575                 logging.info("  {:3d} Processing a table ...".format(count))
 576                 try:
 577                     element["template"] = self._replace_tags(
 578                         element["template"],
 579                         self._specification["environment"]["paths"])
 580                 except KeyError:
 581                     pass
 582                 self._specification["tables"].append(element)
 583                 count += 1
 584
 585             elif element["type"] == "plot":
 586                 logging.info("  {:3d} Processing a plot ...".format(count))
 587
 588                 # Add layout to the plots:
 589                 layout = element["layout"].get("layout", None)
 590                 if layout is not None:
 591                     element["layout"].pop("layout")
 592                     try:
 593                         for key, val in (self.configuration["plot-layouts"]
 594                                          [layout].items()):
 595                             element["layout"][key] = val
 596                     except KeyError:
 597                         raise PresentationError("Layout {0} is not defined in "
 598                                                 "the configuration section.".
 599                                                 format(layout))
 600                 self._specification["plots"].append(element)
 601                 count += 1
 602
 603             elif element["type"] == "file":
 604                 logging.info("  {:3d} Processing a file ...".format(count))
 605                 try:
 606                     element["dir-tables"] = self._replace_tags(
 607                         element["dir-tables"],
 608                         self._specification["environment"]["paths"])
 609                 except KeyError:
 610                     pass
 611                 self._specification["files"].append(element)
 612                 count += 1
 613
 614             elif element["type"] == "cpta":
 615                 logging.info("  {:3d} Processing Continuous Performance "
 616                              "Trending and Analysis ...".format(count))
 617
 618                 for plot in element["plots"]:
 619                     # Add layout to the plots:
 620                     layout = plot.get("layout", None)
 621                     if layout is not None:
 622                         try:
 623                             plot["layout"] = \
 624                                 self.configuration["plot-layouts"][layout]
 625                         except KeyError:
 626                             raise PresentationError(
 627                                 "Layout {0} is not defined in the "
 628                                 "configuration section.".format(layout))
 629                     # Add data sets:
 630                     if isinstance(plot.get("data", None), str):
 631                         data_set = plot["data"]
 632                         try:
 633                             plot["data"] = \
 634                                 self.configuration["data-sets"][data_set]
 635                         except KeyError:
 636                             raise PresentationError(
 637                                 "Data set {0} is not defined in "
 638                                 "the configuration section.".
 639                                 format(data_set))
 640                 self._specification["cpta"] = element
 641                 count += 1
 642
 643         logging.info("Done.")
 644
 645     def read_specification(self):
 646         """Parse specification in the specification YAML file.
 647
 648         :raises: PresentationError if an error occurred while parsing the
 649         specification file.
 650         """
 651         try:
 652             self._cfg_yaml = load(self._cfg_file)
 653         except YAMLError as err:
 654             raise PresentationError(msg="An error occurred while parsing the "
 655                                         "specification file.",
 656                                     details=str(err))
 657
 658         self._parse_env()
 659         self._parse_configuration()
 660         self._parse_debug()
 661         if not self.debug:
 662             self._parse_input()
 663         self._parse_output()
 664         self._parse_static()
 665         self._parse_elements()
 666
 667         logging.debug("Specification: \n{}".
 668                       format(pformat(self._specification)))