resources/tools/presentation/new/specification_parser.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 import logging
  21 from yaml import load, YAMLError
  22 from pprint import pformat
  23
  24 from errors import PresentationError
  25 from utils import get_last_successful_build_number
  26 from utils import get_last_completed_build_number
  27
  28
  29 class Specification(object):
  30     """Specification of Presentation and analytics layer.
  31
  32     - based on specification specified in the specification YAML file
  33     - presentation and analytics layer is model driven
  34     """
  35
  36     # Tags are used in specification YAML file and replaced while the file is
  37     # parsed.
  38     TAG_OPENER = "{"
  39     TAG_CLOSER = "}"
  40
  41     def __init__(self, cfg_file):
  42         """Initialization.
  43
  44         :param cfg_file: File handler for the specification YAML file.
  45         :type cfg_file: BinaryIO
  46         """
  47         self._cfg_file = cfg_file
  48         self._cfg_yaml = None
  49
  50         self._specification = {"environment": dict(),
  51                                "configuration": dict(),
  52                                "static": dict(),
  53                                "input": dict(),
  54                                "output": dict(),
  55                                "tables": list(),
  56                                "plots": list(),
  57                                "files": list(),
  58                                "cpta": dict()}
  59
  60     @property
  61     def specification(self):
  62         """Getter - specification.
  63
  64         :returns: Specification.
  65         :rtype: dict
  66         """
  67         return self._specification
  68
  69     @property
  70     def environment(self):
  71         """Getter - environment.
  72
  73         :returns: Environment specification.
  74         :rtype: dict
  75         """
  76         return self._specification["environment"]
  77
  78     @property
  79     def configuration(self):
  80         """Getter - configuration.
  81
  82         :returns: Configuration of PAL.
  83         :rtype: dict
  84         """
  85         return self._specification["configuration"]
  86
  87     @property
  88     def static(self):
  89         """Getter - static content.
  90
  91         :returns: Static content specification.
  92         :rtype: dict
  93         """
  94         return self._specification["static"]
  95
  96     @property
  97     def is_debug(self):
  98         """Getter - debug mode
  99
 100         :returns: True if debug mode is on, otherwise False.
 101         :rtype: bool
 102         """
 103
 104         try:
 105             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
 106                 return True
 107             else:
 108                 return False
 109         except KeyError:
 110             return False
 111
 112     @property
 113     def input(self):
 114         """Getter - specification - inputs.
 115         - jobs and builds.
 116
 117         :returns: Inputs.
 118         :rtype: dict
 119         """
 120         return self._specification["input"]
 121
 122     @property
 123     def builds(self):
 124         """Getter - builds defined in specification.
 125
 126         :returns: Builds defined in the specification.
 127         :rtype: dict
 128         """
 129         return self.input["builds"]
 130
 131     @property
 132     def output(self):
 133         """Getter - specification - output formats and versions to be generated.
 134         - formats: html, pdf
 135         - versions: full, ...
 136
 137         :returns: Outputs to be generated.
 138         :rtype: dict
 139         """
 140         return self._specification["output"]
 141
 142     @property
 143     def tables(self):
 144         """Getter - tables to be generated.
 145
 146         :returns: List of specifications of tables to be generated.
 147         :rtype: list
 148         """
 149         return self._specification["tables"]
 150
 151     @property
 152     def plots(self):
 153         """Getter - plots to be generated.
 154
 155         :returns: List of specifications of plots to be generated.
 156         :rtype: list
 157         """
 158         return self._specification["plots"]
 159
 160     @property
 161     def files(self):
 162         """Getter - files to be generated.
 163
 164         :returns: List of specifications of files to be generated.
 165         :rtype: list
 166         """
 167         return self._specification["files"]
 168
 169     @property
 170     def cpta(self):
 171         """Getter - Continuous Performance Trending and Analysis to be
 172         generated.
 173
 174         :returns: List of specifications of Continuous Performance Trending and
 175         Analysis to be generated.
 176         :rtype: list
 177         """
 178         return self._specification["cpta"]
 179
 180     def set_input_state(self, job, build_nr, state):
 181         """Set the state of input
 182
 183         :param job:
 184         :param build_nr:
 185         :param state:
 186         :return:
 187         """
 188
 189         try:
 190             for build in self._specification["input"]["builds"][job]:
 191                 if build["build"] == build_nr:
 192                     build["status"] = state
 193                     break
 194             else:
 195                 raise PresentationError("Build '{}' is not defined for job '{}'"
 196                                         " in specification file.".
 197                                         format(build_nr, job))
 198         except KeyError:
 199             raise PresentationError("Job '{}' and build '{}' is not defined in "
 200                                     "specification file.".format(job, build_nr))
 201
 202     def set_input_file_name(self, job, build_nr, file_name):
 203         """Set the state of input
 204
 205         :param job:
 206         :param build_nr:
 207         :param file_name:
 208         :return:
 209         """
 210
 211         try:
 212             for build in self._specification["input"]["builds"][job]:
 213                 if build["build"] == build_nr:
 214                     build["file-name"] = file_name
 215                     break
 216             else:
 217                 raise PresentationError("Build '{}' is not defined for job '{}'"
 218                                         " in specification file.".
 219                                         format(build_nr, job))
 220         except KeyError:
 221             raise PresentationError("Job '{}' and build '{}' is not defined in "
 222                                     "specification file.".format(job, build_nr))
 223
 224     def _get_build_number(self, job, build_type):
 225         """Get the number of the job defined by its name:
 226          - lastSuccessfulBuild
 227          - lastCompletedBuild
 228
 229         :param job: Job name.
 230         :param build_type: Build type:
 231          - lastSuccessfulBuild
 232          - lastCompletedBuild
 233         :type job" str
 234         :raises PresentationError: If it is not possible to get the build
 235         number.
 236         :returns: The build number.
 237         :rtype: int
 238         """
 239
 240         # defined as a range <start, end>
 241         if build_type == "lastSuccessfulBuild":
 242             # defined as a range <start, lastSuccessfulBuild>
 243             ret_code, build_nr, _ = get_last_successful_build_number(
 244                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
 245         elif build_type == "lastCompletedBuild":
 246             # defined as a range <start, lastCompletedBuild>
 247             ret_code, build_nr, _ = get_last_completed_build_number(
 248                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
 249         else:
 250             raise PresentationError("Not supported build type: '{0}'".
 251                                     format(build_type))
 252         if ret_code != 0:
 253             raise PresentationError("Not possible to get the number of the "
 254                                     "build number.")
 255         try:
 256             build_nr = int(build_nr)
 257             return build_nr
 258         except ValueError as err:
 259             raise PresentationError("Not possible to get the number of the "
 260                                     "build number.\nReason: {0}".format(err))
 261
 262     def _get_type_index(self, item_type):
 263         """Get index of item type (environment, input, output, ...) in
 264         specification YAML file.
 265
 266         :param item_type: Item type: Top level items in specification YAML file,
 267         e.g.: environment, input, output.
 268         :type item_type: str
 269         :returns: Index of the given item type.
 270         :rtype: int
 271         """
 272
 273         index = 0
 274         for item in self._cfg_yaml:
 275             if item["type"] == item_type:
 276                 return index
 277             index += 1
 278         return None
 279
 280     def _find_tag(self, text):
 281         """Find the first tag in the given text. The tag is enclosed by the
 282         TAG_OPENER and TAG_CLOSER.
 283
 284         :param text: Text to be searched.
 285         :type text: str
 286         :returns: The tag, or None if not found.
 287         :rtype: str
 288         """
 289         try:
 290             start = text.index(self.TAG_OPENER)
 291             end = text.index(self.TAG_CLOSER, start + 1) + 1
 292             return text[start:end]
 293         except ValueError:
 294             return None
 295
 296     def _replace_tags(self, data, src_data=None):
 297         """Replace tag(s) in the data by their values.
 298
 299         :param data: The data where the tags will be replaced by their values.
 300         :param src_data: Data where the tags are defined. It is dictionary where
 301         the key is the tag and the value is the tag value. If not given, 'data'
 302         is used instead.
 303         :type data: str or dict
 304         :type src_data: dict
 305         :returns: Data with the tags replaced.
 306         :rtype: str or dict
 307         :raises: PresentationError if it is not possible to replace the tag or
 308         the data is not the supported data type (str, dict).
 309         """
 310
 311         if src_data is None:
 312             src_data = data
 313
 314         if isinstance(data, str):
 315             tag = self._find_tag(data)
 316             if tag is not None:
 317                 data = data.replace(tag, src_data[tag[1:-1]])
 318
 319         elif isinstance(data, dict):
 320             counter = 0
 321             for key, value in data.items():
 322                 tag = self._find_tag(value)
 323                 if tag is not None:
 324                     try:
 325                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 326                         counter += 1
 327                     except KeyError:
 328                         raise PresentationError("Not possible to replace the "
 329                                                 "tag '{}'".format(tag))
 330             if counter:
 331                 self._replace_tags(data, src_data)
 332         else:
 333             raise PresentationError("Replace tags: Not supported data type.")
 334
 335         return data
 336
 337     def _parse_env(self):
 338         """Parse environment specification in the specification YAML file.
 339         """
 340
 341         logging.info("Parsing specification file: environment ...")
 342
 343         idx = self._get_type_index("environment")
 344         if idx is None:
 345             return None
 346
 347         try:
 348             self._specification["environment"]["configuration"] = \
 349                 self._cfg_yaml[idx]["configuration"]
 350         except KeyError:
 351             self._specification["environment"]["configuration"] = None
 352
 353         try:
 354             self._specification["environment"]["paths"] = \
 355                 self._replace_tags(self._cfg_yaml[idx]["paths"])
 356         except KeyError:
 357             self._specification["environment"]["paths"] = None
 358
 359         try:
 360             self._specification["environment"]["urls"] = \
 361                 self._replace_tags(self._cfg_yaml[idx]["urls"])
 362         except KeyError:
 363             self._specification["environment"]["urls"] = None
 364
 365         try:
 366             self._specification["environment"]["make-dirs"] = \
 367                 self._cfg_yaml[idx]["make-dirs"]
 368         except KeyError:
 369             self._specification["environment"]["make-dirs"] = None
 370
 371         try:
 372             self._specification["environment"]["remove-dirs"] = \
 373                 self._cfg_yaml[idx]["remove-dirs"]
 374         except KeyError:
 375             self._specification["environment"]["remove-dirs"] = None
 376
 377         try:
 378             self._specification["environment"]["build-dirs"] = \
 379                 self._cfg_yaml[idx]["build-dirs"]
 380         except KeyError:
 381             self._specification["environment"]["build-dirs"] = None
 382
 383         logging.info("Done.")
 384
 385     def _parse_configuration(self):
 386         """Parse configuration of PAL in the specification YAML file.
 387         """
 388
 389         logging.info("Parsing specification file: configuration ...")
 390
 391         idx = self._get_type_index("configuration")
 392         if idx is None:
 393             logging.warning("No configuration information in the specification "
 394                             "file.")
 395             return None
 396
 397         try:
 398             self._specification["configuration"] = self._cfg_yaml[idx]
 399
 400         except KeyError:
 401             raise PresentationError("No configuration defined.")
 402
 403         # Data sets: Replace ranges by lists
 404         for set_name, data_set in self.configuration["data-sets"].items():
 405             for job, builds in data_set.items():
 406                 if builds:
 407                     if isinstance(builds, dict):
 408                         build_nr = builds.get("end", None)
 409                         try:
 410                             build_nr = int(build_nr)
 411                         except ValueError:
 412                             # defined as a range <start, build_type>
 413                             build_nr = self._get_build_number(job, build_nr)
 414                         builds = [x for x in range(builds["start"], build_nr+1)]
 415                         self.configuration["data-sets"][set_name][job] = builds
 416         logging.info("Done.")
 417
 418     def _parse_input(self):
 419         """Parse input specification in the specification YAML file.
 420
 421         :raises: PresentationError if there are no data to process.
 422         """
 423
 424         logging.info("Parsing specification file: input ...")
 425
 426         idx = self._get_type_index("input")
 427         if idx is None:
 428             raise PresentationError("No data to process.")
 429
 430         try:
 431             for key, value in self._cfg_yaml[idx]["general"].items():
 432                 self._specification["input"][key] = value
 433             self._specification["input"]["builds"] = dict()
 434
 435             for job, builds in self._cfg_yaml[idx]["builds"].items():
 436                 if builds:
 437                     if isinstance(builds, dict):
 438                         build_nr = builds.get("end", None)
 439                         try:
 440                             build_nr = int(build_nr)
 441                         except ValueError:
 442                             # defined as a range <start, build_type>
 443                             build_nr = self._get_build_number(job, build_nr)
 444                         builds = [x for x in range(builds["start"], build_nr+1)]
 445                     self._specification["input"]["builds"][job] = list()
 446                     for build in builds:
 447                         self._specification["input"]["builds"][job]. \
 448                             append({"build": build, "status": None})
 449
 450                 else:
 451                     logging.warning("No build is defined for the job '{}'. "
 452                                     "Trying to continue without it.".
 453                                     format(job))
 454         except KeyError:
 455             raise PresentationError("No data to process.")
 456
 457         logging.info("Done.")
 458
 459     def _parse_output(self):
 460         """Parse output specification in the specification YAML file.
 461
 462         :raises: PresentationError if there is no output defined.
 463         """
 464
 465         logging.info("Parsing specification file: output ...")
 466
 467         idx = self._get_type_index("output")
 468         if idx is None:
 469             raise PresentationError("No output defined.")
 470
 471         try:
 472             self._specification["output"] = self._cfg_yaml[idx]
 473         except (KeyError, IndexError):
 474             raise PresentationError("No output defined.")
 475
 476         logging.info("Done.")
 477
 478     def _parse_static(self):
 479         """Parse specification of the static content in the specification YAML
 480         file.
 481         """
 482
 483         logging.info("Parsing specification file: static content ...")
 484
 485         idx = self._get_type_index("static")
 486         if idx is None:
 487             logging.warning("No static content specified.")
 488
 489         for key, value in self._cfg_yaml[idx].items():
 490             if isinstance(value, str):
 491                 try:
 492                     self._cfg_yaml[idx][key] = self._replace_tags(
 493                         value, self._specification["environment"]["paths"])
 494                 except KeyError:
 495                     pass
 496
 497         self._specification["static"] = self._cfg_yaml[idx]
 498
 499         logging.info("Done.")
 500
 501     def _parse_elements(self):
 502         """Parse elements (tables, plots) specification in the specification
 503         YAML file.
 504         """
 505
 506         logging.info("Parsing specification file: elements ...")
 507
 508         count = 1
 509         for element in self._cfg_yaml:
 510             try:
 511                 element["output-file"] = self._replace_tags(
 512                     element["output-file"],
 513                     self._specification["environment"]["paths"])
 514             except KeyError:
 515                 pass
 516
 517             try:
 518                 element["input-file"] = self._replace_tags(
 519                     element["input-file"],
 520                     self._specification["environment"]["paths"])
 521             except KeyError:
 522                 pass
 523
 524             # add data sets to the elements:
 525             if isinstance(element.get("data", None), str):
 526                 data_set = element["data"]
 527                 try:
 528                     element["data"] = self.configuration["data-sets"][data_set]
 529                 except KeyError:
 530                     raise PresentationError("Data set {0} is not defined in "
 531                                             "the configuration section.".
 532                                             format(data_set))
 533
 534             if element["type"] == "table":
 535                 logging.info("  {:3d} Processing a table ...".format(count))
 536                 try:
 537                     element["template"] = self._replace_tags(
 538                         element["template"],
 539                         self._specification["environment"]["paths"])
 540                 except KeyError:
 541                     pass
 542                 self._specification["tables"].append(element)
 543                 count += 1
 544
 545             elif element["type"] == "plot":
 546                 logging.info("  {:3d} Processing a plot ...".format(count))
 547
 548                 # Add layout to the plots:
 549                 layout = element["layout"].get("layout", None)
 550                 if layout is not None:
 551                     element["layout"].pop("layout")
 552                     try:
 553                         for key, val in (self.configuration["plot-layouts"]
 554                                          [layout].items()):
 555                             element["layout"][key] = val
 556                     except KeyError:
 557                         raise PresentationError("Layout {0} is not defined in "
 558                                                 "the configuration section.".
 559                                                 format(layout))
 560                 self._specification["plots"].append(element)
 561                 count += 1
 562
 563             elif element["type"] == "file":
 564                 logging.info("  {:3d} Processing a file ...".format(count))
 565                 try:
 566                     element["dir-tables"] = self._replace_tags(
 567                         element["dir-tables"],
 568                         self._specification["environment"]["paths"])
 569                 except KeyError:
 570                     pass
 571                 self._specification["files"].append(element)
 572                 count += 1
 573
 574             elif element["type"] == "cpta":
 575                 logging.info("  {:3d} Processing Continuous Performance "
 576                              "Trending and Analysis ...".format(count))
 577
 578                 for plot in element["plots"]:
 579                     # Add layout to the plots:
 580                     layout = plot.get("layout", None)
 581                     if layout is not None:
 582                         try:
 583                             plot["layout"] = \
 584                                 self.configuration["plot-layouts"][layout]
 585                         except KeyError:
 586                             raise PresentationError(
 587                                 "Layout {0} is not defined in the "
 588                                 "configuration section.".format(layout))
 589                     # Add data sets:
 590                     if isinstance(plot.get("data", None), str):
 591                         data_set = plot["data"]
 592                         try:
 593                             plot["data"] = \
 594                                 self.configuration["data-sets"][data_set]
 595                         except KeyError:
 596                             raise PresentationError(
 597                                 "Data set {0} is not defined in "
 598                                 "the configuration section.".
 599                                 format(data_set))
 600                 self._specification["cpta"] = element
 601                 count += 1
 602
 603         logging.info("Done.")
 604
 605     def read_specification(self):
 606         """Parse specification in the specification YAML file.
 607
 608         :raises: PresentationError if an error occurred while parsing the
 609         specification file.
 610         """
 611         try:
 612             self._cfg_yaml = load(self._cfg_file)
 613         except YAMLError as err:
 614             raise PresentationError(msg="An error occurred while parsing the "
 615                                         "specification file.",
 616                                     details=str(err))
 617
 618         self._parse_env()
 619         self._parse_configuration()
 620         self._parse_input()
 621         self._parse_output()
 622         self._parse_static()
 623         self._parse_elements()
 624
 625         logging.debug("Specification: \n{}".
 626                       format(pformat(self._specification)))