resources/tools/presentation_new/specification_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 import logging
  21 from yaml import load, YAMLError
  22 from pprint import pformat
  23
  24 from errors import PresentationError
  25 from utils import get_last_successful_build_number
  26 from utils import get_last_completed_build_number
  27
  28
  29 class Specification(object):
  30     """Specification of Presentation and analytics layer.
  31
  32     - based on specification specified in the specification YAML file
  33     - presentation and analytics layer is model driven
  34     """
  35
  36     # Tags are used in specification YAML file and replaced while the file is
  37     # parsed.
  38     TAG_OPENER = "{"
  39     TAG_CLOSER = "}"
  40
  41     def __init__(self, cfg_file):
  42         """Initialization.
  43
  44         :param cfg_file: File handler for the specification YAML file.
  45         :type cfg_file: BinaryIO
  46         """
  47         self._cfg_file = cfg_file
  48         self._cfg_yaml = None
  49
  50         self._specification = {"environment": dict(),
  51                                "configuration": dict(),
  52                                "static": dict(),
  53                                "input": dict(),
  54                                "output": dict(),
  55                                "tables": list(),
  56                                "plots": list(),
  57                                "files": list(),
  58                                "cpta": dict()}
  59
  60     @property
  61     def specification(self):
  62         """Getter - specification.
  63
  64         :returns: Specification.
  65         :rtype: dict
  66         """
  67         return self._specification
  68
  69     @property
  70     def environment(self):
  71         """Getter - environment.
  72
  73         :returns: Environment specification.
  74         :rtype: dict
  75         """
  76         return self._specification["environment"]
  77
  78     @property
  79     def configuration(self):
  80         """Getter - configuration.
  81
  82         :returns: Configuration of PAL.
  83         :rtype: dict
  84         """
  85         return self._specification["configuration"]
  86
  87     @property
  88     def static(self):
  89         """Getter - static content.
  90
  91         :returns: Static content specification.
  92         :rtype: dict
  93         """
  94         return self._specification["static"]
  95
  96     @property
  97     def mapping(self):
  98         """Getter - Mapping.
  99
 100         :returns: Mapping of the old names of test cases to the new (actual)
 101             one.
 102         :rtype: dict
 103         """
 104         return self._specification["configuration"]["mapping"]
 105
 106     @property
 107     def ignore(self):
 108         """Getter - Ignore list.
 109
 110         :returns: List of ignored test cases.
 111         :rtype: list
 112         """
 113         return self._specification["configuration"]["ignore"]
 114
 115     @property
 116     def alerting(self):
 117         """Getter - Alerting.
 118
 119         :returns: Specification of alerts.
 120         :rtype: dict
 121         """
 122         return self._specification["configuration"]["alerting"]
 123
 124     @property
 125     def input(self):
 126         """Getter - specification - inputs.
 127         - jobs and builds.
 128
 129         :returns: Inputs.
 130         :rtype: dict
 131         """
 132         return self._specification["input"]
 133
 134     @property
 135     def builds(self):
 136         """Getter - builds defined in specification.
 137
 138         :returns: Builds defined in the specification.
 139         :rtype: dict
 140         """
 141         return self.input["builds"]
 142
 143     @property
 144     def output(self):
 145         """Getter - specification - output formats and versions to be generated.
 146         - formats: html, pdf
 147         - versions: full, ...
 148
 149         :returns: Outputs to be generated.
 150         :rtype: dict
 151         """
 152         return self._specification["output"]
 153
 154     @property
 155     def tables(self):
 156         """Getter - tables to be generated.
 157
 158         :returns: List of specifications of tables to be generated.
 159         :rtype: list
 160         """
 161         return self._specification["tables"]
 162
 163     @property
 164     def plots(self):
 165         """Getter - plots to be generated.
 166
 167         :returns: List of specifications of plots to be generated.
 168         :rtype: list
 169         """
 170         return self._specification["plots"]
 171
 172     @property
 173     def files(self):
 174         """Getter - files to be generated.
 175
 176         :returns: List of specifications of files to be generated.
 177         :rtype: list
 178         """
 179         return self._specification["files"]
 180
 181     @property
 182     def cpta(self):
 183         """Getter - Continuous Performance Trending and Analysis to be
 184         generated.
 185
 186         :returns: List of specifications of Continuous Performance Trending and
 187         Analysis to be generated.
 188         :rtype: list
 189         """
 190         return self._specification["cpta"]
 191
 192     def set_input_state(self, job, build_nr, state):
 193         """Set the state of input
 194
 195         :param job:
 196         :param build_nr:
 197         :param state:
 198         :return:
 199         """
 200
 201         try:
 202             for build in self._specification["input"]["builds"][job]:
 203                 if build["build"] == build_nr:
 204                     build["status"] = state
 205                     break
 206             else:
 207                 raise PresentationError("Build '{}' is not defined for job '{}'"
 208                                         " in specification file.".
 209                                         format(build_nr, job))
 210         except KeyError:
 211             raise PresentationError("Job '{}' and build '{}' is not defined in "
 212                                     "specification file.".format(job, build_nr))
 213
 214     def set_input_file_name(self, job, build_nr, file_name):
 215         """Set the state of input
 216
 217         :param job:
 218         :param build_nr:
 219         :param file_name:
 220         :return:
 221         """
 222
 223         try:
 224             for build in self._specification["input"]["builds"][job]:
 225                 if build["build"] == build_nr:
 226                     build["file-name"] = file_name
 227                     break
 228             else:
 229                 raise PresentationError("Build '{}' is not defined for job '{}'"
 230                                         " in specification file.".
 231                                         format(build_nr, job))
 232         except KeyError:
 233             raise PresentationError("Job '{}' and build '{}' is not defined in "
 234                                     "specification file.".format(job, build_nr))
 235
 236     def _get_build_number(self, job, build_type):
 237         """Get the number of the job defined by its name:
 238          - lastSuccessfulBuild
 239          - lastCompletedBuild
 240
 241         :param job: Job name.
 242         :param build_type: Build type:
 243          - lastSuccessfulBuild
 244          - lastCompletedBuild
 245         :type job" str
 246         :raises PresentationError: If it is not possible to get the build
 247         number.
 248         :returns: The build number.
 249         :rtype: int
 250         """
 251
 252         # defined as a range <start, end>
 253         if build_type == "lastSuccessfulBuild":
 254             # defined as a range <start, lastSuccessfulBuild>
 255             ret_code, build_nr, _ = get_last_successful_build_number(
 256                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
 257         elif build_type == "lastCompletedBuild":
 258             # defined as a range <start, lastCompletedBuild>
 259             ret_code, build_nr, _ = get_last_completed_build_number(
 260                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
 261         else:
 262             raise PresentationError("Not supported build type: '{0}'".
 263                                     format(build_type))
 264         if ret_code != 0:
 265             raise PresentationError("Not possible to get the number of the "
 266                                     "build number.")
 267         try:
 268             build_nr = int(build_nr)
 269             return build_nr
 270         except ValueError as err:
 271             raise PresentationError("Not possible to get the number of the "
 272                                     "build number.\nReason: {0}".format(err))
 273
 274     def _get_type_index(self, item_type):
 275         """Get index of item type (environment, input, output, ...) in
 276         specification YAML file.
 277
 278         :param item_type: Item type: Top level items in specification YAML file,
 279         e.g.: environment, input, output.
 280         :type item_type: str
 281         :returns: Index of the given item type.
 282         :rtype: int
 283         """
 284
 285         index = 0
 286         for item in self._cfg_yaml:
 287             if item["type"] == item_type:
 288                 return index
 289             index += 1
 290         return None
 291
 292     def _find_tag(self, text):
 293         """Find the first tag in the given text. The tag is enclosed by the
 294         TAG_OPENER and TAG_CLOSER.
 295
 296         :param text: Text to be searched.
 297         :type text: str
 298         :returns: The tag, or None if not found.
 299         :rtype: str
 300         """
 301         try:
 302             start = text.index(self.TAG_OPENER)
 303             end = text.index(self.TAG_CLOSER, start + 1) + 1
 304             return text[start:end]
 305         except ValueError:
 306             return None
 307
 308     def _replace_tags(self, data, src_data=None):
 309         """Replace tag(s) in the data by their values.
 310
 311         :param data: The data where the tags will be replaced by their values.
 312         :param src_data: Data where the tags are defined. It is dictionary where
 313         the key is the tag and the value is the tag value. If not given, 'data'
 314         is used instead.
 315         :type data: str or dict
 316         :type src_data: dict
 317         :returns: Data with the tags replaced.
 318         :rtype: str or dict
 319         :raises: PresentationError if it is not possible to replace the tag or
 320         the data is not the supported data type (str, dict).
 321         """
 322
 323         if src_data is None:
 324             src_data = data
 325
 326         if isinstance(data, str):
 327             tag = self._find_tag(data)
 328             if tag is not None:
 329                 data = data.replace(tag, src_data[tag[1:-1]])
 330
 331         elif isinstance(data, dict):
 332             counter = 0
 333             for key, value in data.items():
 334                 tag = self._find_tag(value)
 335                 if tag is not None:
 336                     try:
 337                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 338                         counter += 1
 339                     except KeyError:
 340                         raise PresentationError("Not possible to replace the "
 341                                                 "tag '{}'".format(tag))
 342             if counter:
 343                 self._replace_tags(data, src_data)
 344         else:
 345             raise PresentationError("Replace tags: Not supported data type.")
 346
 347         return data
 348
 349     def _parse_env(self):
 350         """Parse environment specification in the specification YAML file.
 351         """
 352
 353         logging.info("Parsing specification file: environment ...")
 354
 355         idx = self._get_type_index("environment")
 356         if idx is None:
 357             return None
 358
 359         try:
 360             self._specification["environment"]["configuration"] = \
 361                 self._cfg_yaml[idx]["configuration"]
 362         except KeyError:
 363             self._specification["environment"]["configuration"] = None
 364
 365         try:
 366             self._specification["environment"]["paths"] = \
 367                 self._replace_tags(self._cfg_yaml[idx]["paths"])
 368         except KeyError:
 369             self._specification["environment"]["paths"] = None
 370
 371         try:
 372             self._specification["environment"]["urls"] = \
 373                 self._replace_tags(self._cfg_yaml[idx]["urls"])
 374         except KeyError:
 375             self._specification["environment"]["urls"] = None
 376
 377         try:
 378             self._specification["environment"]["make-dirs"] = \
 379                 self._cfg_yaml[idx]["make-dirs"]
 380         except KeyError:
 381             self._specification["environment"]["make-dirs"] = None
 382
 383         try:
 384             self._specification["environment"]["remove-dirs"] = \
 385                 self._cfg_yaml[idx]["remove-dirs"]
 386         except KeyError:
 387             self._specification["environment"]["remove-dirs"] = None
 388
 389         try:
 390             self._specification["environment"]["build-dirs"] = \
 391                 self._cfg_yaml[idx]["build-dirs"]
 392         except KeyError:
 393             self._specification["environment"]["build-dirs"] = None
 394
 395         try:
 396             self._specification["environment"]["testbeds"] = \
 397                 self._cfg_yaml[idx]["testbeds"]
 398         except KeyError:
 399             self._specification["environment"]["testbeds"] = None
 400
 401         logging.info("Done.")
 402
 403     def _parse_configuration(self):
 404         """Parse configuration of PAL in the specification YAML file.
 405         """
 406
 407         logging.info("Parsing specification file: configuration ...")
 408
 409         idx = self._get_type_index("configuration")
 410         if idx is None:
 411             logging.warning("No configuration information in the specification "
 412                             "file.")
 413             return None
 414
 415         try:
 416             self._specification["configuration"] = self._cfg_yaml[idx]
 417
 418         except KeyError:
 419             raise PresentationError("No configuration defined.")
 420
 421         # Data sets: Replace ranges by lists
 422         for set_name, data_set in self.configuration["data-sets"].items():
 423             for job, builds in data_set.items():
 424                 if builds:
 425                     if isinstance(builds, dict):
 426                         build_nr = builds.get("end", None)
 427                         try:
 428                             build_nr = int(build_nr)
 429                         except ValueError:
 430                             # defined as a range <start, build_type>
 431                             build_nr = self._get_build_number(job, build_nr)
 432                         builds = [x for x in range(builds["start"], build_nr+1)]
 433                         self.configuration["data-sets"][set_name][job] = builds
 434
 435         # Mapping table:
 436         mapping = None
 437         mapping_file_name = self._specification["configuration"].\
 438             get("mapping-file", None)
 439         if mapping_file_name:
 440             logging.debug("Mapping file: '{0}'".format(mapping_file_name))
 441             try:
 442                 with open(mapping_file_name, 'r') as mfile:
 443                     mapping = load(mfile)
 444                 logging.debug("Loaded mapping table:\n{0}".format(mapping))
 445             except (YAMLError, IOError) as err:
 446                 raise PresentationError(
 447                     msg="An error occurred while parsing the mapping file "
 448                         "'{0}'.".format(mapping_file_name),
 449                     details=repr(err))
 450         # Make sure everything is lowercase
 451         if mapping:
 452             self._specification["configuration"]["mapping"] = \
 453                 {key.lower(): val.lower() for key, val in mapping.iteritems()}
 454         else:
 455             self._specification["configuration"]["mapping"] = dict()
 456
 457         # Ignore list:
 458         ignore = None
 459         ignore_list_name = self._specification["configuration"].\
 460             get("ignore-list", None)
 461         if ignore_list_name:
 462             logging.debug("Ignore list file: '{0}'".format(ignore_list_name))
 463             try:
 464                 with open(ignore_list_name, 'r') as ifile:
 465                     ignore = load(ifile)
 466                 logging.debug("Loaded ignore list:\n{0}".format(ignore))
 467             except (YAMLError, IOError) as err:
 468                 raise PresentationError(
 469                     msg="An error occurred while parsing the ignore list file "
 470                         "'{0}'.".format(ignore_list_name),
 471                     details=repr(err))
 472         # Make sure everything is lowercase
 473         if ignore:
 474             self._specification["configuration"]["ignore"] = \
 475                 [item.lower() for item in ignore]
 476         else:
 477             self._specification["configuration"]["ignore"] = list()
 478
 479         logging.info("Done.")
 480
 481     def _parse_input(self):
 482         """Parse input specification in the specification YAML file.
 483
 484         :raises: PresentationError if there are no data to process.
 485         """
 486
 487         logging.info("Parsing specification file: input ...")
 488
 489         idx = self._get_type_index("input")
 490         if idx is None:
 491             raise PresentationError("No data to process.")
 492
 493         try:
 494             for key, value in self._cfg_yaml[idx]["general"].items():
 495                 self._specification["input"][key] = value
 496             self._specification["input"]["builds"] = dict()
 497
 498             for job, builds in self._cfg_yaml[idx]["builds"].items():
 499                 if builds:
 500                     if isinstance(builds, dict):
 501                         build_nr = builds.get("end", None)
 502                         try:
 503                             build_nr = int(build_nr)
 504                         except ValueError:
 505                             # defined as a range <start, build_type>
 506                             build_nr = self._get_build_number(job, build_nr)
 507                         builds = [x for x in range(builds["start"], build_nr+1)]
 508                     self._specification["input"]["builds"][job] = list()
 509                     for build in builds:
 510                         self._specification["input"]["builds"][job]. \
 511                             append({"build": build, "status": None})
 512
 513                 else:
 514                     logging.warning("No build is defined for the job '{}'. "
 515                                     "Trying to continue without it.".
 516                                     format(job))
 517         except KeyError:
 518             raise PresentationError("No data to process.")
 519
 520         logging.info("Done.")
 521
 522     def _parse_output(self):
 523         """Parse output specification in the specification YAML file.
 524
 525         :raises: PresentationError if there is no output defined.
 526         """
 527
 528         logging.info("Parsing specification file: output ...")
 529
 530         idx = self._get_type_index("output")
 531         if idx is None:
 532             raise PresentationError("No output defined.")
 533
 534         try:
 535             self._specification["output"] = self._cfg_yaml[idx]
 536         except (KeyError, IndexError):
 537             raise PresentationError("No output defined.")
 538
 539         logging.info("Done.")
 540
 541     def _parse_static(self):
 542         """Parse specification of the static content in the specification YAML
 543         file.
 544         """
 545
 546         logging.info("Parsing specification file: static content ...")
 547
 548         idx = self._get_type_index("static")
 549         if idx is None:
 550             logging.warning("No static content specified.")
 551
 552         for key, value in self._cfg_yaml[idx].items():
 553             if isinstance(value, str):
 554                 try:
 555                     self._cfg_yaml[idx][key] = self._replace_tags(
 556                         value, self._specification["environment"]["paths"])
 557                 except KeyError:
 558                     pass
 559
 560         self._specification["static"] = self._cfg_yaml[idx]
 561
 562         logging.info("Done.")
 563
 564     def _parse_elements(self):
 565         """Parse elements (tables, plots) specification in the specification
 566         YAML file.
 567         """
 568
 569         logging.info("Parsing specification file: elements ...")
 570
 571         count = 1
 572         for element in self._cfg_yaml:
 573             try:
 574                 element["output-file"] = self._replace_tags(
 575                     element["output-file"],
 576                     self._specification["environment"]["paths"])
 577             except KeyError:
 578                 pass
 579
 580             try:
 581                 element["input-file"] = self._replace_tags(
 582                     element["input-file"],
 583                     self._specification["environment"]["paths"])
 584             except KeyError:
 585                 pass
 586
 587             # add data sets to the elements:
 588             if isinstance(element.get("data", None), str):
 589                 data_set = element["data"]
 590                 try:
 591                     element["data"] = self.configuration["data-sets"][data_set]
 592                 except KeyError:
 593                     raise PresentationError("Data set {0} is not defined in "
 594                                             "the configuration section.".
 595                                             format(data_set))
 596
 597             if element["type"] == "table":
 598                 logging.info("  {:3d} Processing a table ...".format(count))
 599                 try:
 600                     element["template"] = self._replace_tags(
 601                         element["template"],
 602                         self._specification["environment"]["paths"])
 603                 except KeyError:
 604                     pass
 605                 self._specification["tables"].append(element)
 606                 count += 1
 607
 608             elif element["type"] == "plot":
 609                 logging.info("  {:3d} Processing a plot ...".format(count))
 610
 611                 # Add layout to the plots:
 612                 layout = element["layout"].get("layout", None)
 613                 if layout is not None:
 614                     element["layout"].pop("layout")
 615                     try:
 616                         for key, val in (self.configuration["plot-layouts"]
 617                                          [layout].items()):
 618                             element["layout"][key] = val
 619                     except KeyError:
 620                         raise PresentationError("Layout {0} is not defined in "
 621                                                 "the configuration section.".
 622                                                 format(layout))
 623                 self._specification["plots"].append(element)
 624                 count += 1
 625
 626             elif element["type"] == "file":
 627                 logging.info("  {:3d} Processing a file ...".format(count))
 628                 try:
 629                     element["dir-tables"] = self._replace_tags(
 630                         element["dir-tables"],
 631                         self._specification["environment"]["paths"])
 632                 except KeyError:
 633                     pass
 634                 self._specification["files"].append(element)
 635                 count += 1
 636
 637             elif element["type"] == "cpta":
 638                 logging.info("  {:3d} Processing Continuous Performance "
 639                              "Trending and Analysis ...".format(count))
 640
 641                 for plot in element["plots"]:
 642                     # Add layout to the plots:
 643                     layout = plot.get("layout", None)
 644                     if layout is not None:
 645                         try:
 646                             plot["layout"] = \
 647                                 self.configuration["plot-layouts"][layout]
 648                         except KeyError:
 649                             raise PresentationError(
 650                                 "Layout {0} is not defined in the "
 651                                 "configuration section.".format(layout))
 652                     # Add data sets:
 653                     if isinstance(plot.get("data", None), str):
 654                         data_set = plot["data"]
 655                         try:
 656                             plot["data"] = \
 657                                 self.configuration["data-sets"][data_set]
 658                         except KeyError:
 659                             raise PresentationError(
 660                                 "Data set {0} is not defined in "
 661                                 "the configuration section.".
 662                                 format(data_set))
 663                 self._specification["cpta"] = element
 664                 count += 1
 665
 666         logging.info("Done.")
 667
 668     def read_specification(self):
 669         """Parse specification in the specification YAML file.
 670
 671         :raises: PresentationError if an error occurred while parsing the
 672         specification file.
 673         """
 674         try:
 675             self._cfg_yaml = load(self._cfg_file)
 676         except YAMLError as err:
 677             raise PresentationError(msg="An error occurred while parsing the "
 678                                         "specification file.",
 679                                     details=str(err))
 680
 681         self._parse_env()
 682         self._parse_configuration()
 683         self._parse_input()
 684         self._parse_output()
 685         self._parse_static()
 686         self._parse_elements()
 687
 688         logging.debug("Specification: \n{}".
 689                       format(pformat(self._specification)))