resources/tools/presentation/specification_parser.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 import logging
  21 from yaml import load, YAMLError
  22 from pprint import pformat
  23
  24 from errors import PresentationError
  25 from utils import get_last_successful_build_number
  26 from utils import get_last_completed_build_number
  27
  28
  29 class Specification(object):
  30     """Specification of Presentation and analytics layer.
  31
  32     - based on specification specified in the specification YAML file
  33     - presentation and analytics layer is model driven
  34     """
  35
  36     # Tags are used in specification YAML file and replaced while the file is
  37     # parsed.
  38     TAG_OPENER = "{"
  39     TAG_CLOSER = "}"
  40
  41     def __init__(self, cfg_file):
  42         """Initialization.
  43
  44         :param cfg_file: File handler for the specification YAML file.
  45         :type cfg_file: BinaryIO
  46         """
  47         self._cfg_file = cfg_file
  48         self._cfg_yaml = None
  49
  50         self._specification = {"environment": dict(),
  51                                "configuration": dict(),
  52                                "static": dict(),
  53                                "input": dict(),
  54                                "output": dict(),
  55                                "tables": list(),
  56                                "plots": list(),
  57                                "files": list(),
  58                                "cpta": dict()}
  59
  60     @property
  61     def specification(self):
  62         """Getter - specification.
  63
  64         :returns: Specification.
  65         :rtype: dict
  66         """
  67         return self._specification
  68
  69     @property
  70     def environment(self):
  71         """Getter - environment.
  72
  73         :returns: Environment specification.
  74         :rtype: dict
  75         """
  76         return self._specification["environment"]
  77
  78     @property
  79     def configuration(self):
  80         """Getter - configuration.
  81
  82         :returns: Configuration of PAL.
  83         :rtype: dict
  84         """
  85         return self._specification["configuration"]
  86
  87     @property
  88     def static(self):
  89         """Getter - static content.
  90
  91         :returns: Static content specification.
  92         :rtype: dict
  93         """
  94         return self._specification["static"]
  95
  96     @property
  97     def mapping(self):
  98         """Getter - Mapping.
  99
 100         :returns: Mapping of the old names of test cases to the new (actual)
 101             one.
 102         :rtype: dict
 103         """
 104         return self._specification["configuration"]["mapping"]
 105
 106     @property
 107     def ignore(self):
 108         """Getter - Ignore list.
 109
 110         :returns: List of ignored test cases.
 111         :rtype: list
 112         """
 113         return self._specification["configuration"]["ignore"]
 114
 115     @property
 116     def input(self):
 117         """Getter - specification - inputs.
 118         - jobs and builds.
 119
 120         :returns: Inputs.
 121         :rtype: dict
 122         """
 123         return self._specification["input"]
 124
 125     @property
 126     def builds(self):
 127         """Getter - builds defined in specification.
 128
 129         :returns: Builds defined in the specification.
 130         :rtype: dict
 131         """
 132         return self.input["builds"]
 133
 134     @property
 135     def output(self):
 136         """Getter - specification - output formats and versions to be generated.
 137         - formats: html, pdf
 138         - versions: full, ...
 139
 140         :returns: Outputs to be generated.
 141         :rtype: dict
 142         """
 143         return self._specification["output"]
 144
 145     @property
 146     def tables(self):
 147         """Getter - tables to be generated.
 148
 149         :returns: List of specifications of tables to be generated.
 150         :rtype: list
 151         """
 152         return self._specification["tables"]
 153
 154     @property
 155     def plots(self):
 156         """Getter - plots to be generated.
 157
 158         :returns: List of specifications of plots to be generated.
 159         :rtype: list
 160         """
 161         return self._specification["plots"]
 162
 163     @property
 164     def files(self):
 165         """Getter - files to be generated.
 166
 167         :returns: List of specifications of files to be generated.
 168         :rtype: list
 169         """
 170         return self._specification["files"]
 171
 172     @property
 173     def cpta(self):
 174         """Getter - Continuous Performance Trending and Analysis to be
 175         generated.
 176
 177         :returns: List of specifications of Continuous Performance Trending and
 178         Analysis to be generated.
 179         :rtype: list
 180         """
 181         return self._specification["cpta"]
 182
 183     def set_input_state(self, job, build_nr, state):
 184         """Set the state of input
 185
 186         :param job:
 187         :param build_nr:
 188         :param state:
 189         :return:
 190         """
 191
 192         try:
 193             for build in self._specification["input"]["builds"][job]:
 194                 if build["build"] == build_nr:
 195                     build["status"] = state
 196                     break
 197             else:
 198                 raise PresentationError("Build '{}' is not defined for job '{}'"
 199                                         " in specification file.".
 200                                         format(build_nr, job))
 201         except KeyError:
 202             raise PresentationError("Job '{}' and build '{}' is not defined in "
 203                                     "specification file.".format(job, build_nr))
 204
 205     def set_input_file_name(self, job, build_nr, file_name):
 206         """Set the state of input
 207
 208         :param job:
 209         :param build_nr:
 210         :param file_name:
 211         :return:
 212         """
 213
 214         try:
 215             for build in self._specification["input"]["builds"][job]:
 216                 if build["build"] == build_nr:
 217                     build["file-name"] = file_name
 218                     break
 219             else:
 220                 raise PresentationError("Build '{}' is not defined for job '{}'"
 221                                         " in specification file.".
 222                                         format(build_nr, job))
 223         except KeyError:
 224             raise PresentationError("Job '{}' and build '{}' is not defined in "
 225                                     "specification file.".format(job, build_nr))
 226
 227     def _get_build_number(self, job, build_type):
 228         """Get the number of the job defined by its name:
 229          - lastSuccessfulBuild
 230          - lastCompletedBuild
 231
 232         :param job: Job name.
 233         :param build_type: Build type:
 234          - lastSuccessfulBuild
 235          - lastCompletedBuild
 236         :type job" str
 237         :raises PresentationError: If it is not possible to get the build
 238         number.
 239         :returns: The build number.
 240         :rtype: int
 241         """
 242
 243         # defined as a range <start, end>
 244         if build_type == "lastSuccessfulBuild":
 245             # defined as a range <start, lastSuccessfulBuild>
 246             ret_code, build_nr, _ = get_last_successful_build_number(
 247                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
 248         elif build_type == "lastCompletedBuild":
 249             # defined as a range <start, lastCompletedBuild>
 250             ret_code, build_nr, _ = get_last_completed_build_number(
 251                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
 252         else:
 253             raise PresentationError("Not supported build type: '{0}'".
 254                                     format(build_type))
 255         if ret_code != 0:
 256             raise PresentationError("Not possible to get the number of the "
 257                                     "build number.")
 258         try:
 259             build_nr = int(build_nr)
 260             return build_nr
 261         except ValueError as err:
 262             raise PresentationError("Not possible to get the number of the "
 263                                     "build number.\nReason: {0}".format(err))
 264
 265     def _get_type_index(self, item_type):
 266         """Get index of item type (environment, input, output, ...) in
 267         specification YAML file.
 268
 269         :param item_type: Item type: Top level items in specification YAML file,
 270         e.g.: environment, input, output.
 271         :type item_type: str
 272         :returns: Index of the given item type.
 273         :rtype: int
 274         """
 275
 276         index = 0
 277         for item in self._cfg_yaml:
 278             if item["type"] == item_type:
 279                 return index
 280             index += 1
 281         return None
 282
 283     def _find_tag(self, text):
 284         """Find the first tag in the given text. The tag is enclosed by the
 285         TAG_OPENER and TAG_CLOSER.
 286
 287         :param text: Text to be searched.
 288         :type text: str
 289         :returns: The tag, or None if not found.
 290         :rtype: str
 291         """
 292         try:
 293             start = text.index(self.TAG_OPENER)
 294             end = text.index(self.TAG_CLOSER, start + 1) + 1
 295             return text[start:end]
 296         except ValueError:
 297             return None
 298
 299     def _replace_tags(self, data, src_data=None):
 300         """Replace tag(s) in the data by their values.
 301
 302         :param data: The data where the tags will be replaced by their values.
 303         :param src_data: Data where the tags are defined. It is dictionary where
 304         the key is the tag and the value is the tag value. If not given, 'data'
 305         is used instead.
 306         :type data: str or dict
 307         :type src_data: dict
 308         :returns: Data with the tags replaced.
 309         :rtype: str or dict
 310         :raises: PresentationError if it is not possible to replace the tag or
 311         the data is not the supported data type (str, dict).
 312         """
 313
 314         if src_data is None:
 315             src_data = data
 316
 317         if isinstance(data, str):
 318             tag = self._find_tag(data)
 319             if tag is not None:
 320                 data = data.replace(tag, src_data[tag[1:-1]])
 321
 322         elif isinstance(data, dict):
 323             counter = 0
 324             for key, value in data.items():
 325                 tag = self._find_tag(value)
 326                 if tag is not None:
 327                     try:
 328                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 329                         counter += 1
 330                     except KeyError:
 331                         raise PresentationError("Not possible to replace the "
 332                                                 "tag '{}'".format(tag))
 333             if counter:
 334                 self._replace_tags(data, src_data)
 335         else:
 336             raise PresentationError("Replace tags: Not supported data type.")
 337
 338         return data
 339
 340     def _parse_env(self):
 341         """Parse environment specification in the specification YAML file.
 342         """
 343
 344         logging.info("Parsing specification file: environment ...")
 345
 346         idx = self._get_type_index("environment")
 347         if idx is None:
 348             return None
 349
 350         try:
 351             self._specification["environment"]["configuration"] = \
 352                 self._cfg_yaml[idx]["configuration"]
 353         except KeyError:
 354             self._specification["environment"]["configuration"] = None
 355
 356         try:
 357             self._specification["environment"]["paths"] = \
 358                 self._replace_tags(self._cfg_yaml[idx]["paths"])
 359         except KeyError:
 360             self._specification["environment"]["paths"] = None
 361
 362         try:
 363             self._specification["environment"]["urls"] = \
 364                 self._replace_tags(self._cfg_yaml[idx]["urls"])
 365         except KeyError:
 366             self._specification["environment"]["urls"] = None
 367
 368         try:
 369             self._specification["environment"]["make-dirs"] = \
 370                 self._cfg_yaml[idx]["make-dirs"]
 371         except KeyError:
 372             self._specification["environment"]["make-dirs"] = None
 373
 374         try:
 375             self._specification["environment"]["remove-dirs"] = \
 376                 self._cfg_yaml[idx]["remove-dirs"]
 377         except KeyError:
 378             self._specification["environment"]["remove-dirs"] = None
 379
 380         try:
 381             self._specification["environment"]["build-dirs"] = \
 382                 self._cfg_yaml[idx]["build-dirs"]
 383         except KeyError:
 384             self._specification["environment"]["build-dirs"] = None
 385
 386         logging.info("Done.")
 387
 388     def _parse_configuration(self):
 389         """Parse configuration of PAL in the specification YAML file.
 390         """
 391
 392         logging.info("Parsing specification file: configuration ...")
 393
 394         idx = self._get_type_index("configuration")
 395         if idx is None:
 396             logging.warning("No configuration information in the specification "
 397                             "file.")
 398             return None
 399
 400         try:
 401             self._specification["configuration"] = self._cfg_yaml[idx]
 402
 403         except KeyError:
 404             raise PresentationError("No configuration defined.")
 405
 406         # Data sets: Replace ranges by lists
 407         for set_name, data_set in self.configuration["data-sets"].items():
 408             for job, builds in data_set.items():
 409                 if builds:
 410                     if isinstance(builds, dict):
 411                         build_nr = builds.get("end", None)
 412                         try:
 413                             build_nr = int(build_nr)
 414                         except ValueError:
 415                             # defined as a range <start, build_type>
 416                             build_nr = self._get_build_number(job, build_nr)
 417                         builds = [x for x in range(builds["start"], build_nr+1)]
 418                         self.configuration["data-sets"][set_name][job] = builds
 419
 420         # Mapping table:
 421         mapping = None
 422         mapping_file_name = self._specification["configuration"].\
 423             get("mapping-file", None)
 424         if mapping_file_name:
 425             logging.debug("Mapping file: '{0}'".format(mapping_file_name))
 426             try:
 427                 with open(mapping_file_name, 'r') as mfile:
 428                     mapping = load(mfile)
 429                 logging.debug("Loaded mapping table:\n{0}".format(mapping))
 430             except (YAMLError, IOError) as err:
 431                 raise PresentationError(
 432                     msg="An error occurred while parsing the mapping file "
 433                         "'{0}'.".format(mapping_file_name),
 434                     details=repr(err))
 435         # Make sure everything is lowercase
 436         if mapping:
 437             self._specification["configuration"]["mapping"] = \
 438                 {key.lower(): val.lower() for key, val in mapping.iteritems()}
 439         else:
 440             self._specification["configuration"]["mapping"] = dict()
 441
 442         # Ignore list:
 443         ignore = None
 444         ignore_list_name = self._specification["configuration"].\
 445             get("ignore-list", None)
 446         if ignore_list_name:
 447             logging.debug("Ignore list file: '{0}'".format(ignore_list_name))
 448             try:
 449                 with open(ignore_list_name, 'r') as ifile:
 450                     ignore = load(ifile)
 451                 logging.debug("Loaded ignore list:\n{0}".format(ignore))
 452             except (YAMLError, IOError) as err:
 453                 raise PresentationError(
 454                     msg="An error occurred while parsing the ignore list file "
 455                         "'{0}'.".format(ignore_list_name),
 456                     details=repr(err))
 457         # Make sure everything is lowercase
 458         if ignore:
 459             self._specification["configuration"]["ignore"] = \
 460                 [item.lower() for item in ignore]
 461         else:
 462             self._specification["configuration"]["ignore"] = list()
 463
 464         logging.info("Done.")
 465
 466     def _parse_input(self):
 467         """Parse input specification in the specification YAML file.
 468
 469         :raises: PresentationError if there are no data to process.
 470         """
 471
 472         logging.info("Parsing specification file: input ...")
 473
 474         idx = self._get_type_index("input")
 475         if idx is None:
 476             raise PresentationError("No data to process.")
 477
 478         try:
 479             for key, value in self._cfg_yaml[idx]["general"].items():
 480                 self._specification["input"][key] = value
 481             self._specification["input"]["builds"] = dict()
 482
 483             for job, builds in self._cfg_yaml[idx]["builds"].items():
 484                 if builds:
 485                     if isinstance(builds, dict):
 486                         build_nr = builds.get("end", None)
 487                         try:
 488                             build_nr = int(build_nr)
 489                         except ValueError:
 490                             # defined as a range <start, build_type>
 491                             build_nr = self._get_build_number(job, build_nr)
 492                         builds = [x for x in range(builds["start"], build_nr+1)]
 493                     self._specification["input"]["builds"][job] = list()
 494                     for build in builds:
 495                         self._specification["input"]["builds"][job]. \
 496                             append({"build": build, "status": None})
 497
 498                 else:
 499                     logging.warning("No build is defined for the job '{}'. "
 500                                     "Trying to continue without it.".
 501                                     format(job))
 502         except KeyError:
 503             raise PresentationError("No data to process.")
 504
 505         logging.info("Done.")
 506
 507     def _parse_output(self):
 508         """Parse output specification in the specification YAML file.
 509
 510         :raises: PresentationError if there is no output defined.
 511         """
 512
 513         logging.info("Parsing specification file: output ...")
 514
 515         idx = self._get_type_index("output")
 516         if idx is None:
 517             raise PresentationError("No output defined.")
 518
 519         try:
 520             self._specification["output"] = self._cfg_yaml[idx]
 521         except (KeyError, IndexError):
 522             raise PresentationError("No output defined.")
 523
 524         logging.info("Done.")
 525
 526     def _parse_static(self):
 527         """Parse specification of the static content in the specification YAML
 528         file.
 529         """
 530
 531         logging.info("Parsing specification file: static content ...")
 532
 533         idx = self._get_type_index("static")
 534         if idx is None:
 535             logging.warning("No static content specified.")
 536
 537         for key, value in self._cfg_yaml[idx].items():
 538             if isinstance(value, str):
 539                 try:
 540                     self._cfg_yaml[idx][key] = self._replace_tags(
 541                         value, self._specification["environment"]["paths"])
 542                 except KeyError:
 543                     pass
 544
 545         self._specification["static"] = self._cfg_yaml[idx]
 546
 547         logging.info("Done.")
 548
 549     def _parse_elements(self):
 550         """Parse elements (tables, plots) specification in the specification
 551         YAML file.
 552         """
 553
 554         logging.info("Parsing specification file: elements ...")
 555
 556         count = 1
 557         for element in self._cfg_yaml:
 558             try:
 559                 element["output-file"] = self._replace_tags(
 560                     element["output-file"],
 561                     self._specification["environment"]["paths"])
 562             except KeyError:
 563                 pass
 564
 565             try:
 566                 element["input-file"] = self._replace_tags(
 567                     element["input-file"],
 568                     self._specification["environment"]["paths"])
 569             except KeyError:
 570                 pass
 571
 572             # add data sets to the elements:
 573             if isinstance(element.get("data", None), str):
 574                 data_set = element["data"]
 575                 try:
 576                     element["data"] = self.configuration["data-sets"][data_set]
 577                 except KeyError:
 578                     raise PresentationError("Data set {0} is not defined in "
 579                                             "the configuration section.".
 580                                             format(data_set))
 581
 582             if element["type"] == "table":
 583                 logging.info("  {:3d} Processing a table ...".format(count))
 584                 try:
 585                     element["template"] = self._replace_tags(
 586                         element["template"],
 587                         self._specification["environment"]["paths"])
 588                 except KeyError:
 589                     pass
 590                 self._specification["tables"].append(element)
 591                 count += 1
 592
 593             elif element["type"] == "plot":
 594                 logging.info("  {:3d} Processing a plot ...".format(count))
 595
 596                 # Add layout to the plots:
 597                 layout = element["layout"].get("layout", None)
 598                 if layout is not None:
 599                     element["layout"].pop("layout")
 600                     try:
 601                         for key, val in (self.configuration["plot-layouts"]
 602                                          [layout].items()):
 603                             element["layout"][key] = val
 604                     except KeyError:
 605                         raise PresentationError("Layout {0} is not defined in "
 606                                                 "the configuration section.".
 607                                                 format(layout))
 608                 self._specification["plots"].append(element)
 609                 count += 1
 610
 611             elif element["type"] == "file":
 612                 logging.info("  {:3d} Processing a file ...".format(count))
 613                 try:
 614                     element["dir-tables"] = self._replace_tags(
 615                         element["dir-tables"],
 616                         self._specification["environment"]["paths"])
 617                 except KeyError:
 618                     pass
 619                 self._specification["files"].append(element)
 620                 count += 1
 621
 622             elif element["type"] == "cpta":
 623                 logging.info("  {:3d} Processing Continuous Performance "
 624                              "Trending and Analysis ...".format(count))
 625
 626                 for plot in element["plots"]:
 627                     # Add layout to the plots:
 628                     layout = plot.get("layout", None)
 629                     if layout is not None:
 630                         try:
 631                             plot["layout"] = \
 632                                 self.configuration["plot-layouts"][layout]
 633                         except KeyError:
 634                             raise PresentationError(
 635                                 "Layout {0} is not defined in the "
 636                                 "configuration section.".format(layout))
 637                     # Add data sets:
 638                     if isinstance(plot.get("data", None), str):
 639                         data_set = plot["data"]
 640                         try:
 641                             plot["data"] = \
 642                                 self.configuration["data-sets"][data_set]
 643                         except KeyError:
 644                             raise PresentationError(
 645                                 "Data set {0} is not defined in "
 646                                 "the configuration section.".
 647                                 format(data_set))
 648                 self._specification["cpta"] = element
 649                 count += 1
 650
 651         logging.info("Done.")
 652
 653     def read_specification(self):
 654         """Parse specification in the specification YAML file.
 655
 656         :raises: PresentationError if an error occurred while parsing the
 657         specification file.
 658         """
 659         try:
 660             self._cfg_yaml = load(self._cfg_file)
 661         except YAMLError as err:
 662             raise PresentationError(msg="An error occurred while parsing the "
 663                                         "specification file.",
 664                                     details=str(err))
 665
 666         self._parse_env()
 667         self._parse_configuration()
 668         self._parse_input()
 669         self._parse_output()
 670         self._parse_static()
 671         self._parse_elements()
 672
 673         logging.debug("Specification: \n{}".
 674                       format(pformat(self._specification)))