resources/tools/presentation/specification_parser.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 import logging
  21 from yaml import load, YAMLError
  22 from pprint import pformat
  23
  24 from errors import PresentationError
  25 from utils import get_last_build_number
  26
  27
  28 class Specification(object):
  29     """Specification of Presentation and analytics layer.
  30
  31     - based on specification specified in the specification YAML file
  32     - presentation and analytics layer is model driven
  33     """
  34
  35     # Tags are used in specification YAML file and replaced while the file is
  36     # parsed.
  37     TAG_OPENER = "{"
  38     TAG_CLOSER = "}"
  39
  40     def __init__(self, cfg_file):
  41         """Initialization.
  42
  43         :param cfg_file: File handler for the specification YAML file.
  44         :type cfg_file: BinaryIO
  45         """
  46         self._cfg_file = cfg_file
  47         self._cfg_yaml = None
  48
  49         self._specification = {"environment": dict(),
  50                                "configuration": dict(),
  51                                "debug": dict(),
  52                                "static": dict(),
  53                                "input": dict(),
  54                                "output": dict(),
  55                                "tables": list(),
  56                                "plots": list(),
  57                                "files": list(),
  58                                "cpta": dict()}
  59
  60     @property
  61     def specification(self):
  62         """Getter - specification.
  63
  64         :returns: Specification.
  65         :rtype: dict
  66         """
  67         return self._specification
  68
  69     @property
  70     def environment(self):
  71         """Getter - environment.
  72
  73         :returns: Environment specification.
  74         :rtype: dict
  75         """
  76         return self._specification["environment"]
  77
  78     @property
  79     def configuration(self):
  80         """Getter - configuration.
  81
  82         :returns: Configuration of PAL.
  83         :rtype: dict
  84         """
  85         return self._specification["configuration"]
  86
  87     @property
  88     def static(self):
  89         """Getter - static content.
  90
  91         :returns: Static content specification.
  92         :rtype: dict
  93         """
  94         return self._specification["static"]
  95
  96     @property
  97     def debug(self):
  98         """Getter - debug
  99
 100         :returns: Debug specification
 101         :rtype: dict
 102         """
 103         return self._specification["debug"]
 104
 105     @property
 106     def is_debug(self):
 107         """Getter - debug mode
 108
 109         :returns: True if debug mode is on, otherwise False.
 110         :rtype: bool
 111         """
 112
 113         try:
 114             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
 115                 return True
 116             else:
 117                 return False
 118         except KeyError:
 119             return False
 120
 121     @property
 122     def input(self):
 123         """Getter - specification - inputs.
 124         - jobs and builds.
 125
 126         :returns: Inputs.
 127         :rtype: dict
 128         """
 129         return self._specification["input"]
 130
 131     @property
 132     def builds(self):
 133         """Getter - builds defined in specification.
 134
 135         :returns: Builds defined in the specification.
 136         :rtype: dict
 137         """
 138         return self.input["builds"]
 139
 140     @property
 141     def output(self):
 142         """Getter - specification - output formats and versions to be generated.
 143         - formats: html, pdf
 144         - versions: full, ...
 145
 146         :returns: Outputs to be generated.
 147         :rtype: dict
 148         """
 149         return self._specification["output"]
 150
 151     @property
 152     def tables(self):
 153         """Getter - tables to be generated.
 154
 155         :returns: List of specifications of tables to be generated.
 156         :rtype: list
 157         """
 158         return self._specification["tables"]
 159
 160     @property
 161     def plots(self):
 162         """Getter - plots to be generated.
 163
 164         :returns: List of specifications of plots to be generated.
 165         :rtype: list
 166         """
 167         return self._specification["plots"]
 168
 169     @property
 170     def files(self):
 171         """Getter - files to be generated.
 172
 173         :returns: List of specifications of files to be generated.
 174         :rtype: list
 175         """
 176         return self._specification["files"]
 177
 178     @property
 179     def cpta(self):
 180         """Getter - Continuous Performance Trending and Analysis to be
 181         generated.
 182
 183         :returns: List of specifications of Continuous Performance Trending and
 184         Analysis to be generated.
 185         :rtype: list
 186         """
 187         return self._specification["cpta"]
 188
 189     def set_input_state(self, job, build_nr, state):
 190         """Set the state of input
 191
 192         :param job:
 193         :param build_nr:
 194         :param state:
 195         :return:
 196         """
 197
 198         try:
 199             for build in self._specification["input"]["builds"][job]:
 200                 if build["build"] == build_nr:
 201                     build["status"] = state
 202                     break
 203             else:
 204                 raise PresentationError("Build '{}' is not defined for job '{}'"
 205                                         " in specification file.".
 206                                         format(build_nr, job))
 207         except KeyError:
 208             raise PresentationError("Job '{}' and build '{}' is not defined in "
 209                                     "specification file.".format(job, build_nr))
 210
 211     def set_input_file_name(self, job, build_nr, file_name):
 212         """Set the state of input
 213
 214         :param job:
 215         :param build_nr:
 216         :param file_name:
 217         :return:
 218         """
 219
 220         try:
 221             for build in self._specification["input"]["builds"][job]:
 222                 if build["build"] == build_nr:
 223                     build["file-name"] = file_name
 224                     break
 225             else:
 226                 raise PresentationError("Build '{}' is not defined for job '{}'"
 227                                         " in specification file.".
 228                                         format(build_nr, job))
 229         except KeyError:
 230             raise PresentationError("Job '{}' and build '{}' is not defined in "
 231                                     "specification file.".format(job, build_nr))
 232
 233     def _get_type_index(self, item_type):
 234         """Get index of item type (environment, input, output, ...) in
 235         specification YAML file.
 236
 237         :param item_type: Item type: Top level items in specification YAML file,
 238         e.g.: environment, input, output.
 239         :type item_type: str
 240         :returns: Index of the given item type.
 241         :rtype: int
 242         """
 243
 244         index = 0
 245         for item in self._cfg_yaml:
 246             if item["type"] == item_type:
 247                 return index
 248             index += 1
 249         return None
 250
 251     def _find_tag(self, text):
 252         """Find the first tag in the given text. The tag is enclosed by the
 253         TAG_OPENER and TAG_CLOSER.
 254
 255         :param text: Text to be searched.
 256         :type text: str
 257         :returns: The tag, or None if not found.
 258         :rtype: str
 259         """
 260         try:
 261             start = text.index(self.TAG_OPENER)
 262             end = text.index(self.TAG_CLOSER, start + 1) + 1
 263             return text[start:end]
 264         except ValueError:
 265             return None
 266
 267     def _replace_tags(self, data, src_data=None):
 268         """Replace tag(s) in the data by their values.
 269
 270         :param data: The data where the tags will be replaced by their values.
 271         :param src_data: Data where the tags are defined. It is dictionary where
 272         the key is the tag and the value is the tag value. If not given, 'data'
 273         is used instead.
 274         :type data: str or dict
 275         :type src_data: dict
 276         :returns: Data with the tags replaced.
 277         :rtype: str or dict
 278         :raises: PresentationError if it is not possible to replace the tag or
 279         the data is not the supported data type (str, dict).
 280         """
 281
 282         if src_data is None:
 283             src_data = data
 284
 285         if isinstance(data, str):
 286             tag = self._find_tag(data)
 287             if tag is not None:
 288                 data = data.replace(tag, src_data[tag[1:-1]])
 289
 290         elif isinstance(data, dict):
 291             counter = 0
 292             for key, value in data.items():
 293                 tag = self._find_tag(value)
 294                 if tag is not None:
 295                     try:
 296                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 297                         counter += 1
 298                     except KeyError:
 299                         raise PresentationError("Not possible to replace the "
 300                                                 "tag '{}'".format(tag))
 301             if counter:
 302                 self._replace_tags(data, src_data)
 303         else:
 304             raise PresentationError("Replace tags: Not supported data type.")
 305
 306         return data
 307
 308     def _parse_env(self):
 309         """Parse environment specification in the specification YAML file.
 310         """
 311
 312         logging.info("Parsing specification file: environment ...")
 313
 314         idx = self._get_type_index("environment")
 315         if idx is None:
 316             return None
 317
 318         try:
 319             self._specification["environment"]["configuration"] = \
 320                 self._cfg_yaml[idx]["configuration"]
 321         except KeyError:
 322             self._specification["environment"]["configuration"] = None
 323
 324         try:
 325             self._specification["environment"]["paths"] = \
 326                 self._replace_tags(self._cfg_yaml[idx]["paths"])
 327         except KeyError:
 328             self._specification["environment"]["paths"] = None
 329
 330         try:
 331             self._specification["environment"]["urls"] = \
 332                 self._replace_tags(self._cfg_yaml[idx]["urls"])
 333         except KeyError:
 334             self._specification["environment"]["urls"] = None
 335
 336         try:
 337             self._specification["environment"]["make-dirs"] = \
 338                 self._cfg_yaml[idx]["make-dirs"]
 339         except KeyError:
 340             self._specification["environment"]["make-dirs"] = None
 341
 342         try:
 343             self._specification["environment"]["remove-dirs"] = \
 344                 self._cfg_yaml[idx]["remove-dirs"]
 345         except KeyError:
 346             self._specification["environment"]["remove-dirs"] = None
 347
 348         try:
 349             self._specification["environment"]["build-dirs"] = \
 350                 self._cfg_yaml[idx]["build-dirs"]
 351         except KeyError:
 352             self._specification["environment"]["build-dirs"] = None
 353
 354         logging.info("Done.")
 355
 356     def _parse_configuration(self):
 357         """Parse configuration of PAL in the specification YAML file.
 358         """
 359
 360         logging.info("Parsing specification file: configuration ...")
 361
 362         idx = self._get_type_index("configuration")
 363         if idx is None:
 364             logging.warning("No configuration information in the specification "
 365                             "file.")
 366             return None
 367
 368         try:
 369             self._specification["configuration"] = self._cfg_yaml[idx]
 370
 371         except KeyError:
 372             raise PresentationError("No configuration defined.")
 373
 374         # Data sets: Replace ranges by lists
 375         for set_name, data_set in self.configuration["data-sets"].items():
 376             for job, builds in data_set.items():
 377                 if builds:
 378                     if isinstance(builds, dict):
 379                         # defined as a range <start, end>
 380                         if builds.get("end", None) == "lastSuccessfulBuild":
 381                             # defined as a range <start, lastSuccessfulBuild>
 382                             ret_code, build_nr, _ = get_last_build_number(
 383                                 self.environment["urls"]["URL[JENKINS,CSIT]"],
 384                                 job)
 385                             if ret_code != 0:
 386                                 raise PresentationError(
 387                                     "Not possible to get the number of the "
 388                                     "last successful  build.")
 389                         else:
 390                             # defined as a range <start, end (build number)>
 391                             build_nr = builds.get("end", None)
 392                         builds = [x for x in range(builds["start"],
 393                                                    int(build_nr)+1)]
 394                         self.configuration["data-sets"][set_name][job] = builds
 395
 396         logging.info("Done.")
 397
 398     def _parse_debug(self):
 399         """Parse debug specification in the specification YAML file.
 400         """
 401
 402         if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
 403             return None
 404
 405         logging.info("Parsing specification file: debug ...")
 406
 407         idx = self._get_type_index("debug")
 408         if idx is None:
 409             self.environment["configuration"]["CFG[DEBUG]"] = 0
 410             return None
 411
 412         try:
 413             for key, value in self._cfg_yaml[idx]["general"].items():
 414                 self._specification["debug"][key] = value
 415
 416             self._specification["input"]["builds"] = dict()
 417             for job, builds in self._cfg_yaml[idx]["builds"].items():
 418                 if builds:
 419                     self._specification["input"]["builds"][job] = list()
 420                     for build in builds:
 421                         self._specification["input"]["builds"][job].\
 422                             append({"build": build["build"],
 423                                     "status": "downloaded",
 424                                     "file-name": self._replace_tags(
 425                                         build["file"],
 426                                         self.environment["paths"])})
 427                 else:
 428                     logging.warning("No build is defined for the job '{}'. "
 429                                     "Trying to continue without it.".
 430                                     format(job))
 431
 432         except KeyError:
 433             raise PresentationError("No data to process.")
 434
 435     def _parse_input(self):
 436         """Parse input specification in the specification YAML file.
 437
 438         :raises: PresentationError if there are no data to process.
 439         """
 440
 441         logging.info("Parsing specification file: input ...")
 442
 443         idx = self._get_type_index("input")
 444         if idx is None:
 445             raise PresentationError("No data to process.")
 446
 447         try:
 448             for key, value in self._cfg_yaml[idx]["general"].items():
 449                 self._specification["input"][key] = value
 450             self._specification["input"]["builds"] = dict()
 451
 452             for job, builds in self._cfg_yaml[idx]["builds"].items():
 453                 if builds:
 454                     if isinstance(builds, dict):
 455                         # defined as a range <start, end>
 456                         if builds.get("end", None) == "lastSuccessfulBuild":
 457                             # defined as a range <start, lastSuccessfulBuild>
 458                             ret_code, build_nr, _ = get_last_build_number(
 459                                 self.environment["urls"]["URL[JENKINS,CSIT]"],
 460                                 job)
 461                             if ret_code != 0:
 462                                 raise PresentationError(
 463                                     "Not possible to get the number of the "
 464                                     "last successful  build.")
 465                         else:
 466                             # defined as a range <start, end (build number)>
 467                             build_nr = builds.get("end", None)
 468                         builds = [x for x in range(builds["start"],
 469                                                    int(build_nr) + 1)]
 470                     self._specification["input"]["builds"][job] = list()
 471                     for build in builds:
 472                         self._specification["input"]["builds"][job].\
 473                             append({"build": build, "status": None})
 474                 else:
 475                     logging.warning("No build is defined for the job '{}'. "
 476                                     "Trying to continue without it.".
 477                                     format(job))
 478         except KeyError:
 479             raise PresentationError("No data to process.")
 480
 481         logging.info("Done.")
 482
 483     def _parse_output(self):
 484         """Parse output specification in the specification YAML file.
 485
 486         :raises: PresentationError if there is no output defined.
 487         """
 488
 489         logging.info("Parsing specification file: output ...")
 490
 491         idx = self._get_type_index("output")
 492         if idx is None:
 493             raise PresentationError("No output defined.")
 494
 495         try:
 496             self._specification["output"] = self._cfg_yaml[idx]
 497         except (KeyError, IndexError):
 498             raise PresentationError("No output defined.")
 499
 500         logging.info("Done.")
 501
 502     def _parse_static(self):
 503         """Parse specification of the static content in the specification YAML
 504         file.
 505         """
 506
 507         logging.info("Parsing specification file: static content ...")
 508
 509         idx = self._get_type_index("static")
 510         if idx is None:
 511             logging.warning("No static content specified.")
 512
 513         for key, value in self._cfg_yaml[idx].items():
 514             if isinstance(value, str):
 515                 try:
 516                     self._cfg_yaml[idx][key] = self._replace_tags(
 517                         value, self._specification["environment"]["paths"])
 518                 except KeyError:
 519                     pass
 520
 521         self._specification["static"] = self._cfg_yaml[idx]
 522
 523         logging.info("Done.")
 524
 525     def _parse_elements(self):
 526         """Parse elements (tables, plots) specification in the specification
 527         YAML file.
 528         """
 529
 530         logging.info("Parsing specification file: elements ...")
 531
 532         count = 1
 533         for element in self._cfg_yaml:
 534             try:
 535                 element["output-file"] = self._replace_tags(
 536                     element["output-file"],
 537                     self._specification["environment"]["paths"])
 538             except KeyError:
 539                 pass
 540
 541             # add data sets to the elements:
 542             if isinstance(element.get("data", None), str):
 543                 data_set = element["data"]
 544                 try:
 545                     element["data"] = self.configuration["data-sets"][data_set]
 546                 except KeyError:
 547                     raise PresentationError("Data set {0} is not defined in "
 548                                             "the configuration section.".
 549                                             format(data_set))
 550
 551             if element["type"] == "table":
 552                 logging.info("  {:3d} Processing a table ...".format(count))
 553                 try:
 554                     element["template"] = self._replace_tags(
 555                         element["template"],
 556                         self._specification["environment"]["paths"])
 557                 except KeyError:
 558                     pass
 559                 self._specification["tables"].append(element)
 560                 count += 1
 561
 562             elif element["type"] == "plot":
 563                 logging.info("  {:3d} Processing a plot ...".format(count))
 564
 565                 # Add layout to the plots:
 566                 layout = element["layout"].get("layout", None)
 567                 if layout is not None:
 568                     element["layout"].pop("layout")
 569                     try:
 570                         for key, val in (self.configuration["plot-layouts"]
 571                                          [layout].items()):
 572                             element["layout"][key] = val
 573                     except KeyError:
 574                         raise PresentationError("Layout {0} is not defined in "
 575                                                 "the configuration section.".
 576                                                 format(layout))
 577                 self._specification["plots"].append(element)
 578                 count += 1
 579
 580             elif element["type"] == "file":
 581                 logging.info("  {:3d} Processing a file ...".format(count))
 582                 try:
 583                     element["dir-tables"] = self._replace_tags(
 584                         element["dir-tables"],
 585                         self._specification["environment"]["paths"])
 586                 except KeyError:
 587                     pass
 588                 self._specification["files"].append(element)
 589                 count += 1
 590
 591             elif element["type"] == "cpta":
 592                 logging.info("  {:3d} Processing Continuous Performance "
 593                              "Trending and Analysis ...".format(count))
 594
 595                 for plot in element["plots"]:
 596                     # Add layout to the plots:
 597                     layout = plot.get("layout", None)
 598                     if layout is not None:
 599                         try:
 600                             plot["layout"] = \
 601                                 self.configuration["plot-layouts"][layout]
 602                         except KeyError:
 603                             raise PresentationError(
 604                                 "Layout {0} is not defined in the "
 605                                 "configuration section.".format(layout))
 606                     # Add data sets:
 607                     if isinstance(plot.get("data", None), str):
 608                         data_set = plot["data"]
 609                         try:
 610                             plot["data"] = \
 611                                 self.configuration["data-sets"][data_set]
 612                         except KeyError:
 613                             raise PresentationError(
 614                                 "Data set {0} is not defined in "
 615                                 "the configuration section.".
 616                                 format(data_set))
 617                 self._specification["cpta"] = element
 618                 count += 1
 619
 620         logging.info("Done.")
 621
 622     def read_specification(self):
 623         """Parse specification in the specification YAML file.
 624
 625         :raises: PresentationError if an error occurred while parsing the
 626         specification file.
 627         """
 628         try:
 629             self._cfg_yaml = load(self._cfg_file)
 630         except YAMLError as err:
 631             raise PresentationError(msg="An error occurred while parsing the "
 632                                         "specification file.",
 633                                     details=str(err))
 634
 635         self._parse_env()
 636         self._parse_configuration()
 637         self._parse_debug()
 638         if not self.debug:
 639             self._parse_input()
 640         self._parse_output()
 641         self._parse_static()
 642         self._parse_elements()
 643
 644         logging.debug("Specification: \n{}".
 645                       format(pformat(self._specification)))