resources/tools/presentation/specification_parser.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 from os.path import join, exists
  21
  22 import logging
  23 from pprint import pformat
  24
  25 from yaml import load, FullLoader, YAMLError
  26
  27 from pal_errors import PresentationError
  28 from pal_utils import (
  29     get_last_successful_build_nr, get_last_completed_build_number
  30 )
  31
  32
  33 class Specification:
  34     """Specification of Presentation and analytics layer.
  35
  36     - based on specification specified in the specification YAML files
  37     - presentation and analytics layer is model driven
  38     """
  39
  40     # Tags are used in specification YAML file and replaced while the file is
  41     # parsed.
  42     TAG_OPENER = u"{"
  43     TAG_CLOSER = u"}"
  44
  45     def __init__(self, cfg_dir):
  46         """Initialization.
  47
  48         :param cfg_dir: Directory with the specification files.
  49         :type cfg_dir: str
  50         """
  51         self._cfg_dir = cfg_dir
  52         self._cfg_yaml = None
  53
  54         self._specification = {
  55             u"environment": dict(),
  56             u"data_sets": dict(),
  57             u"layouts": dict(),
  58             u"static": dict(),
  59             u"input": dict(),
  60             u"output": dict(),
  61             u"tables": list(),
  62             u"plots": list(),
  63             u"files": list(),
  64             u"cpta": dict()
  65         }
  66
  67     @property
  68     def specification(self):
  69         """Getter - specification.
  70
  71         :returns: Specification.
  72         :rtype: dict
  73         """
  74         return self._specification
  75
  76     @property
  77     def environment(self):
  78         """Getter - environment.
  79
  80         :returns: Environment specification.
  81         :rtype: dict
  82         """
  83         return self._specification[u"environment"]
  84
  85     @property
  86     def data_sets(self):
  87         """Getter - data_sets.
  88
  89         :returns: Data sets.
  90         :rtype: dict
  91         """
  92         return self._specification[u"data_sets"]
  93
  94     @property
  95     def layouts(self):
  96         """Getter - layouts.
  97
  98         :returns: Layouts.
  99         :rtype: dict
 100         """
 101         return self._specification[u"layouts"]
 102
 103     @property
 104     def static(self):
 105         """Getter - static content.
 106
 107         :returns: Static content specification.
 108         :rtype: dict
 109         """
 110         return self._specification[u"static"]
 111
 112     @property
 113     def mapping(self):
 114         """Getter - Mapping.
 115
 116         :returns: Mapping of the old names of test cases to the new (actual)
 117             one.
 118         :rtype: dict
 119         """
 120         return self.environment[u"mapping"]
 121
 122     @property
 123     def ignore(self):
 124         """Getter - Ignore list.
 125
 126         :returns: List of ignored test cases.
 127         :rtype: list
 128         """
 129         return self.environment[u"ignore"]
 130
 131     @property
 132     def alerting(self):
 133         """Getter - Alerting.
 134
 135         # TODO
 136
 137         :returns: Specification of alerts.
 138         :rtype: dict
 139         """
 140         return self.environment[u"alerting"]
 141
 142     @property
 143     def input(self):
 144         """Getter - specification - inputs.
 145         - jobs and builds.
 146
 147         :returns: Inputs.
 148         :rtype: dict
 149         """
 150         return self._specification[u"input"]
 151
 152     @input.setter
 153     def input(self, new_value):
 154         """Setter - specification - inputs.
 155
 156         :param new_value: New value to be set.
 157         :type new_value: dict
 158         """
 159         self._specification[u"input"] = new_value
 160
 161     def add_build(self, job, build):
 162         """Add a build to the list of builds if it does not exist there.
 163
 164         :param job: The job which run the build.
 165         :param build: The build to be added.
 166         :type job: str
 167         :type build: dict
 168         """
 169         if self.input.get(job, None) is None:
 170             self.input[job] = list()
 171         for existing_build in self.input[job]:
 172             if existing_build[u"build"] == build[u"build"]:
 173                 break
 174         else:
 175             self.input[job].append(build)
 176
 177     @property
 178     def output(self):
 179         """Getter - specification - output formats and versions to be generated.
 180         - formats: html, pdf
 181         - versions: full, ...
 182
 183         :returns: Outputs to be generated.
 184         :rtype: dict
 185         """
 186         return self._specification[u"output"]
 187
 188     @property
 189     def tables(self):
 190         """Getter - tables to be generated.
 191
 192         :returns: List of specifications of tables to be generated.
 193         :rtype: list
 194         """
 195         return self._specification.get(u"tables", list())
 196
 197     @property
 198     def plots(self):
 199         """Getter - plots to be generated.
 200
 201         :returns: List of specifications of plots to be generated.
 202         :rtype: list
 203         """
 204         return self._specification.get(u"plots", list())
 205
 206     @property
 207     def files(self):
 208         """Getter - files to be generated.
 209
 210         :returns: List of specifications of files to be generated.
 211         :rtype: list
 212         """
 213         return self._specification.get(u"files", list())
 214
 215     @property
 216     def cpta(self):
 217         """Getter - Continuous Performance Trending and Analysis to be
 218         generated.
 219
 220         :returns: List of specifications of Continuous Performance Trending and
 221             Analysis to be generated.
 222         :rtype: list
 223         """
 224         return self._specification[u"cpta"]
 225
 226     def set_input_state(self, job, build_nr, state):
 227         """Set the state of the input.
 228
 229         :param job: Job name.
 230         :param build_nr: Build number.
 231         :param state: The new input state.
 232         :type job: str
 233         :type build_nr: int
 234         :type state: str
 235         :raises: PresentationError if wrong job and/or build is provided.
 236         """
 237
 238         try:
 239             for build in self.input[job]:
 240                 if build[u"build"] == build_nr:
 241                     build[u"status"] = state
 242                     break
 243             else:
 244                 raise PresentationError(
 245                     f"Build {build_nr} is not defined for job {job} in "
 246                     f"specification file."
 247                 )
 248         except KeyError:
 249             raise PresentationError(
 250                 f"Job {job} and build {build_nr} is not defined in "
 251                 f"specification file."
 252             )
 253
 254     def set_input_file_name(self, job, build_nr, file_name):
 255         """Set the file name for the input.
 256
 257         :param job: Job name.
 258         :param build_nr: Build number.
 259         :param file_name: The new file name.
 260         :type job: str
 261         :type build_nr: int
 262         :type file_name: str
 263         :raises: PresentationError if wrong job and/or build is provided.
 264         """
 265
 266         try:
 267             for build in self.input[job]:
 268                 if build[u"build"] == build_nr:
 269                     build[u"file-name"] = file_name
 270                     break
 271             else:
 272                 raise PresentationError(
 273                     f"Build {build_nr} is not defined for job {job} in "
 274                     f"specification file."
 275                 )
 276         except KeyError:
 277             raise PresentationError(
 278                 f"Job {job} and build {build_nr} is not defined in "
 279                 f"specification file."
 280             )
 281
 282     def _get_build_number(self, job, build_type):
 283         """Get the number of the job defined by its name:
 284          - lastSuccessfulBuild
 285          - lastCompletedBuild
 286
 287         :param job: Job name.
 288         :param build_type: Build type:
 289          - lastSuccessfulBuild
 290          - lastCompletedBuild
 291         :type job" str
 292         :raises PresentationError: If it is not possible to get the build
 293             number.
 294         :returns: The build number.
 295         :rtype: int
 296         """
 297
 298         # defined as a range <start, end>
 299         if build_type == u"lastSuccessfulBuild":
 300             # defined as a range <start, lastSuccessfulBuild>
 301             ret_code, build_nr, _ = get_last_successful_build_nr(
 302                 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
 303         elif build_type == u"lastCompletedBuild":
 304             # defined as a range <start, lastCompletedBuild>
 305             ret_code, build_nr, _ = get_last_completed_build_number(
 306                 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
 307         else:
 308             raise PresentationError(f"Not supported build type: {build_type}")
 309         if ret_code != 0:
 310             raise PresentationError(
 311                 f"Not possible to get the build number of {job}."
 312             )
 313         try:
 314             build_nr = int(build_nr)
 315             return build_nr
 316         except ValueError as err:
 317             raise PresentationError(
 318                 f"Not possible to get the build number of {job}. Reason:\n"
 319                 f"{repr(err)}"
 320             )
 321
 322     def _get_type_index(self, item_type):
 323         """Get index of item type (environment, input, output, ...) in
 324         specification YAML file.
 325
 326         :param item_type: Item type: Top level items in specification YAML file,
 327             e.g.: environment, input, output.
 328         :type item_type: str
 329         :returns: Index of the given item type.
 330         :rtype: int
 331         """
 332
 333         index = 0
 334         for item in self._cfg_yaml:
 335             if item[u"type"] == item_type:
 336                 return index
 337             index += 1
 338         return None
 339
 340     def _find_tag(self, text):
 341         """Find the first tag in the given text. The tag is enclosed by the
 342         TAG_OPENER and TAG_CLOSER.
 343
 344         :param text: Text to be searched.
 345         :type text: str
 346         :returns: The tag, or None if not found.
 347         :rtype: str
 348         """
 349         try:
 350             start = text.index(self.TAG_OPENER)
 351             end = text.index(self.TAG_CLOSER, start + 1) + 1
 352             return text[start:end]
 353         except ValueError:
 354             return None
 355
 356     def _replace_tags(self, data, src_data=None):
 357         """Replace tag(s) in the data by their values.
 358
 359         :param data: The data where the tags will be replaced by their values.
 360         :param src_data: Data where the tags are defined. It is dictionary where
 361             the key is the tag and the value is the tag value. If not given,
 362             'data' is used instead.
 363         :type data: str, list or dict
 364         :type src_data: dict
 365         :returns: Data with the tags replaced.
 366         :rtype: str, list or dict
 367         :raises: PresentationError if it is not possible to replace the tag or
 368             the data is not the supported data type (str, list or dict).
 369         """
 370
 371         if src_data is None:
 372             src_data = data
 373
 374         if isinstance(data, str):
 375             tag = self._find_tag(data)
 376             if tag is not None:
 377                 data = data.replace(tag, src_data[tag[1:-1]])
 378             return data
 379
 380         if isinstance(data, list):
 381             new_list = list()
 382             for item in data:
 383                 new_list.append(self._replace_tags(item, src_data))
 384             return new_list
 385
 386         if isinstance(data, dict):
 387             counter = 0
 388             for key, value in data.items():
 389                 tag = self._find_tag(value)
 390                 if tag is not None:
 391                     try:
 392                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 393                         counter += 1
 394                     except KeyError:
 395                         raise PresentationError(
 396                             f"Not possible to replace the tag {tag}"
 397                         )
 398             if counter:
 399                 self._replace_tags(data, src_data)
 400             return data
 401
 402         raise PresentationError(u"Replace tags: Not supported data type.")
 403
 404     def _parse_env(self):
 405         """Parse environment specification in the specification YAML file.
 406         """
 407
 408         logging.info(u"Parsing specification: ENVIRONMENT")
 409
 410         idx = self._get_type_index(u"environment")
 411         if idx is None:
 412             return
 413
 414         self._specification[u"environment"][u"spec-files"] = \
 415             self._cfg_yaml[idx].get(u"spec-files", None)
 416
 417         try:
 418             self._specification[u"environment"][u"paths"] = \
 419                 self._replace_tags(self._cfg_yaml[idx][u"paths"])
 420         except KeyError:
 421             self._specification[u"environment"][u"paths"] = None
 422
 423         self._specification[u"environment"][u"data-sources"] = \
 424             self._cfg_yaml[idx].get(u"data-sources", tuple())
 425         # Add statistics:
 426         for source in self._specification[u"environment"][u"data-sources"]:
 427             source[u"successful-downloads"] = 0
 428
 429         self._specification[u"environment"][u"make-dirs"] = \
 430             self._cfg_yaml[idx].get(u"make-dirs", None)
 431
 432         self._specification[u"environment"][u"remove-dirs"] = \
 433             self._cfg_yaml[idx].get(u"remove-dirs", None)
 434
 435         self._specification[u"environment"][u"build-dirs"] = \
 436             self._cfg_yaml[idx].get(u"build-dirs", None)
 437
 438         self._specification[u"environment"][u"testbeds"] = \
 439             self._cfg_yaml[idx].get(u"testbeds", None)
 440
 441         self._specification[u"environment"][u"limits"] = \
 442             self._cfg_yaml[idx].get(u"limits", None)
 443
 444         self._specification[u"environment"][u"urls"] = \
 445             self._cfg_yaml[idx].get(u"urls", None)
 446
 447         self._specification[u"environment"][u"archive-inputs"] = \
 448             self._cfg_yaml[idx].get(u"archive-inputs", False)
 449
 450         self._specification[u"environment"][u"reverse-input"] = \
 451             self._cfg_yaml[idx].get(u"reverse-input", False)
 452
 453         self._specification[u"environment"][u"time-period"] = \
 454             self._cfg_yaml[idx].get(u"time-period", None)
 455
 456         self._specification[u"environment"][u"alerting"] = \
 457             self._cfg_yaml[idx].get(u"alerting", None)
 458
 459         self._specification[u"environment"][u"mapping-file"] = \
 460             self._cfg_yaml[idx].get(u"mapping-file", None)
 461
 462         self._specification[u"environment"][u"ignore-list"] = \
 463             self._cfg_yaml[idx].get(u"ignore-list", None)
 464
 465         # Mapping table:
 466         self._load_mapping_table()
 467
 468         # Ignore list:
 469         self._load_ignore_list()
 470
 471     def _parse_layouts(self):
 472         """Parse layouts specification in the specification YAML file.
 473         """
 474
 475         logging.info(u"Parsing specification: LAYOUTS")
 476
 477         idx = self._get_type_index(u"layouts")
 478         if idx is None:
 479             return
 480
 481         try:
 482             self._specification[u"layouts"] = self._cfg_yaml[idx]
 483         except KeyError:
 484             raise PresentationError(u"No layouts defined.")
 485
 486     def _parse_data_sets(self):
 487         """Parse data sets specification in the specification YAML file.
 488         """
 489
 490         logging.info(u"Parsing specification: DATA SETS")
 491
 492         idx = self._get_type_index(u"data-sets")
 493         if idx is None:
 494             return
 495
 496         try:
 497             self._specification[u"data_sets"] = self._cfg_yaml[idx]
 498         except KeyError:
 499             raise PresentationError(u"No Data sets defined.")
 500
 501         # Replace ranges by lists
 502         for set_name, data_set in self.data_sets.items():
 503             if not isinstance(data_set, dict):
 504                 continue
 505             for job, builds in data_set.items():
 506                 if not builds:
 507                     continue
 508                 if isinstance(builds, dict):
 509                     build_end = builds.get(u"end", None)
 510                     max_builds = builds.get(u"max-builds", None)
 511                     reverse = builds.get(u"reverse", False)
 512                     try:
 513                         build_end = int(build_end)
 514                     except ValueError:
 515                         # defined as a range <start, build_type>
 516                         build_end = self._get_build_number(job, build_end)
 517                     builds = list(range(builds[u"start"], build_end + 1))
 518                     if max_builds and max_builds < len(builds):
 519                         builds = builds[-max_builds:]
 520                     if reverse:
 521                         builds.reverse()
 522                     self.data_sets[set_name][job] = builds
 523                 elif isinstance(builds, list):
 524                     for idx, item in enumerate(builds):
 525                         try:
 526                             builds[idx] = int(item)
 527                         except ValueError:
 528                             # defined as a range <build_type>
 529                             builds[idx] = self._get_build_number(job, item)
 530
 531         # Add sub-sets to sets (only one level):
 532         for set_name, data_set in self.data_sets.items():
 533             if isinstance(data_set, list):
 534                 new_set = dict()
 535                 for item in data_set:
 536                     try:
 537                         for key, val in self.data_sets[item].items():
 538                             new_set[key] = val
 539                     except KeyError:
 540                         raise PresentationError(
 541                             f"Data set {item} is not defined."
 542                         )
 543                 self.data_sets[set_name] = new_set
 544
 545     def _load_mapping_table(self):
 546         """Load a mapping table if it is specified. If not, use empty dict.
 547         """
 548
 549         mapping_file_name = self.environment.get(u"mapping-file", None)
 550         if mapping_file_name:
 551             try:
 552                 with open(mapping_file_name, u'r') as mfile:
 553                     mapping = load(mfile, Loader=FullLoader)
 554                     # Make sure everything is lowercase
 555                     self.environment[u"mapping"] = \
 556                         {key.lower(): val.lower() for key, val in
 557                          mapping.items()}
 558                 logging.debug(f"Loaded mapping table:\n{mapping}")
 559             except (YAMLError, IOError) as err:
 560                 raise PresentationError(
 561                     msg=f"An error occurred while parsing the mapping file "
 562                     f"{mapping_file_name}",
 563                     details=repr(err)
 564                 )
 565         else:
 566             self.environment[u"mapping"] = dict()
 567
 568     def _load_ignore_list(self):
 569         """Load an ignore list if it is specified. If not, use empty list.
 570         """
 571
 572         ignore_list_name = self.environment.get(u"ignore-list", None)
 573         if ignore_list_name:
 574             try:
 575                 with open(ignore_list_name, u'r') as ifile:
 576                     ignore = load(ifile, Loader=FullLoader)
 577                     # Make sure everything is lowercase
 578                     self.environment[u"ignore"] = \
 579                         [item.lower() for item in ignore]
 580                 logging.debug(f"Loaded ignore list:\n{ignore}")
 581             except (YAMLError, IOError) as err:
 582                 raise PresentationError(
 583                     msg=f"An error occurred while parsing the ignore list file "
 584                     f"{ignore_list_name}.",
 585                     details=repr(err)
 586                 )
 587         else:
 588             self.environment[u"ignore"] = list()
 589
 590     def _parse_output(self):
 591         """Parse output specification in the specification YAML file.
 592
 593         :raises: PresentationError if there is no output defined.
 594         """
 595
 596         logging.info(u"Parsing specification: OUTPUT")
 597
 598         idx = self._get_type_index(u"output")
 599         if idx is None:
 600             raise PresentationError(u"No output defined.")
 601
 602         try:
 603             self._specification[u"output"] = self._cfg_yaml[idx]
 604         except (KeyError, IndexError):
 605             raise PresentationError(u"No output defined.")
 606
 607     def _parse_static(self):
 608         """Parse specification of the static content in the specification YAML
 609         file.
 610         """
 611
 612         logging.info(u"Parsing specification: STATIC CONTENT")
 613
 614         idx = self._get_type_index(u"static")
 615         if idx is None:
 616             logging.warning(u"No static content specified.")
 617             self._specification[u"static"] = dict()
 618             return
 619
 620         for key, value in self._cfg_yaml[idx].items():
 621             if isinstance(value, str):
 622                 try:
 623                     self._cfg_yaml[idx][key] = self._replace_tags(
 624                         value, self._specification[u"environment"][u"paths"])
 625                 except KeyError:
 626                     pass
 627
 628         self._specification[u"static"] = self._cfg_yaml[idx]
 629
 630     def _parse_elements_tables(self, table):
 631         """Parse tables from the specification YAML file.
 632
 633         :param table: Table to be parsed from the specification file.
 634         :type table: dict
 635         :raises PresentationError: If wrong data set is used.
 636         """
 637
 638         try:
 639             table[u"template"] = self._replace_tags(
 640                 table[u"template"],
 641                 self._specification[u"environment"][u"paths"])
 642         except KeyError:
 643             pass
 644
 645         # Add data sets
 646         try:
 647             for item in (u"reference", u"compare"):
 648                 if table.get(item, None):
 649                     data_set = table[item].get(u"data", None)
 650                     if isinstance(data_set, str):
 651                         table[item][u"data"] = self.data_sets[data_set]
 652                     data_set = table[item].get(u"data-replacement", None)
 653                     if isinstance(data_set, str):
 654                         table[item][u"data-replacement"] = \
 655                             self.data_sets[data_set]
 656
 657             if table.get(u"columns", None):
 658                 for i in range(len(table[u"columns"])):
 659                     data_set = table[u"columns"][i].get(u"data-set", None)
 660                     if isinstance(data_set, str):
 661                         table[u"columns"][i][u"data-set"] = \
 662                             self.data_sets[data_set]
 663                     data_set = table[u"columns"][i].get(
 664                         u"data-replacement", None)
 665                     if isinstance(data_set, str):
 666                         table[u"columns"][i][u"data-replacement"] = \
 667                             self.data_sets[data_set]
 668
 669             if table.get(u"lines", None):
 670                 for i in range(len(table[u"lines"])):
 671                     data_set = table[u"lines"][i].get(u"data-set", None)
 672                     if isinstance(data_set, str):
 673                         table[u"lines"][i][u"data-set"] = \
 674                             self.data_sets[data_set]
 675
 676         except KeyError:
 677             raise PresentationError(
 678                 f"Wrong set '{data_set}' used in {table.get(u'title', u'')}."
 679             )
 680
 681         self._specification[u"tables"].append(table)
 682
 683     def _parse_elements_plots(self, plot):
 684         """Parse plots from the specification YAML file.
 685
 686         :param plot: Plot to be parsed from the specification file.
 687         :type plot: dict
 688         :raises PresentationError: If plot layout is not defined.
 689         """
 690
 691         # Add layout to the plots:
 692         layout = plot[u"layout"].get(u"layout", None)
 693         if layout is not None:
 694             plot[u"layout"].pop(u"layout")
 695             try:
 696                 for key, val in self.layouts[layout].items():
 697                     plot[u"layout"][key] = val
 698             except KeyError:
 699                 raise PresentationError(f"Layout {layout} is not defined.")
 700         self._specification[u"plots"].append(plot)
 701
 702     def _parse_elements_files(self, file):
 703         """Parse files from the specification YAML file.
 704
 705         :param file: File to be parsed from the specification file.
 706         :type file: dict
 707         """
 708
 709         try:
 710             file[u"dir-tables"] = self._replace_tags(
 711                 file[u"dir-tables"],
 712                 self._specification[u"environment"][u"paths"])
 713         except KeyError:
 714             pass
 715         self._specification[u"files"].append(file)
 716
 717     def _parse_elements_cpta(self, cpta):
 718         """Parse cpta from the specification YAML file.
 719
 720         :param cpta: cpta to be parsed from the specification file.
 721         :type cpta: dict
 722         :raises PresentationError: If wrong data set is used or if plot layout
 723             is not defined.
 724         """
 725
 726         for plot in cpta[u"plots"]:
 727             # Add layout to the plots:
 728             layout = plot.get(u"layout", None)
 729             if layout is not None:
 730                 try:
 731                     plot[u"layout"] = self.layouts[layout]
 732                 except KeyError:
 733                     raise PresentationError(f"Layout {layout} is not defined.")
 734             # Add data sets:
 735             if isinstance(plot.get(u"data", None), str):
 736                 data_set = plot[u"data"]
 737                 try:
 738                     plot[u"data"] = self.data_sets[data_set]
 739                 except KeyError:
 740                     raise PresentationError(
 741                         f"Data set {data_set} is not defined."
 742                     )
 743         self._specification[u"cpta"] = cpta
 744
 745     def _parse_elements(self):
 746         """Parse elements (tables, plots, ..) specification in the specification
 747         YAML file.
 748         """
 749
 750         logging.info(u"Parsing specification: ELEMENTS")
 751
 752         count = 1
 753         for element in self._cfg_yaml:
 754
 755             # Replace tags:
 756             try:
 757                 element[u"output-file"] = self._replace_tags(
 758                     element[u"output-file"],
 759                     self.environment[u"paths"]
 760                 )
 761             except KeyError:
 762                 pass
 763
 764             try:
 765                 element[u"input-file"] = self._replace_tags(
 766                     element[u"input-file"],
 767                     self.environment[u"paths"]
 768                 )
 769             except KeyError:
 770                 pass
 771
 772             try:
 773                 element[u"output-file-links"] = self._replace_tags(
 774                     element[u"output-file-links"],
 775                     self.environment[u"paths"]
 776                 )
 777             except KeyError:
 778                 pass
 779
 780             # Add data sets to the elements:
 781             if isinstance(element.get(u"data", None), str):
 782                 data_set = element[u"data"]
 783                 try:
 784                     element[u"data"] = self.data_sets[data_set]
 785                 except KeyError:
 786                     raise PresentationError(
 787                         f"Data set {data_set} is not defined."
 788                     )
 789             elif isinstance(element.get(u"data", None), list):
 790                 new_list = list()
 791                 for item in element[u"data"]:
 792                     try:
 793                         new_list.append(self.data_sets[item])
 794                     except KeyError:
 795                         raise PresentationError(
 796                             f"Data set {item} is not defined."
 797                         )
 798                 element[u"data"] = new_list
 799
 800             # Parse elements:
 801             if element[u"type"] == u"table":
 802                 logging.info(f"  {count:3d} Processing a table ...")
 803                 self._parse_elements_tables(element)
 804                 count += 1
 805             elif element[u"type"] == u"plot":
 806                 logging.info(f"  {count:3d} Processing a plot ...")
 807                 self._parse_elements_plots(element)
 808                 count += 1
 809             elif element[u"type"] == u"file":
 810                 logging.info(f"  {count:3d} Processing a file ...")
 811                 self._parse_elements_files(element)
 812                 count += 1
 813             elif element[u"type"] == u"cpta":
 814                 logging.info(
 815                     f"  {count:3d} Processing Continuous Performance Trending "
 816                     f"and Analysis ..."
 817                 )
 818                 self._parse_elements_cpta(element)
 819                 count += 1
 820
 821     def _prepare_input(self):
 822         """Use information from data sets and generate list of jobs and builds
 823         to download.
 824         """
 825
 826         logging.info(u"Parsing specification: INPUT")
 827
 828         idx = self._get_type_index(u"input")
 829         if idx is None:
 830             logging.info(u"Creating the list of inputs from data sets.")
 831             for data_set in self.data_sets.values():
 832                 if data_set == "data-sets":
 833                     continue
 834                 for job, builds in data_set.items():
 835                     for build in builds:
 836                         self.add_build(
 837                             job,
 838                             {
 839                                 u"build": build,
 840                                 u"status": None,
 841                                 u"file-name": None,
 842                                 u"source": None
 843                             }
 844                         )
 845         else:
 846             logging.info(u"Reading pre-defined inputs.")
 847             for job, builds in self._cfg_yaml[idx][u"builds"].items():
 848                 for build in builds:
 849                     self.add_build(
 850                         job,
 851                         {
 852                             u"build": build,
 853                             u"status": None,
 854                             u"file-name": None,
 855                             u"source": None
 856                         }
 857                     )
 858
 859         if self.environment[u"reverse-input"]:
 860             for builds in self.input.values():
 861                 builds.sort(key=lambda k: k[u"build"], reverse=True)
 862
 863     def read_specification(self):
 864         """Parse specification in the specification YAML files.
 865
 866         :raises: PresentationError if an error occurred while parsing the
 867             specification file.
 868         """
 869
 870         # It always starts with environment.yaml file, it must be present.
 871         spec_file = join(self._cfg_dir, u"environment.yaml")
 872         logging.info(f"Reading {spec_file}")
 873         if not exists(spec_file):
 874             raise PresentationError(f"The file {spec_file} does not exist.")
 875
 876         with open(spec_file, u"r") as file_read:
 877             try:
 878                 self._cfg_yaml = load(file_read, Loader=FullLoader)
 879             except YAMLError as err:
 880                 raise PresentationError(
 881                     f"An error occurred while parsing the specification file "
 882                     f"{spec_file}",
 883                     details=repr(err)
 884                 )
 885
 886         # Load the other specification files specified in the environment.yaml
 887         idx = self._get_type_index(u"environment")
 888         if idx is None:
 889             raise PresentationError(
 890                 f"No environment defined in the file {spec_file}"
 891             )
 892         for spec_file in self._cfg_yaml[idx].get(u"spec-files", tuple()):
 893             logging.info(f"Reading {spec_file}")
 894             if not exists(spec_file):
 895                 raise PresentationError(f"The file {spec_file} does not exist.")
 896             spec = None
 897             with open(spec_file, u"r") as file_read:
 898                 try:
 899                     spec = load(file_read, Loader=FullLoader)
 900                 except YAMLError as err:
 901                     raise PresentationError(
 902                         f"An error occurred while parsing the specification "
 903                         f"file {spec_file}",
 904                         details=repr(err)
 905                     )
 906             if spec:
 907                 self._cfg_yaml.extend(spec)
 908
 909         self._parse_env()
 910         self._parse_layouts()
 911         self._parse_data_sets()
 912         self._parse_output()
 913         self._parse_static()
 914         self._parse_elements()
 915         self._prepare_input()
 916
 917         logging.debug(f"Specification: \n{pformat(self.specification)}")