resources/tools/presentation/specification_parser.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 from os.path import join, exists
  21
  22 import logging
  23 from pprint import pformat
  24
  25 from yaml import load, FullLoader, YAMLError
  26
  27 from pal_errors import PresentationError
  28 from pal_utils import (
  29     get_last_successful_build_nr, get_last_completed_build_number
  30 )
  31
  32
  33 class Specification:
  34     """Specification of Presentation and analytics layer.
  35
  36     - based on specification specified in the specification YAML files
  37     - presentation and analytics layer is model driven
  38     """
  39
  40     # Tags are used in specification YAML file and replaced while the file is
  41     # parsed.
  42     TAG_OPENER = u"{"
  43     TAG_CLOSER = u"}"
  44
  45     def __init__(self, cfg_dir):
  46         """Initialization.
  47
  48         :param cfg_dir: Directory with the specification files.
  49         :type cfg_dir: str
  50         """
  51         self._cfg_dir = cfg_dir
  52         self._cfg_yaml = None
  53
  54         self._specification = {
  55             u"environment": dict(),
  56             u"data_sets": dict(),
  57             u"layouts": dict(),
  58             u"static": dict(),
  59             u"input": dict(),
  60             u"output": dict(),
  61             u"tables": list(),
  62             u"plots": list(),
  63             u"files": list(),
  64             u"cpta": dict()
  65         }
  66
  67     @property
  68     def specification(self):
  69         """Getter - specification.
  70
  71         :returns: Specification.
  72         :rtype: dict
  73         """
  74         return self._specification
  75
  76     @property
  77     def environment(self):
  78         """Getter - environment.
  79
  80         :returns: Environment specification.
  81         :rtype: dict
  82         """
  83         return self._specification[u"environment"]
  84
  85     @property
  86     def data_sets(self):
  87         """Getter - data_sets.
  88
  89         :returns: Data sets.
  90         :rtype: dict
  91         """
  92         return self._specification[u"data_sets"]
  93
  94     @property
  95     def layouts(self):
  96         """Getter - layouts.
  97
  98         :returns: Layouts.
  99         :rtype: dict
 100         """
 101         return self._specification[u"layouts"]
 102
 103     @property
 104     def static(self):
 105         """Getter - static content.
 106
 107         :returns: Static content specification.
 108         :rtype: dict
 109         """
 110         return self._specification[u"static"]
 111
 112     @property
 113     def mapping(self):
 114         """Getter - Mapping.
 115
 116         :returns: Mapping of the old names of test cases to the new (actual)
 117             one.
 118         :rtype: dict
 119         """
 120         return self.environment[u"mapping"]
 121
 122     @property
 123     def ignore(self):
 124         """Getter - Ignore list.
 125
 126         :returns: List of ignored test cases.
 127         :rtype: list
 128         """
 129         return self.environment[u"ignore"]
 130
 131     @property
 132     def alerting(self):
 133         """Getter - Alerting.
 134
 135         # TODO
 136
 137         :returns: Specification of alerts.
 138         :rtype: dict
 139         """
 140         return self.environment[u"alerting"]
 141
 142     @property
 143     def input(self):
 144         """Getter - specification - inputs.
 145         - jobs and builds.
 146
 147         :returns: Inputs.
 148         :rtype: dict
 149         """
 150         return self._specification[u"input"]
 151
 152     @input.setter
 153     def input(self, new_value):
 154         """Setter - specification - inputs.
 155
 156         :param new_value: New value to be set.
 157         :type new_value: dict
 158         """
 159         self._specification[u"input"] = new_value
 160
 161     def add_build(self, job, build):
 162         """Add a build to the list of builds if it does not exist there.
 163
 164         :param job: The job which run the build.
 165         :param build: The build to be added.
 166         :type job: str
 167         :type build: dict
 168         """
 169         if self.input.get(job, None) is None:
 170             self.input[job] = list()
 171         for existing_build in self.input[job]:
 172             if existing_build[u"build"] == build[u"build"]:
 173                 break
 174         else:
 175             self.input[job].append(build)
 176
 177     @property
 178     def output(self):
 179         """Getter - specification - output formats and versions to be generated.
 180         - formats: html, pdf
 181         - versions: full, ...
 182
 183         :returns: Outputs to be generated.
 184         :rtype: dict
 185         """
 186         return self._specification[u"output"]
 187
 188     @property
 189     def tables(self):
 190         """Getter - tables to be generated.
 191
 192         :returns: List of specifications of tables to be generated.
 193         :rtype: list
 194         """
 195         return self._specification.get(u"tables", list())
 196
 197     @property
 198     def plots(self):
 199         """Getter - plots to be generated.
 200
 201         :returns: List of specifications of plots to be generated.
 202         :rtype: list
 203         """
 204         return self._specification.get(u"plots", list())
 205
 206     @property
 207     def files(self):
 208         """Getter - files to be generated.
 209
 210         :returns: List of specifications of files to be generated.
 211         :rtype: list
 212         """
 213         return self._specification.get(u"files", list())
 214
 215     @property
 216     def cpta(self):
 217         """Getter - Continuous Performance Trending and Analysis to be
 218         generated.
 219
 220         :returns: List of specifications of Continuous Performance Trending and
 221             Analysis to be generated.
 222         :rtype: list
 223         """
 224         return self._specification[u"cpta"]
 225
 226     def set_input_state(self, job, build_nr, state):
 227         """Set the state of the input.
 228
 229         :param job: Job name.
 230         :param build_nr: Build number.
 231         :param state: The new input state.
 232         :type job: str
 233         :type build_nr: int
 234         :type state: str
 235         :raises: PresentationError if wrong job and/or build is provided.
 236         """
 237
 238         try:
 239             for build in self.input[job]:
 240                 if build[u"build"] == build_nr:
 241                     build[u"status"] = state
 242                     break
 243             else:
 244                 raise PresentationError(
 245                     f"Build {build_nr} is not defined for job {job} in "
 246                     f"specification file."
 247                 )
 248         except KeyError:
 249             raise PresentationError(
 250                 f"Job {job} and build {build_nr} is not defined in "
 251                 f"specification file."
 252             )
 253
 254     def set_input_file_name(self, job, build_nr, file_name):
 255         """Set the file name for the input.
 256
 257         :param job: Job name.
 258         :param build_nr: Build number.
 259         :param file_name: The new file name.
 260         :type job: str
 261         :type build_nr: int
 262         :type file_name: str
 263         :raises: PresentationError if wrong job and/or build is provided.
 264         """
 265
 266         try:
 267             for build in self.input[job]:
 268                 if build[u"build"] == build_nr:
 269                     build[u"file-name"] = file_name
 270                     break
 271             else:
 272                 raise PresentationError(
 273                     f"Build {build_nr} is not defined for job {job} in "
 274                     f"specification file."
 275                 )
 276         except KeyError:
 277             raise PresentationError(
 278                 f"Job {job} and build {build_nr} is not defined in "
 279                 f"specification file."
 280             )
 281
 282     def _get_build_number(self, job, build_type):
 283         """Get the number of the job defined by its name:
 284          - lastSuccessfulBuild
 285          - lastCompletedBuild
 286
 287         :param job: Job name.
 288         :param build_type: Build type:
 289          - lastSuccessfulBuild
 290          - lastCompletedBuild
 291         :type job" str
 292         :raises PresentationError: If it is not possible to get the build
 293             number.
 294         :returns: The build number.
 295         :rtype: int
 296         """
 297
 298         # defined as a range <start, end>
 299         if build_type == u"lastSuccessfulBuild":
 300             # defined as a range <start, lastSuccessfulBuild>
 301             ret_code, build_nr, _ = get_last_successful_build_nr(
 302                 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
 303         elif build_type == u"lastCompletedBuild":
 304             # defined as a range <start, lastCompletedBuild>
 305             ret_code, build_nr, _ = get_last_completed_build_number(
 306                 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
 307         else:
 308             raise PresentationError(f"Not supported build type: {build_type}")
 309         if ret_code != 0:
 310             raise PresentationError(
 311                 f"Not possible to get the build number of {job}."
 312             )
 313         try:
 314             build_nr = int(build_nr)
 315             return build_nr
 316         except ValueError as err:
 317             raise PresentationError(
 318                 f"Not possible to get the build number of {job}. Reason:\n"
 319                 f"{repr(err)}"
 320             )
 321
 322     def _get_type_index(self, item_type):
 323         """Get index of item type (environment, input, output, ...) in
 324         specification YAML file.
 325
 326         :param item_type: Item type: Top level items in specification YAML file,
 327             e.g.: environment, input, output.
 328         :type item_type: str
 329         :returns: Index of the given item type.
 330         :rtype: int
 331         """
 332
 333         index = 0
 334         for item in self._cfg_yaml:
 335             if item[u"type"] == item_type:
 336                 return index
 337             index += 1
 338         return None
 339
 340     def _find_tag(self, text):
 341         """Find the first tag in the given text. The tag is enclosed by the
 342         TAG_OPENER and TAG_CLOSER.
 343
 344         :param text: Text to be searched.
 345         :type text: str
 346         :returns: The tag, or None if not found.
 347         :rtype: str
 348         """
 349         try:
 350             start = text.index(self.TAG_OPENER)
 351             end = text.index(self.TAG_CLOSER, start + 1) + 1
 352             return text[start:end]
 353         except ValueError:
 354             return None
 355
 356     def _replace_tags(self, data, src_data=None):
 357         """Replace tag(s) in the data by their values.
 358
 359         :param data: The data where the tags will be replaced by their values.
 360         :param src_data: Data where the tags are defined. It is dictionary where
 361             the key is the tag and the value is the tag value. If not given,
 362             'data' is used instead.
 363         :type data: str, list or dict
 364         :type src_data: dict
 365         :returns: Data with the tags replaced.
 366         :rtype: str, list or dict
 367         :raises: PresentationError if it is not possible to replace the tag or
 368             the data is not the supported data type (str, list or dict).
 369         """
 370
 371         if src_data is None:
 372             src_data = data
 373
 374         if isinstance(data, str):
 375             tag = self._find_tag(data)
 376             if tag is not None:
 377                 data = data.replace(tag, src_data[tag[1:-1]])
 378             return data
 379
 380         if isinstance(data, list):
 381             new_list = list()
 382             for item in data:
 383                 new_list.append(self._replace_tags(item, src_data))
 384             return new_list
 385
 386         if isinstance(data, dict):
 387             counter = 0
 388             for key, value in data.items():
 389                 tag = self._find_tag(value)
 390                 if tag is not None:
 391                     try:
 392                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 393                         counter += 1
 394                     except KeyError:
 395                         raise PresentationError(
 396                             f"Not possible to replace the tag {tag}"
 397                         )
 398             if counter:
 399                 self._replace_tags(data, src_data)
 400             return data
 401
 402         raise PresentationError(u"Replace tags: Not supported data type.")
 403
 404     def _parse_env(self):
 405         """Parse environment specification in the specification YAML file.
 406         """
 407
 408         logging.info(u"Parsing specification: ENVIRONMENT")
 409
 410         idx = self._get_type_index(u"environment")
 411         if idx is None:
 412             return
 413
 414         self._specification[u"environment"][u"spec-files"] = \
 415             self._cfg_yaml[idx].get(u"spec-files", None)
 416
 417         try:
 418             self._specification[u"environment"][u"paths"] = \
 419                 self._replace_tags(self._cfg_yaml[idx][u"paths"])
 420         except KeyError:
 421             self._specification[u"environment"][u"paths"] = None
 422
 423         self._specification[u"environment"][u"data-sources"] = \
 424             self._cfg_yaml[idx].get(u"data-sources", tuple())
 425         # Add statistics:
 426         for source in self._specification[u"environment"][u"data-sources"]:
 427             source[u"successful-downloads"] = 0
 428
 429         self._specification[u"environment"][u"make-dirs"] = \
 430             self._cfg_yaml[idx].get(u"make-dirs", None)
 431
 432         self._specification[u"environment"][u"remove-dirs"] = \
 433             self._cfg_yaml[idx].get(u"remove-dirs", None)
 434
 435         self._specification[u"environment"][u"build-dirs"] = \
 436             self._cfg_yaml[idx].get(u"build-dirs", None)
 437
 438         self._specification[u"environment"][u"testbeds"] = \
 439             self._cfg_yaml[idx].get(u"testbeds", None)
 440
 441         self._specification[u"environment"][u"limits"] = \
 442             self._cfg_yaml[idx].get(u"limits", None)
 443
 444         self._specification[u"environment"][u"urls"] = \
 445             self._cfg_yaml[idx].get(u"urls", None)
 446
 447         self._specification[u"environment"][u"archive-inputs"] = \
 448             self._cfg_yaml[idx].get(u"archive-inputs", False)
 449
 450         self._specification[u"environment"][u"reverse-input"] = \
 451             self._cfg_yaml[idx].get(u"reverse-input", False)
 452
 453         self._specification[u"environment"][u"time-period"] = \
 454             self._cfg_yaml[idx].get(u"time-period", None)
 455
 456         self._specification[u"environment"][u"alerting"] = \
 457             self._cfg_yaml[idx].get(u"alerting", None)
 458
 459         self._specification[u"environment"][u"mapping-file"] = \
 460             self._cfg_yaml[idx].get(u"mapping-file", None)
 461
 462         self._specification[u"environment"][u"ignore-list"] = \
 463             self._cfg_yaml[idx].get(u"ignore-list", None)
 464
 465         # Mapping table:
 466         self._load_mapping_table()
 467
 468         # Ignore list:
 469         self._load_ignore_list()
 470
 471     def _parse_layouts(self):
 472         """Parse layouts specification in the specification YAML file.
 473         """
 474
 475         logging.info(u"Parsing specification: LAYOUTS")
 476
 477         idx = self._get_type_index(u"layouts")
 478         if idx is None:
 479             return
 480
 481         try:
 482             self._specification[u"layouts"] = self._cfg_yaml[idx]
 483         except KeyError:
 484             raise PresentationError(u"No layouts defined.")
 485
 486     def _parse_data_sets(self):
 487         """Parse data sets specification in the specification YAML file.
 488         """
 489
 490         logging.info(u"Parsing specification: DATA SETS")
 491
 492         idx = self._get_type_index(u"data-sets")
 493         if idx is None:
 494             return
 495
 496         try:
 497             self._specification[u"data_sets"] = self._cfg_yaml[idx]
 498         except KeyError:
 499             raise PresentationError(u"No Data sets defined.")
 500
 501         # Replace ranges by lists
 502         for set_name, data_set in self.data_sets.items():
 503             if not isinstance(data_set, dict):
 504                 continue
 505             for job, builds in data_set.items():
 506                 if not builds:
 507                     continue
 508                 if isinstance(builds, dict):
 509                     build_end = builds.get(u"end", None)
 510                     max_builds = builds.get(u"max-builds", None)
 511                     reverse = builds.get(u"reverse", False)
 512                     try:
 513                         build_end = int(build_end)
 514                     except ValueError:
 515                         # defined as a range <start, build_type>
 516                         build_end = self._get_build_number(job, build_end)
 517                     builds = list(range(builds[u"start"], build_end + 1))
 518                     if max_builds and max_builds < len(builds):
 519                         builds = builds[-max_builds:]
 520                     if reverse:
 521                         builds.reverse()
 522                     self.data_sets[set_name][job] = builds
 523                 elif isinstance(builds, list):
 524                     for idx, item in enumerate(builds):
 525                         try:
 526                             builds[idx] = int(item)
 527                         except ValueError:
 528                             # defined as a range <build_type>
 529                             builds[idx] = self._get_build_number(job, item)
 530
 531         # Add sub-sets to sets (only one level):
 532         for set_name, data_set in self.data_sets.items():
 533             if isinstance(data_set, list):
 534                 new_set = dict()
 535                 for item in data_set:
 536                     try:
 537                         for key, val in self.data_sets[item].items():
 538                             new_set[key] = val
 539                     except KeyError:
 540                         raise PresentationError(
 541                             f"Data set {item} is not defined."
 542                         )
 543                 self.data_sets[set_name] = new_set
 544
 545     def _load_mapping_table(self):
 546         """Load a mapping table if it is specified. If not, use empty dict.
 547         """
 548
 549         mapping_file_name = self.environment.get(u"mapping-file", None)
 550         if mapping_file_name:
 551             try:
 552                 with open(mapping_file_name, u'r') as mfile:
 553                     mapping = load(mfile, Loader=FullLoader)
 554                     # Make sure everything is lowercase
 555                     self.environment[u"mapping"] = \
 556                         {key.lower(): val.lower() for key, val in
 557                          mapping.items()}
 558                 logging.debug(f"Loaded mapping table:\n{mapping}")
 559             except (YAMLError, IOError) as err:
 560                 raise PresentationError(
 561                     msg=f"An error occurred while parsing the mapping file "
 562                     f"{mapping_file_name}",
 563                     details=repr(err)
 564                 )
 565         else:
 566             self.environment[u"mapping"] = dict()
 567
 568     def _load_ignore_list(self):
 569         """Load an ignore list if it is specified. If not, use empty list.
 570         """
 571
 572         ignore_list_name = self.environment.get(u"ignore-list", None)
 573         if ignore_list_name:
 574             try:
 575                 with open(ignore_list_name, u'r') as ifile:
 576                     ignore = load(ifile, Loader=FullLoader)
 577                     # Make sure everything is lowercase
 578                     self.environment[u"ignore"] = \
 579                         [item.lower() for item in ignore]
 580                 logging.debug(f"Loaded ignore list:\n{ignore}")
 581             except (YAMLError, IOError) as err:
 582                 raise PresentationError(
 583                     msg=f"An error occurred while parsing the ignore list file "
 584                     f"{ignore_list_name}.",
 585                     details=repr(err)
 586                 )
 587         else:
 588             self.environment[u"ignore"] = list()
 589
 590     def _parse_output(self):
 591         """Parse output specification in the specification YAML file.
 592
 593         :raises: PresentationError if there is no output defined.
 594         """
 595
 596         logging.info(u"Parsing specification: OUTPUT")
 597
 598         idx = self._get_type_index(u"output")
 599         if idx is None:
 600             raise PresentationError(u"No output defined.")
 601
 602         try:
 603             self._specification[u"output"] = self._cfg_yaml[idx]
 604         except (KeyError, IndexError):
 605             raise PresentationError(u"No output defined.")
 606
 607     def _parse_static(self):
 608         """Parse specification of the static content in the specification YAML
 609         file.
 610         """
 611
 612         logging.info(u"Parsing specification: STATIC CONTENT")
 613
 614         idx = self._get_type_index(u"static")
 615         if idx is None:
 616             logging.warning(u"No static content specified.")
 617             self._specification[u"static"] = dict()
 618             return
 619
 620         for key, value in self._cfg_yaml[idx].items():
 621             if isinstance(value, str):
 622                 try:
 623                     self._cfg_yaml[idx][key] = self._replace_tags(
 624                         value, self._specification[u"environment"][u"paths"])
 625                 except KeyError:
 626                     pass
 627
 628         self._specification[u"static"] = self._cfg_yaml[idx]
 629
 630     def _parse_elements_tables(self, table):
 631         """Parse tables from the specification YAML file.
 632
 633         :param table: Table to be parsed from the specification file.
 634         :type table: dict
 635         :raises PresentationError: If wrong data set is used.
 636         """
 637
 638         try:
 639             table[u"template"] = self._replace_tags(
 640                 table[u"template"],
 641                 self._specification[u"environment"][u"paths"])
 642         except KeyError:
 643             pass
 644
 645         # Add data sets
 646         try:
 647             for item in (u"reference", u"compare"):
 648                 if table.get(item, None):
 649                     data_set = table[item].get(u"data", None)
 650                     if isinstance(data_set, str):
 651                         table[item][u"data"] = self.data_sets[data_set]
 652                     data_set = table[item].get(u"data-replacement", None)
 653                     if isinstance(data_set, str):
 654                         table[item][u"data-replacement"] = \
 655                             self.data_sets[data_set]
 656
 657             if table.get(u"columns", None):
 658                 for i in range(len(table[u"columns"])):
 659                     data_set = table[u"columns"][i].get(u"data-set", None)
 660                     if isinstance(data_set, str):
 661                         table[u"columns"][i][u"data-set"] = \
 662                             self.data_sets[data_set]
 663                     data_set = table[u"columns"][i].get(
 664                         u"data-replacement", None)
 665                     if isinstance(data_set, str):
 666                         table[u"columns"][i][u"data-replacement"] = \
 667                             self.data_sets[data_set]
 668
 669         except KeyError:
 670             raise PresentationError(
 671                 f"Wrong data set used in {table.get(u'title', u'')}."
 672             )
 673
 674         self._specification[u"tables"].append(table)
 675
 676     def _parse_elements_plots(self, plot):
 677         """Parse plots from the specification YAML file.
 678
 679         :param plot: Plot to be parsed from the specification file.
 680         :type plot: dict
 681         :raises PresentationError: If plot layout is not defined.
 682         """
 683
 684         # Add layout to the plots:
 685         layout = plot[u"layout"].get(u"layout", None)
 686         if layout is not None:
 687             plot[u"layout"].pop(u"layout")
 688             try:
 689                 for key, val in self.layouts[layout].items():
 690                     plot[u"layout"][key] = val
 691             except KeyError:
 692                 raise PresentationError(f"Layout {layout} is not defined.")
 693         self._specification[u"plots"].append(plot)
 694
 695     def _parse_elements_files(self, file):
 696         """Parse files from the specification YAML file.
 697
 698         :param file: File to be parsed from the specification file.
 699         :type file: dict
 700         """
 701
 702         try:
 703             file[u"dir-tables"] = self._replace_tags(
 704                 file[u"dir-tables"],
 705                 self._specification[u"environment"][u"paths"])
 706         except KeyError:
 707             pass
 708         self._specification[u"files"].append(file)
 709
 710     def _parse_elements_cpta(self, cpta):
 711         """Parse cpta from the specification YAML file.
 712
 713         :param cpta: cpta to be parsed from the specification file.
 714         :type cpta: dict
 715         :raises PresentationError: If wrong data set is used or if plot layout
 716             is not defined.
 717         """
 718
 719         for plot in cpta[u"plots"]:
 720             # Add layout to the plots:
 721             layout = plot.get(u"layout", None)
 722             if layout is not None:
 723                 try:
 724                     plot[u"layout"] = self.layouts[layout]
 725                 except KeyError:
 726                     raise PresentationError(f"Layout {layout} is not defined.")
 727             # Add data sets:
 728             if isinstance(plot.get(u"data", None), str):
 729                 data_set = plot[u"data"]
 730                 try:
 731                     plot[u"data"] = self.data_sets[data_set]
 732                 except KeyError:
 733                     raise PresentationError(
 734                         f"Data set {data_set} is not defined."
 735                     )
 736         self._specification[u"cpta"] = cpta
 737
 738     def _parse_elements(self):
 739         """Parse elements (tables, plots, ..) specification in the specification
 740         YAML file.
 741         """
 742
 743         logging.info(u"Parsing specification: ELEMENTS")
 744
 745         count = 1
 746         for element in self._cfg_yaml:
 747
 748             # Replace tags:
 749             try:
 750                 element[u"output-file"] = self._replace_tags(
 751                     element[u"output-file"],
 752                     self.environment[u"paths"]
 753                 )
 754             except KeyError:
 755                 pass
 756
 757             try:
 758                 element[u"input-file"] = self._replace_tags(
 759                     element[u"input-file"],
 760                     self.environment[u"paths"]
 761                 )
 762             except KeyError:
 763                 pass
 764
 765             try:
 766                 element[u"output-file-links"] = self._replace_tags(
 767                     element[u"output-file-links"],
 768                     self.environment[u"paths"]
 769                 )
 770             except KeyError:
 771                 pass
 772
 773             # Add data sets to the elements:
 774             if isinstance(element.get(u"data", None), str):
 775                 data_set = element[u"data"]
 776                 try:
 777                     element[u"data"] = self.data_sets[data_set]
 778                 except KeyError:
 779                     raise PresentationError(
 780                         f"Data set {data_set} is not defined."
 781                     )
 782             elif isinstance(element.get(u"data", None), list):
 783                 new_list = list()
 784                 for item in element[u"data"]:
 785                     try:
 786                         new_list.append(self.data_sets[item])
 787                     except KeyError:
 788                         raise PresentationError(
 789                             f"Data set {item} is not defined."
 790                         )
 791                 element[u"data"] = new_list
 792
 793             # Parse elements:
 794             if element[u"type"] == u"table":
 795                 logging.info(f"  {count:3d} Processing a table ...")
 796                 self._parse_elements_tables(element)
 797                 count += 1
 798             elif element[u"type"] == u"plot":
 799                 logging.info(f"  {count:3d} Processing a plot ...")
 800                 self._parse_elements_plots(element)
 801                 count += 1
 802             elif element[u"type"] == u"file":
 803                 logging.info(f"  {count:3d} Processing a file ...")
 804                 self._parse_elements_files(element)
 805                 count += 1
 806             elif element[u"type"] == u"cpta":
 807                 logging.info(
 808                     f"  {count:3d} Processing Continuous Performance Trending "
 809                     f"and Analysis ..."
 810                 )
 811                 self._parse_elements_cpta(element)
 812                 count += 1
 813
 814     def _prepare_input(self):
 815         """Use information from data sets and generate list of jobs and builds
 816         to download.
 817         """
 818
 819         logging.info(u"Parsing specification: INPUT")
 820
 821         idx = self._get_type_index(u"input")
 822         if idx is None:
 823             logging.info(u"Creating the list of inputs from data sets.")
 824             for data_set in self.data_sets.values():
 825                 if data_set == "data-sets":
 826                     continue
 827                 for job, builds in data_set.items():
 828                     for build in builds:
 829                         self.add_build(
 830                             job,
 831                             {
 832                                 u"build": build,
 833                                 u"status": None,
 834                                 u"file-name": None,
 835                                 u"source": None
 836                             }
 837                         )
 838         else:
 839             logging.info(u"Reading pre-defined inputs.")
 840             for job, builds in self._cfg_yaml[idx][u"builds"].items():
 841                 for build in builds:
 842                     self.add_build(
 843                         job,
 844                         {
 845                             u"build": build,
 846                             u"status": None,
 847                             u"file-name": None,
 848                             u"source": None
 849                         }
 850                     )
 851
 852         if self.environment[u"reverse-input"]:
 853             for builds in self.input.values():
 854                 builds.sort(key=lambda k: k[u"build"], reverse=True)
 855
 856     def read_specification(self):
 857         """Parse specification in the specification YAML files.
 858
 859         :raises: PresentationError if an error occurred while parsing the
 860             specification file.
 861         """
 862
 863         # It always starts with environment.yaml file, it must be present.
 864         spec_file = join(self._cfg_dir, u"environment.yaml")
 865         logging.info(f"Reading {spec_file}")
 866         if not exists(spec_file):
 867             raise PresentationError(f"The file {spec_file} does not exist.")
 868
 869         with open(spec_file, u"r") as file_read:
 870             try:
 871                 self._cfg_yaml = load(file_read, Loader=FullLoader)
 872             except YAMLError as err:
 873                 raise PresentationError(
 874                     f"An error occurred while parsing the specification file "
 875                     f"{spec_file}",
 876                     details=repr(err)
 877                 )
 878
 879         # Load the other specification files specified in the environment.yaml
 880         idx = self._get_type_index(u"environment")
 881         if idx is None:
 882             raise PresentationError(
 883                 f"No environment defined in the file {spec_file}"
 884             )
 885         for spec_file in self._cfg_yaml[idx].get(u"spec-files", tuple()):
 886             logging.info(f"Reading {spec_file}")
 887             if not exists(spec_file):
 888                 raise PresentationError(f"The file {spec_file} does not exist.")
 889             spec = None
 890             with open(spec_file, u"r") as file_read:
 891                 try:
 892                     spec = load(file_read, Loader=FullLoader)
 893                 except YAMLError as err:
 894                     raise PresentationError(
 895                         f"An error occurred while parsing the specification "
 896                         f"file {spec_file}",
 897                         details=repr(err)
 898                     )
 899             if spec:
 900                 self._cfg_yaml.extend(spec)
 901
 902         self._parse_env()
 903         self._parse_layouts()
 904         self._parse_data_sets()
 905         self._parse_output()
 906         self._parse_static()
 907         self._parse_elements()
 908         self._prepare_input()
 909
 910         logging.debug(f"Specification: \n{pformat(self.specification)}")