resources/tools/presentation/specification_parser.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Specification
  15
  16 Parsing of the specification YAML file.
  17 """
  18
  19
  20 from os.path import join, exists
  21
  22 import logging
  23 from pprint import pformat
  24
  25 from yaml import load, FullLoader, YAMLError
  26
  27 from pal_errors import PresentationError
  28 from pal_utils import (
  29     get_last_successful_build_nr, get_last_completed_build_number
  30 )
  31
  32
  33 class Specification:
  34     """Specification of Presentation and analytics layer.
  35
  36     - based on specification specified in the specification YAML files
  37     - presentation and analytics layer is model driven
  38     """
  39
  40     # Tags are used in specification YAML file and replaced while the file is
  41     # parsed.
  42     TAG_OPENER = u"{"
  43     TAG_CLOSER = u"}"
  44
  45     def __init__(self, cfg_dir):
  46         """Initialization.
  47
  48         :param cfg_dir: Directory with the specification files.
  49         :type cfg_dir: str
  50         """
  51         self._cfg_dir = cfg_dir
  52         self._cfg_yaml = None
  53
  54         self._specification = {
  55             u"environment": dict(),
  56             u"data_sets": dict(),
  57             u"layouts": dict(),
  58             u"static": dict(),
  59             u"input": dict(),
  60             u"output": dict(),
  61             u"tables": list(),
  62             u"plots": list(),
  63             u"files": list(),
  64             u"cpta": dict()
  65         }
  66
  67     @property
  68     def specification(self):
  69         """Getter - specification.
  70
  71         :returns: Specification.
  72         :rtype: dict
  73         """
  74         return self._specification
  75
  76     @property
  77     def environment(self):
  78         """Getter - environment.
  79
  80         :returns: Environment specification.
  81         :rtype: dict
  82         """
  83         return self._specification[u"environment"]
  84
  85     @property
  86     def data_sets(self):
  87         """Getter - data_sets.
  88
  89         :returns: Data sets.
  90         :rtype: dict
  91         """
  92         return self._specification[u"data_sets"]
  93
  94     @property
  95     def layouts(self):
  96         """Getter - layouts.
  97
  98         :returns: Layouts.
  99         :rtype: dict
 100         """
 101         return self._specification[u"layouts"]
 102
 103     @property
 104     def static(self):
 105         """Getter - static content.
 106
 107         :returns: Static content specification.
 108         :rtype: dict
 109         """
 110         return self._specification[u"static"]
 111
 112     @property
 113     def mapping(self):
 114         """Getter - Mapping.
 115
 116         :returns: Mapping of the old names of test cases to the new (actual)
 117             one.
 118         :rtype: dict
 119         """
 120         return self.environment[u"mapping"]
 121
 122     @property
 123     def ignore(self):
 124         """Getter - Ignore list.
 125
 126         :returns: List of ignored test cases.
 127         :rtype: list
 128         """
 129         return self.environment[u"ignore"]
 130
 131     @property
 132     def alerting(self):
 133         """Getter - Alerting.
 134
 135         # TODO
 136
 137         :returns: Specification of alerts.
 138         :rtype: dict
 139         """
 140         return self.environment[u"alerting"]
 141
 142     @property
 143     def input(self):
 144         """Getter - specification - inputs.
 145         - jobs and builds.
 146
 147         :returns: Inputs.
 148         :rtype: dict
 149         """
 150         return self._specification[u"input"]
 151
 152     @input.setter
 153     def input(self, new_value):
 154         """Setter - specification - inputs.
 155
 156         :param new_value: New value to be set.
 157         :type new_value: dict
 158         """
 159         self._specification[u"input"] = new_value
 160
 161     def add_build(self, job, build):
 162         """Add a build to the list of builds if it does not exist there.
 163
 164         :param job: The job which run the build.
 165         :param build: The build to be added.
 166         :type job: str
 167         :type build: dict
 168         """
 169         if self.input.get(job, None) is None:
 170             self.input[job] = list()
 171         for existing_build in self.input[job]:
 172             if existing_build[u"build"] == build[u"build"]:
 173                 break
 174         else:
 175             self.input[job].append(build)
 176
 177     @property
 178     def output(self):
 179         """Getter - specification - output formats and versions to be generated.
 180         - formats: html, pdf
 181         - versions: full, ...
 182
 183         :returns: Outputs to be generated.
 184         :rtype: dict
 185         """
 186         return self._specification[u"output"]
 187
 188     @property
 189     def tables(self):
 190         """Getter - tables to be generated.
 191
 192         :returns: List of specifications of tables to be generated.
 193         :rtype: list
 194         """
 195         return self._specification.get(u"tables", list())
 196
 197     @property
 198     def plots(self):
 199         """Getter - plots to be generated.
 200
 201         :returns: List of specifications of plots to be generated.
 202         :rtype: list
 203         """
 204         return self._specification.get(u"plots", list())
 205
 206     @property
 207     def files(self):
 208         """Getter - files to be generated.
 209
 210         :returns: List of specifications of files to be generated.
 211         :rtype: list
 212         """
 213         return self._specification.get(u"files", list())
 214
 215     @property
 216     def cpta(self):
 217         """Getter - Continuous Performance Trending and Analysis to be
 218         generated.
 219
 220         :returns: List of specifications of Continuous Performance Trending and
 221             Analysis to be generated.
 222         :rtype: list
 223         """
 224         return self._specification[u"cpta"]
 225
 226     def set_input_state(self, job, build_nr, state):
 227         """Set the state of the input.
 228
 229         :param job: Job name.
 230         :param build_nr: Build number.
 231         :param state: The new input state.
 232         :type job: str
 233         :type build_nr: int
 234         :type state: str
 235         :raises: PresentationError if wrong job and/or build is provided.
 236         """
 237
 238         try:
 239             for build in self.input[job]:
 240                 if build[u"build"] == build_nr:
 241                     build[u"status"] = state
 242                     break
 243             else:
 244                 raise PresentationError(
 245                     f"Build {build_nr} is not defined for job {job} in "
 246                     f"specification file."
 247                 )
 248         except KeyError:
 249             raise PresentationError(
 250                 f"Job {job} and build {build_nr} is not defined in "
 251                 f"specification file."
 252             )
 253
 254     def set_input_file_name(self, job, build_nr, file_name):
 255         """Set the file name for the input.
 256
 257         :param job: Job name.
 258         :param build_nr: Build number.
 259         :param file_name: The new file name.
 260         :type job: str
 261         :type build_nr: int
 262         :type file_name: str
 263         :raises: PresentationError if wrong job and/or build is provided.
 264         """
 265
 266         try:
 267             for build in self.input[job]:
 268                 if build[u"build"] == build_nr:
 269                     build[u"file-name"] = file_name
 270                     break
 271             else:
 272                 raise PresentationError(
 273                     f"Build {build_nr} is not defined for job {job} in "
 274                     f"specification file."
 275                 )
 276         except KeyError:
 277             raise PresentationError(
 278                 f"Job {job} and build {build_nr} is not defined in "
 279                 f"specification file."
 280             )
 281
 282     def _get_build_number(self, job, build_type):
 283         """Get the number of the job defined by its name:
 284          - lastSuccessfulBuild
 285          - lastCompletedBuild
 286
 287         :param job: Job name.
 288         :param build_type: Build type:
 289          - lastSuccessfulBuild
 290          - lastCompletedBuild
 291         :type job" str
 292         :raises PresentationError: If it is not possible to get the build
 293             number.
 294         :returns: The build number.
 295         :rtype: int
 296         """
 297
 298         # defined as a range <start, end>
 299         if build_type == u"lastSuccessfulBuild":
 300             # defined as a range <start, lastSuccessfulBuild>
 301             ret_code, build_nr, _ = get_last_successful_build_nr(
 302                 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
 303         elif build_type == u"lastCompletedBuild":
 304             # defined as a range <start, lastCompletedBuild>
 305             ret_code, build_nr, _ = get_last_completed_build_number(
 306                 self.environment[u"urls"][u"URL[JENKINS,CSIT]"], job)
 307         else:
 308             raise PresentationError(f"Not supported build type: {build_type}")
 309         if ret_code != 0:
 310             raise PresentationError(
 311                 f"Not possible to get the build number of {job}."
 312             )
 313         try:
 314             build_nr = int(build_nr)
 315             return build_nr
 316         except ValueError as err:
 317             raise PresentationError(
 318                 f"Not possible to get the build number of {job}. Reason:\n"
 319                 f"{repr(err)}"
 320             )
 321
 322     def _get_type_index(self, item_type):
 323         """Get index of item type (environment, input, output, ...) in
 324         specification YAML file.
 325
 326         :param item_type: Item type: Top level items in specification YAML file,
 327             e.g.: environment, input, output.
 328         :type item_type: str
 329         :returns: Index of the given item type.
 330         :rtype: int
 331         """
 332
 333         index = 0
 334         for item in self._cfg_yaml:
 335             if item[u"type"] == item_type:
 336                 return index
 337             index += 1
 338         return None
 339
 340     def _find_tag(self, text):
 341         """Find the first tag in the given text. The tag is enclosed by the
 342         TAG_OPENER and TAG_CLOSER.
 343
 344         :param text: Text to be searched.
 345         :type text: str
 346         :returns: The tag, or None if not found.
 347         :rtype: str
 348         """
 349         try:
 350             start = text.index(self.TAG_OPENER)
 351             end = text.index(self.TAG_CLOSER, start + 1) + 1
 352             return text[start:end]
 353         except ValueError:
 354             return None
 355
 356     def _replace_tags(self, data, src_data=None):
 357         """Replace tag(s) in the data by their values.
 358
 359         :param data: The data where the tags will be replaced by their values.
 360         :param src_data: Data where the tags are defined. It is dictionary where
 361             the key is the tag and the value is the tag value. If not given,
 362             'data' is used instead.
 363         :type data: str, list or dict
 364         :type src_data: dict
 365         :returns: Data with the tags replaced.
 366         :rtype: str, list or dict
 367         :raises: PresentationError if it is not possible to replace the tag or
 368             the data is not the supported data type (str, list or dict).
 369         """
 370
 371         if src_data is None:
 372             src_data = data
 373
 374         if isinstance(data, str):
 375             tag = self._find_tag(data)
 376             if tag is not None:
 377                 data = data.replace(tag, src_data[tag[1:-1]])
 378             return data
 379
 380         if isinstance(data, list):
 381             new_list = list()
 382             for item in data:
 383                 new_list.append(self._replace_tags(item, src_data))
 384             return new_list
 385
 386         if isinstance(data, dict):
 387             counter = 0
 388             for key, value in data.items():
 389                 tag = self._find_tag(value)
 390                 if tag is not None:
 391                     try:
 392                         data[key] = value.replace(tag, src_data[tag[1:-1]])
 393                         counter += 1
 394                     except KeyError:
 395                         raise PresentationError(
 396                             f"Not possible to replace the tag {tag}"
 397                         )
 398             if counter:
 399                 self._replace_tags(data, src_data)
 400             return data
 401
 402         raise PresentationError(u"Replace tags: Not supported data type.")
 403
 404     def _parse_env(self):
 405         """Parse environment specification in the specification YAML file.
 406         """
 407
 408         logging.info(u"Parsing specification: ENVIRONMENT")
 409
 410         idx = self._get_type_index(u"environment")
 411         if idx is None:
 412             return
 413
 414         self._specification[u"environment"][u"spec-files"] = \
 415             self._cfg_yaml[idx].get(u"spec-files", None)
 416
 417         try:
 418             self._specification[u"environment"][u"paths"] = \
 419                 self._replace_tags(self._cfg_yaml[idx][u"paths"])
 420         except KeyError:
 421             self._specification[u"environment"][u"paths"] = None
 422
 423         self._specification[u"environment"][u"data-sources"] = \
 424             self._cfg_yaml[idx].get(u"data-sources", tuple())
 425         # Add statistics:
 426         for source in self._specification[u"environment"][u"data-sources"]:
 427             source[u"successful-downloads"] = 0
 428
 429         self._specification[u"environment"][u"make-dirs"] = \
 430             self._cfg_yaml[idx].get(u"make-dirs", None)
 431
 432         self._specification[u"environment"][u"remove-dirs"] = \
 433             self._cfg_yaml[idx].get(u"remove-dirs", None)
 434
 435         self._specification[u"environment"][u"build-dirs"] = \
 436             self._cfg_yaml[idx].get(u"build-dirs", None)
 437
 438         self._specification[u"environment"][u"testbeds"] = \
 439             self._cfg_yaml[idx].get(u"testbeds", None)
 440
 441         self._specification[u"environment"][u"limits"] = \
 442             self._cfg_yaml[idx].get(u"limits", None)
 443
 444         self._specification[u"environment"][u"frequency"] = \
 445             self._cfg_yaml[idx].get(u"frequency", dict())
 446
 447         self._specification[u"environment"][u"urls"] = \
 448             self._cfg_yaml[idx].get(u"urls", None)
 449
 450         self._specification[u"environment"][u"archive-inputs"] = \
 451             self._cfg_yaml[idx].get(u"archive-inputs", False)
 452
 453         self._specification[u"environment"][u"reverse-input"] = \
 454             self._cfg_yaml[idx].get(u"reverse-input", False)
 455
 456         self._specification[u"environment"][u"time-period"] = \
 457             self._cfg_yaml[idx].get(u"time-period", None)
 458
 459         self._specification[u"environment"][u"alerting"] = \
 460             self._cfg_yaml[idx].get(u"alerting", None)
 461
 462         self._specification[u"environment"][u"mapping-file"] = \
 463             self._cfg_yaml[idx].get(u"mapping-file", None)
 464
 465         self._specification[u"environment"][u"ignore-list"] = \
 466             self._cfg_yaml[idx].get(u"ignore-list", None)
 467
 468         # Mapping table:
 469         self._load_mapping_table()
 470
 471         # Ignore list:
 472         self._load_ignore_list()
 473
 474     def _parse_layouts(self):
 475         """Parse layouts specification in the specification YAML file.
 476         """
 477
 478         logging.info(u"Parsing specification: LAYOUTS")
 479
 480         idx = self._get_type_index(u"layouts")
 481         if idx is None:
 482             return
 483
 484         try:
 485             self._specification[u"layouts"] = self._cfg_yaml[idx]
 486         except KeyError:
 487             raise PresentationError(u"No layouts defined.")
 488
 489     def _parse_data_sets(self):
 490         """Parse data sets specification in the specification YAML file.
 491         """
 492
 493         logging.info(u"Parsing specification: DATA SETS")
 494
 495         idx = self._get_type_index(u"data-sets")
 496         if idx is None:
 497             return
 498
 499         try:
 500             self._specification[u"data_sets"] = self._cfg_yaml[idx]
 501         except KeyError:
 502             raise PresentationError(u"No Data sets defined.")
 503
 504         # Replace ranges by lists
 505         for set_name, data_set in self.data_sets.items():
 506             if not isinstance(data_set, dict):
 507                 continue
 508             for job, builds in data_set.items():
 509                 if not builds:
 510                     continue
 511                 if isinstance(builds, dict):
 512                     build_end = builds.get(u"end", None)
 513                     max_builds = builds.get(u"max-builds", None)
 514                     reverse = builds.get(u"reverse", False)
 515                     try:
 516                         build_end = int(build_end)
 517                     except ValueError:
 518                         # defined as a range <start, build_type>
 519                         build_end = self._get_build_number(job, build_end)
 520                     builds = list(range(builds[u"start"], build_end + 1))
 521                     if max_builds and max_builds < len(builds):
 522                         builds = builds[-max_builds:]
 523                     if reverse:
 524                         builds.reverse()
 525                     self.data_sets[set_name][job] = builds
 526                 elif isinstance(builds, list):
 527                     for idx, item in enumerate(builds):
 528                         try:
 529                             builds[idx] = int(item)
 530                         except ValueError:
 531                             # defined as a range <build_type>
 532                             builds[idx] = self._get_build_number(job, item)
 533
 534         # Add sub-sets to sets (only one level):
 535         for set_name, data_set in self.data_sets.items():
 536             if isinstance(data_set, list):
 537                 new_set = dict()
 538                 for item in data_set:
 539                     try:
 540                         for key, val in self.data_sets[item].items():
 541                             new_set[key] = val
 542                     except KeyError:
 543                         raise PresentationError(
 544                             f"Data set {item} is not defined."
 545                         )
 546                 self.data_sets[set_name] = new_set
 547
 548     def _load_mapping_table(self):
 549         """Load a mapping table if it is specified. If not, use empty dict.
 550         """
 551
 552         mapping_file_name = self.environment.get(u"mapping-file", None)
 553         if mapping_file_name:
 554             try:
 555                 with open(mapping_file_name, u'r') as mfile:
 556                     mapping = load(mfile, Loader=FullLoader)
 557                     # Make sure everything is lowercase
 558                     self.environment[u"mapping"] = \
 559                         {key.lower(): val.lower() for key, val in
 560                          mapping.items()}
 561                 logging.debug(f"Loaded mapping table:\n{mapping}")
 562             except (YAMLError, IOError) as err:
 563                 raise PresentationError(
 564                     msg=f"An error occurred while parsing the mapping file "
 565                     f"{mapping_file_name}",
 566                     details=repr(err)
 567                 )
 568         else:
 569             self.environment[u"mapping"] = dict()
 570
 571     def _load_ignore_list(self):
 572         """Load an ignore list if it is specified. If not, use empty list.
 573         """
 574
 575         ignore_list_name = self.environment.get(u"ignore-list", None)
 576         if ignore_list_name:
 577             try:
 578                 with open(ignore_list_name, u'r') as ifile:
 579                     ignore = load(ifile, Loader=FullLoader)
 580                     # Make sure everything is lowercase
 581                     self.environment[u"ignore"] = \
 582                         [item.lower() for item in ignore]
 583                 logging.debug(f"Loaded ignore list:\n{ignore}")
 584             except (YAMLError, IOError) as err:
 585                 raise PresentationError(
 586                     msg=f"An error occurred while parsing the ignore list file "
 587                     f"{ignore_list_name}.",
 588                     details=repr(err)
 589                 )
 590         else:
 591             self.environment[u"ignore"] = list()
 592
 593     def _parse_output(self):
 594         """Parse output specification in the specification YAML file.
 595
 596         :raises: PresentationError if there is no output defined.
 597         """
 598
 599         logging.info(u"Parsing specification: OUTPUT")
 600
 601         idx = self._get_type_index(u"output")
 602         if idx is None:
 603             raise PresentationError(u"No output defined.")
 604
 605         try:
 606             self._specification[u"output"] = self._cfg_yaml[idx]
 607         except (KeyError, IndexError):
 608             raise PresentationError(u"No output defined.")
 609
 610     def _parse_static(self):
 611         """Parse specification of the static content in the specification YAML
 612         file.
 613         """
 614
 615         logging.info(u"Parsing specification: STATIC CONTENT")
 616
 617         idx = self._get_type_index(u"static")
 618         if idx is None:
 619             logging.warning(u"No static content specified.")
 620             self._specification[u"static"] = dict()
 621             return
 622
 623         for key, value in self._cfg_yaml[idx].items():
 624             if isinstance(value, str):
 625                 try:
 626                     self._cfg_yaml[idx][key] = self._replace_tags(
 627                         value, self._specification[u"environment"][u"paths"])
 628                 except KeyError:
 629                     pass
 630
 631         self._specification[u"static"] = self._cfg_yaml[idx]
 632
 633     def _parse_elements_tables(self, table):
 634         """Parse tables from the specification YAML file.
 635
 636         :param table: Table to be parsed from the specification file.
 637         :type table: dict
 638         :raises PresentationError: If wrong data set is used.
 639         """
 640
 641         try:
 642             table[u"template"] = self._replace_tags(
 643                 table[u"template"],
 644                 self._specification[u"environment"][u"paths"])
 645         except KeyError:
 646             pass
 647
 648         # Add data sets
 649         try:
 650             for item in (u"reference", u"compare"):
 651                 if table.get(item, None):
 652                     data_set = table[item].get(u"data", None)
 653                     if isinstance(data_set, str):
 654                         table[item][u"data"] = self.data_sets[data_set]
 655                     data_set = table[item].get(u"data-replacement", None)
 656                     if isinstance(data_set, str):
 657                         table[item][u"data-replacement"] = \
 658                             self.data_sets[data_set]
 659
 660             if table.get(u"columns", None):
 661                 for i in range(len(table[u"columns"])):
 662                     data_set = table[u"columns"][i].get(u"data-set", None)
 663                     if isinstance(data_set, str):
 664                         table[u"columns"][i][u"data-set"] = \
 665                             self.data_sets[data_set]
 666                     data_set = table[u"columns"][i].get(
 667                         u"data-replacement", None)
 668                     if isinstance(data_set, str):
 669                         table[u"columns"][i][u"data-replacement"] = \
 670                             self.data_sets[data_set]
 671
 672             if table.get(u"lines", None):
 673                 for i in range(len(table[u"lines"])):
 674                     data_set = table[u"lines"][i].get(u"data-set", None)
 675                     if isinstance(data_set, str):
 676                         table[u"lines"][i][u"data-set"] = \
 677                             self.data_sets[data_set]
 678
 679         except KeyError:
 680             raise PresentationError(
 681                 f"Wrong set '{data_set}' used in {table.get(u'title', u'')}."
 682             )
 683
 684         self._specification[u"tables"].append(table)
 685
 686     def _parse_elements_plots(self, plot):
 687         """Parse plots from the specification YAML file.
 688
 689         :param plot: Plot to be parsed from the specification file.
 690         :type plot: dict
 691         :raises PresentationError: If plot layout is not defined.
 692         """
 693
 694         # Add layout to the plots:
 695         layout = plot[u"layout"].get(u"layout", None)
 696         if layout is not None:
 697             plot[u"layout"].pop(u"layout")
 698             try:
 699                 for key, val in self.layouts[layout].items():
 700                     plot[u"layout"][key] = val
 701             except KeyError:
 702                 raise PresentationError(f"Layout {layout} is not defined.")
 703         self._specification[u"plots"].append(plot)
 704
 705     def _parse_elements_files(self, file):
 706         """Parse files from the specification YAML file.
 707
 708         :param file: File to be parsed from the specification file.
 709         :type file: dict
 710         """
 711
 712         try:
 713             file[u"dir-tables"] = self._replace_tags(
 714                 file[u"dir-tables"],
 715                 self._specification[u"environment"][u"paths"])
 716         except KeyError:
 717             pass
 718         self._specification[u"files"].append(file)
 719
 720     def _parse_elements_cpta(self, cpta):
 721         """Parse cpta from the specification YAML file.
 722
 723         :param cpta: cpta to be parsed from the specification file.
 724         :type cpta: dict
 725         :raises PresentationError: If wrong data set is used or if plot layout
 726             is not defined.
 727         """
 728
 729         for plot in cpta[u"plots"]:
 730             # Add layout to the plots:
 731             layout = plot.get(u"layout", None)
 732             if layout is not None:
 733                 try:
 734                     plot[u"layout"] = self.layouts[layout]
 735                 except KeyError:
 736                     raise PresentationError(f"Layout {layout} is not defined.")
 737             # Add data sets:
 738             if isinstance(plot.get(u"data", None), str):
 739                 data_set = plot[u"data"]
 740                 try:
 741                     plot[u"data"] = self.data_sets[data_set]
 742                 except KeyError:
 743                     raise PresentationError(
 744                         f"Data set {data_set} is not defined."
 745                     )
 746         self._specification[u"cpta"] = cpta
 747
 748     def _parse_elements(self):
 749         """Parse elements (tables, plots, ..) specification in the specification
 750         YAML file.
 751         """
 752
 753         logging.info(u"Parsing specification: ELEMENTS")
 754
 755         count = 1
 756         for element in self._cfg_yaml:
 757
 758             # Replace tags:
 759             try:
 760                 element[u"output-file"] = self._replace_tags(
 761                     element[u"output-file"],
 762                     self.environment[u"paths"]
 763                 )
 764             except KeyError:
 765                 pass
 766
 767             try:
 768                 element[u"input-file"] = self._replace_tags(
 769                     element[u"input-file"],
 770                     self.environment[u"paths"]
 771                 )
 772             except KeyError:
 773                 pass
 774
 775             try:
 776                 element[u"output-file-links"] = self._replace_tags(
 777                     element[u"output-file-links"],
 778                     self.environment[u"paths"]
 779                 )
 780             except KeyError:
 781                 pass
 782
 783             # Add data sets to the elements:
 784             if isinstance(element.get(u"data", None), str):
 785                 data_set = element[u"data"]
 786                 try:
 787                     element[u"data"] = self.data_sets[data_set]
 788                 except KeyError:
 789                     raise PresentationError(
 790                         f"Data set {data_set} is not defined."
 791                     )
 792             elif isinstance(element.get(u"data", None), list):
 793                 new_list = list()
 794                 for item in element[u"data"]:
 795                     try:
 796                         new_list.append(self.data_sets[item])
 797                     except KeyError:
 798                         raise PresentationError(
 799                             f"Data set {item} is not defined."
 800                         )
 801                 element[u"data"] = new_list
 802
 803             # Parse elements:
 804             if element[u"type"] == u"table":
 805                 logging.info(f"  {count:3d} Processing a table ...")
 806                 self._parse_elements_tables(element)
 807                 count += 1
 808             elif element[u"type"] == u"plot":
 809                 logging.info(f"  {count:3d} Processing a plot ...")
 810                 self._parse_elements_plots(element)
 811                 count += 1
 812             elif element[u"type"] == u"file":
 813                 logging.info(f"  {count:3d} Processing a file ...")
 814                 self._parse_elements_files(element)
 815                 count += 1
 816             elif element[u"type"] == u"cpta":
 817                 logging.info(
 818                     f"  {count:3d} Processing Continuous Performance Trending "
 819                     f"and Analysis ..."
 820                 )
 821                 self._parse_elements_cpta(element)
 822                 count += 1
 823
 824     def _prepare_input(self):
 825         """Use information from data sets and generate list of jobs and builds
 826         to download.
 827         """
 828
 829         logging.info(u"Parsing specification: INPUT")
 830
 831         idx = self._get_type_index(u"input")
 832         if idx is None:
 833             logging.info(u"Creating the list of inputs from data sets.")
 834             for data_set in self.data_sets.values():
 835                 if data_set == "data-sets":
 836                     continue
 837                 for job, builds in data_set.items():
 838                     for build in builds:
 839                         self.add_build(
 840                             job,
 841                             {
 842                                 u"build": build,
 843                                 u"status": None,
 844                                 u"file-name": None,
 845                                 u"source": None
 846                             }
 847                         )
 848         else:
 849             logging.info(u"Reading pre-defined inputs.")
 850             for job, builds in self._cfg_yaml[idx][u"builds"].items():
 851                 for build in builds:
 852                     self.add_build(
 853                         job,
 854                         {
 855                             u"build": build,
 856                             u"status": None,
 857                             u"file-name": None,
 858                             u"source": None
 859                         }
 860                     )
 861
 862         if self.environment[u"reverse-input"]:
 863             for builds in self.input.values():
 864                 builds.sort(key=lambda k: k[u"build"], reverse=True)
 865
 866     def read_specification(self):
 867         """Parse specification in the specification YAML files.
 868
 869         :raises: PresentationError if an error occurred while parsing the
 870             specification file.
 871         """
 872
 873         # It always starts with environment.yaml file, it must be present.
 874         spec_file = join(self._cfg_dir, u"environment.yaml")
 875         logging.info(f"Reading {spec_file}")
 876         if not exists(spec_file):
 877             raise PresentationError(f"The file {spec_file} does not exist.")
 878
 879         with open(spec_file, u"r") as file_read:
 880             try:
 881                 self._cfg_yaml = load(file_read, Loader=FullLoader)
 882             except YAMLError as err:
 883                 raise PresentationError(
 884                     f"An error occurred while parsing the specification file "
 885                     f"{spec_file}",
 886                     details=repr(err)
 887                 )
 888
 889         # Load the other specification files specified in the environment.yaml
 890         idx = self._get_type_index(u"environment")
 891         if idx is None:
 892             raise PresentationError(
 893                 f"No environment defined in the file {spec_file}"
 894             )
 895         for spec_file in self._cfg_yaml[idx].get(u"spec-files", tuple()):
 896             logging.info(f"Reading {spec_file}")
 897             if not exists(spec_file):
 898                 raise PresentationError(f"The file {spec_file} does not exist.")
 899             spec = None
 900             with open(spec_file, u"r") as file_read:
 901                 try:
 902                     spec = load(file_read, Loader=FullLoader)
 903                 except YAMLError as err:
 904                     raise PresentationError(
 905                         f"An error occurred while parsing the specification "
 906                         f"file {spec_file}",
 907                         details=repr(err)
 908                     )
 909             if spec:
 910                 self._cfg_yaml.extend(spec)
 911
 912         self._parse_env()
 913         self._parse_layouts()
 914         self._parse_data_sets()
 915         self._parse_output()
 916         self._parse_static()
 917         self._parse_elements()
 918         self._prepare_input()
 919
 920         logging.debug(f"Specification: \n{pformat(self.specification)}")