Trending: Alerts
[csit.git] / resources / tools / presentation / specification_parser.py
1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Specification
15
16 Parsing of the specification YAML file.
17 """
18
19
20 import logging
21 from yaml import load, YAMLError
22 from pprint import pformat
23
24 from errors import PresentationError
25 from utils import get_last_successful_build_number
26 from utils import get_last_completed_build_number
27
28
29 class Specification(object):
30     """Specification of Presentation and analytics layer.
31
32     - based on specification specified in the specification YAML file
33     - presentation and analytics layer is model driven
34     """
35
36     # Tags are used in specification YAML file and replaced while the file is
37     # parsed.
38     TAG_OPENER = "{"
39     TAG_CLOSER = "}"
40
41     def __init__(self, cfg_file):
42         """Initialization.
43
44         :param cfg_file: File handler for the specification YAML file.
45         :type cfg_file: BinaryIO
46         """
47         self._cfg_file = cfg_file
48         self._cfg_yaml = None
49
50         self._specification = {"environment": dict(),
51                                "configuration": dict(),
52                                "static": dict(),
53                                "input": dict(),
54                                "output": dict(),
55                                "tables": list(),
56                                "plots": list(),
57                                "files": list(),
58                                "cpta": dict()}
59
60     @property
61     def specification(self):
62         """Getter - specification.
63
64         :returns: Specification.
65         :rtype: dict
66         """
67         return self._specification
68
69     @property
70     def environment(self):
71         """Getter - environment.
72
73         :returns: Environment specification.
74         :rtype: dict
75         """
76         return self._specification["environment"]
77
78     @property
79     def configuration(self):
80         """Getter - configuration.
81
82         :returns: Configuration of PAL.
83         :rtype: dict
84         """
85         return self._specification["configuration"]
86
87     @property
88     def static(self):
89         """Getter - static content.
90
91         :returns: Static content specification.
92         :rtype: dict
93         """
94         return self._specification["static"]
95
96     @property
97     def mapping(self):
98         """Getter - Mapping.
99
100         :returns: Mapping of the old names of test cases to the new (actual)
101             one.
102         :rtype: dict
103         """
104         return self._specification["configuration"]["mapping"]
105
106     @property
107     def ignore(self):
108         """Getter - Ignore list.
109
110         :returns: List of ignored test cases.
111         :rtype: list
112         """
113         return self._specification["configuration"]["ignore"]
114
115     @property
116     def alerting(self):
117         """Getter - Alerting.
118
119         :returns: Specification of alerts.
120         :rtype: dict
121         """
122         return self._specification["configuration"]["alerting"]
123
124     @property
125     def input(self):
126         """Getter - specification - inputs.
127         - jobs and builds.
128
129         :returns: Inputs.
130         :rtype: dict
131         """
132         return self._specification["input"]
133
134     @property
135     def builds(self):
136         """Getter - builds defined in specification.
137
138         :returns: Builds defined in the specification.
139         :rtype: dict
140         """
141         return self.input["builds"]
142
143     @property
144     def output(self):
145         """Getter - specification - output formats and versions to be generated.
146         - formats: html, pdf
147         - versions: full, ...
148
149         :returns: Outputs to be generated.
150         :rtype: dict
151         """
152         return self._specification["output"]
153
154     @property
155     def tables(self):
156         """Getter - tables to be generated.
157
158         :returns: List of specifications of tables to be generated.
159         :rtype: list
160         """
161         return self._specification["tables"]
162
163     @property
164     def plots(self):
165         """Getter - plots to be generated.
166
167         :returns: List of specifications of plots to be generated.
168         :rtype: list
169         """
170         return self._specification["plots"]
171
172     @property
173     def files(self):
174         """Getter - files to be generated.
175
176         :returns: List of specifications of files to be generated.
177         :rtype: list
178         """
179         return self._specification["files"]
180
181     @property
182     def cpta(self):
183         """Getter - Continuous Performance Trending and Analysis to be
184         generated.
185
186         :returns: List of specifications of Continuous Performance Trending and
187         Analysis to be generated.
188         :rtype: list
189         """
190         return self._specification["cpta"]
191
192     def set_input_state(self, job, build_nr, state):
193         """Set the state of input
194
195         :param job:
196         :param build_nr:
197         :param state:
198         :return:
199         """
200
201         try:
202             for build in self._specification["input"]["builds"][job]:
203                 if build["build"] == build_nr:
204                     build["status"] = state
205                     break
206             else:
207                 raise PresentationError("Build '{}' is not defined for job '{}'"
208                                         " in specification file.".
209                                         format(build_nr, job))
210         except KeyError:
211             raise PresentationError("Job '{}' and build '{}' is not defined in "
212                                     "specification file.".format(job, build_nr))
213
214     def set_input_file_name(self, job, build_nr, file_name):
215         """Set the state of input
216
217         :param job:
218         :param build_nr:
219         :param file_name:
220         :return:
221         """
222
223         try:
224             for build in self._specification["input"]["builds"][job]:
225                 if build["build"] == build_nr:
226                     build["file-name"] = file_name
227                     break
228             else:
229                 raise PresentationError("Build '{}' is not defined for job '{}'"
230                                         " in specification file.".
231                                         format(build_nr, job))
232         except KeyError:
233             raise PresentationError("Job '{}' and build '{}' is not defined in "
234                                     "specification file.".format(job, build_nr))
235
236     def _get_build_number(self, job, build_type):
237         """Get the number of the job defined by its name:
238          - lastSuccessfulBuild
239          - lastCompletedBuild
240
241         :param job: Job name.
242         :param build_type: Build type:
243          - lastSuccessfulBuild
244          - lastCompletedBuild
245         :type job" str
246         :raises PresentationError: If it is not possible to get the build
247         number.
248         :returns: The build number.
249         :rtype: int
250         """
251
252         # defined as a range <start, end>
253         if build_type == "lastSuccessfulBuild":
254             # defined as a range <start, lastSuccessfulBuild>
255             ret_code, build_nr, _ = get_last_successful_build_number(
256                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
257         elif build_type == "lastCompletedBuild":
258             # defined as a range <start, lastCompletedBuild>
259             ret_code, build_nr, _ = get_last_completed_build_number(
260                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
261         else:
262             raise PresentationError("Not supported build type: '{0}'".
263                                     format(build_type))
264         if ret_code != 0:
265             raise PresentationError("Not possible to get the number of the "
266                                     "build number.")
267         try:
268             build_nr = int(build_nr)
269             return build_nr
270         except ValueError as err:
271             raise PresentationError("Not possible to get the number of the "
272                                     "build number.\nReason: {0}".format(err))
273
274     def _get_type_index(self, item_type):
275         """Get index of item type (environment, input, output, ...) in
276         specification YAML file.
277
278         :param item_type: Item type: Top level items in specification YAML file,
279         e.g.: environment, input, output.
280         :type item_type: str
281         :returns: Index of the given item type.
282         :rtype: int
283         """
284
285         index = 0
286         for item in self._cfg_yaml:
287             if item["type"] == item_type:
288                 return index
289             index += 1
290         return None
291
292     def _find_tag(self, text):
293         """Find the first tag in the given text. The tag is enclosed by the
294         TAG_OPENER and TAG_CLOSER.
295
296         :param text: Text to be searched.
297         :type text: str
298         :returns: The tag, or None if not found.
299         :rtype: str
300         """
301         try:
302             start = text.index(self.TAG_OPENER)
303             end = text.index(self.TAG_CLOSER, start + 1) + 1
304             return text[start:end]
305         except ValueError:
306             return None
307
308     def _replace_tags(self, data, src_data=None):
309         """Replace tag(s) in the data by their values.
310
311         :param data: The data where the tags will be replaced by their values.
312         :param src_data: Data where the tags are defined. It is dictionary where
313         the key is the tag and the value is the tag value. If not given, 'data'
314         is used instead.
315         :type data: str or dict
316         :type src_data: dict
317         :returns: Data with the tags replaced.
318         :rtype: str or dict
319         :raises: PresentationError if it is not possible to replace the tag or
320         the data is not the supported data type (str, dict).
321         """
322
323         if src_data is None:
324             src_data = data
325
326         if isinstance(data, str):
327             tag = self._find_tag(data)
328             if tag is not None:
329                 data = data.replace(tag, src_data[tag[1:-1]])
330
331         elif isinstance(data, dict):
332             counter = 0
333             for key, value in data.items():
334                 tag = self._find_tag(value)
335                 if tag is not None:
336                     try:
337                         data[key] = value.replace(tag, src_data[tag[1:-1]])
338                         counter += 1
339                     except KeyError:
340                         raise PresentationError("Not possible to replace the "
341                                                 "tag '{}'".format(tag))
342             if counter:
343                 self._replace_tags(data, src_data)
344         else:
345             raise PresentationError("Replace tags: Not supported data type.")
346
347         return data
348
349     def _parse_env(self):
350         """Parse environment specification in the specification YAML file.
351         """
352
353         logging.info("Parsing specification file: environment ...")
354
355         idx = self._get_type_index("environment")
356         if idx is None:
357             return None
358
359         try:
360             self._specification["environment"]["configuration"] = \
361                 self._cfg_yaml[idx]["configuration"]
362         except KeyError:
363             self._specification["environment"]["configuration"] = None
364
365         try:
366             self._specification["environment"]["paths"] = \
367                 self._replace_tags(self._cfg_yaml[idx]["paths"])
368         except KeyError:
369             self._specification["environment"]["paths"] = None
370
371         try:
372             self._specification["environment"]["urls"] = \
373                 self._cfg_yaml[idx]["urls"]
374         except KeyError:
375             self._specification["environment"]["urls"] = None
376
377         try:
378             self._specification["environment"]["make-dirs"] = \
379                 self._cfg_yaml[idx]["make-dirs"]
380         except KeyError:
381             self._specification["environment"]["make-dirs"] = None
382
383         try:
384             self._specification["environment"]["remove-dirs"] = \
385                 self._cfg_yaml[idx]["remove-dirs"]
386         except KeyError:
387             self._specification["environment"]["remove-dirs"] = None
388
389         try:
390             self._specification["environment"]["build-dirs"] = \
391                 self._cfg_yaml[idx]["build-dirs"]
392         except KeyError:
393             self._specification["environment"]["build-dirs"] = None
394
395         try:
396             self._specification["environment"]["testbeds"] = \
397                 self._cfg_yaml[idx]["testbeds"]
398         except KeyError:
399             self._specification["environment"]["testbeds"] = None
400
401         logging.info("Done.")
402
403     def _parse_configuration(self):
404         """Parse configuration of PAL in the specification YAML file.
405         """
406
407         logging.info("Parsing specification file: configuration ...")
408
409         idx = self._get_type_index("configuration")
410         if idx is None:
411             logging.warning("No configuration information in the specification "
412                             "file.")
413             return None
414
415         try:
416             self._specification["configuration"] = self._cfg_yaml[idx]
417
418         except KeyError:
419             raise PresentationError("No configuration defined.")
420
421         # Data sets: Replace ranges by lists
422         for set_name, data_set in self.configuration["data-sets"].items():
423             if not isinstance(data_set, dict):
424                 continue
425             for job, builds in data_set.items():
426                 if builds:
427                     if isinstance(builds, dict):
428                         build_end = builds.get("end", None)
429                         try:
430                             build_end = int(build_end)
431                         except ValueError:
432                             # defined as a range <start, build_type>
433                             build_end = self._get_build_number(job, build_end)
434                         builds = [x for x in range(builds["start"], build_end+1)
435                                   if x not in builds.get("skip", list())]
436                         self.configuration["data-sets"][set_name][job] = builds
437                     elif isinstance(builds, list):
438                         for idx, item in enumerate(builds):
439                             try:
440                                 builds[idx] = int(item)
441                             except ValueError:
442                                 # defined as a range <build_type>
443                                 builds[idx] = self._get_build_number(job, item)
444
445         # Data sets: add sub-sets to sets (only one level):
446         for set_name, data_set in self.configuration["data-sets"].items():
447             if isinstance(data_set, list):
448                 new_set = dict()
449                 for item in data_set:
450                     try:
451                         for key, val in self.configuration["data-sets"][item].\
452                                 items():
453                             new_set[key] = val
454                     except KeyError:
455                         raise PresentationError(
456                             "Data set {0} is not defined in "
457                             "the configuration section.".format(item))
458                 self.configuration["data-sets"][set_name] = new_set
459
460         # Mapping table:
461         mapping = None
462         mapping_file_name = self._specification["configuration"].\
463             get("mapping-file", None)
464         if mapping_file_name:
465             logging.debug("Mapping file: '{0}'".format(mapping_file_name))
466             try:
467                 with open(mapping_file_name, 'r') as mfile:
468                     mapping = load(mfile)
469                 logging.debug("Loaded mapping table:\n{0}".format(mapping))
470             except (YAMLError, IOError) as err:
471                 raise PresentationError(
472                     msg="An error occurred while parsing the mapping file "
473                         "'{0}'.".format(mapping_file_name),
474                     details=repr(err))
475         # Make sure everything is lowercase
476         if mapping:
477             self._specification["configuration"]["mapping"] = \
478                 {key.lower(): val.lower() for key, val in mapping.iteritems()}
479         else:
480             self._specification["configuration"]["mapping"] = dict()
481
482         # Ignore list:
483         ignore = None
484         ignore_list_name = self._specification["configuration"].\
485             get("ignore-list", None)
486         if ignore_list_name:
487             logging.debug("Ignore list file: '{0}'".format(ignore_list_name))
488             try:
489                 with open(ignore_list_name, 'r') as ifile:
490                     ignore = load(ifile)
491                 logging.debug("Loaded ignore list:\n{0}".format(ignore))
492             except (YAMLError, IOError) as err:
493                 raise PresentationError(
494                     msg="An error occurred while parsing the ignore list file "
495                         "'{0}'.".format(ignore_list_name),
496                     details=repr(err))
497         # Make sure everything is lowercase
498         if ignore:
499             self._specification["configuration"]["ignore"] = \
500                 [item.lower() for item in ignore]
501         else:
502             self._specification["configuration"]["ignore"] = list()
503
504         logging.info("Done.")
505
506     def _parse_input(self):
507         """Parse input specification in the specification YAML file.
508
509         :raises: PresentationError if there are no data to process.
510         """
511
512         logging.info("Parsing specification file: input ...")
513
514         idx = self._get_type_index("input")
515         if idx is None:
516             raise PresentationError("No data to process.")
517
518         try:
519             for key, value in self._cfg_yaml[idx]["general"].items():
520                 self._specification["input"][key] = value
521             self._specification["input"]["builds"] = dict()
522
523             for job, builds in self._cfg_yaml[idx]["builds"].items():
524                 if builds:
525                     if isinstance(builds, dict):
526                         build_end = builds.get("end", None)
527                         try:
528                             build_end = int(build_end)
529                         except ValueError:
530                             # defined as a range <start, build_type>
531                             build_end = self._get_build_number(job, build_end)
532                         builds = [x for x in range(builds["start"], build_end+1)
533                                   if x not in builds.get("skip", list())]
534                     self._specification["input"]["builds"][job] = list()
535                     for build in builds:
536                         self._specification["input"]["builds"][job]. \
537                             append({"build": build, "status": None})
538
539                 else:
540                     logging.warning("No build is defined for the job '{}'. "
541                                     "Trying to continue without it.".
542                                     format(job))
543         except KeyError:
544             raise PresentationError("No data to process.")
545
546         logging.info("Done.")
547
548     def _parse_output(self):
549         """Parse output specification in the specification YAML file.
550
551         :raises: PresentationError if there is no output defined.
552         """
553
554         logging.info("Parsing specification file: output ...")
555
556         idx = self._get_type_index("output")
557         if idx is None:
558             raise PresentationError("No output defined.")
559
560         try:
561             self._specification["output"] = self._cfg_yaml[idx]
562         except (KeyError, IndexError):
563             raise PresentationError("No output defined.")
564
565         logging.info("Done.")
566
567     def _parse_static(self):
568         """Parse specification of the static content in the specification YAML
569         file.
570         """
571
572         logging.info("Parsing specification file: static content ...")
573
574         idx = self._get_type_index("static")
575         if idx is None:
576             logging.warning("No static content specified.")
577
578         for key, value in self._cfg_yaml[idx].items():
579             if isinstance(value, str):
580                 try:
581                     self._cfg_yaml[idx][key] = self._replace_tags(
582                         value, self._specification["environment"]["paths"])
583                 except KeyError:
584                     pass
585
586         self._specification["static"] = self._cfg_yaml[idx]
587
588         logging.info("Done.")
589
590     def _parse_elements(self):
591         """Parse elements (tables, plots) specification in the specification
592         YAML file.
593         """
594
595         logging.info("Parsing specification file: elements ...")
596
597         count = 1
598         for element in self._cfg_yaml:
599             try:
600                 element["output-file"] = self._replace_tags(
601                     element["output-file"],
602                     self._specification["environment"]["paths"])
603             except KeyError:
604                 pass
605
606             try:
607                 element["input-file"] = self._replace_tags(
608                     element["input-file"],
609                     self._specification["environment"]["paths"])
610             except KeyError:
611                 pass
612
613             # add data sets to the elements:
614             if isinstance(element.get("data", None), str):
615                 data_set = element["data"]
616                 try:
617                     element["data"] = self.configuration["data-sets"][data_set]
618                 except KeyError:
619                     raise PresentationError("Data set {0} is not defined in "
620                                             "the configuration section.".
621                                             format(data_set))
622
623             if element["type"] == "table":
624                 logging.info("  {:3d} Processing a table ...".format(count))
625                 try:
626                     element["template"] = self._replace_tags(
627                         element["template"],
628                         self._specification["environment"]["paths"])
629                 except KeyError:
630                     pass
631
632                 # add data sets
633                 try:
634                     for item in ("reference", "compare"):
635                         if element.get(item, None):
636                             data_set = element[item].get("data", None)
637                             if isinstance(data_set, str):
638                                 element[item]["data"] = \
639                                     self.configuration["data-sets"][data_set]
640
641                     if element.get("history", None):
642                         for i in range(len(element["history"])):
643                             data_set = element["history"][i].get("data", None)
644                             if isinstance(data_set, str):
645                                 element["history"][i]["data"] = \
646                                     self.configuration["data-sets"][data_set]
647
648                 except KeyError:
649                     raise PresentationError("Wrong data set used in {0}.".
650                                             format(element.get("title", "")))
651
652                 self._specification["tables"].append(element)
653                 count += 1
654
655             elif element["type"] == "plot":
656                 logging.info("  {:3d} Processing a plot ...".format(count))
657
658                 # Add layout to the plots:
659                 layout = element["layout"].get("layout", None)
660                 if layout is not None:
661                     element["layout"].pop("layout")
662                     try:
663                         for key, val in (self.configuration["plot-layouts"]
664                                          [layout].items()):
665                             element["layout"][key] = val
666                     except KeyError:
667                         raise PresentationError("Layout {0} is not defined in "
668                                                 "the configuration section.".
669                                                 format(layout))
670                 self._specification["plots"].append(element)
671                 count += 1
672
673             elif element["type"] == "file":
674                 logging.info("  {:3d} Processing a file ...".format(count))
675                 try:
676                     element["dir-tables"] = self._replace_tags(
677                         element["dir-tables"],
678                         self._specification["environment"]["paths"])
679                 except KeyError:
680                     pass
681                 self._specification["files"].append(element)
682                 count += 1
683
684             elif element["type"] == "cpta":
685                 logging.info("  {:3d} Processing Continuous Performance "
686                              "Trending and Analysis ...".format(count))
687
688                 for plot in element["plots"]:
689                     # Add layout to the plots:
690                     layout = plot.get("layout", None)
691                     if layout is not None:
692                         try:
693                             plot["layout"] = \
694                                 self.configuration["plot-layouts"][layout]
695                         except KeyError:
696                             raise PresentationError(
697                                 "Layout {0} is not defined in the "
698                                 "configuration section.".format(layout))
699                     # Add data sets:
700                     if isinstance(plot.get("data", None), str):
701                         data_set = plot["data"]
702                         try:
703                             plot["data"] = \
704                                 self.configuration["data-sets"][data_set]
705                         except KeyError:
706                             raise PresentationError(
707                                 "Data set {0} is not defined in "
708                                 "the configuration section.".
709                                 format(data_set))
710                 self._specification["cpta"] = element
711                 count += 1
712
713         logging.info("Done.")
714
715     def read_specification(self):
716         """Parse specification in the specification YAML file.
717
718         :raises: PresentationError if an error occurred while parsing the
719         specification file.
720         """
721         try:
722             self._cfg_yaml = load(self._cfg_file)
723         except YAMLError as err:
724             raise PresentationError(msg="An error occurred while parsing the "
725                                         "specification file.",
726                                     details=str(err))
727
728         self._parse_env()
729         self._parse_configuration()
730         self._parse_input()
731         self._parse_output()
732         self._parse_static()
733         self._parse_elements()
734
735         logging.debug("Specification: \n{}".
736                       format(pformat(self._specification)))

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.