Report: Detect outliers for the comparative tables
[csit.git] / resources / tools / presentation / specification_parser.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Specification
15
16 Parsing of the specification YAML file.
17 """
18
19
20 import logging
21 from yaml import load, YAMLError
22 from pprint import pformat
23
24 from errors import PresentationError
25 from utils import get_last_successful_build_number
26 from utils import get_last_completed_build_number
27
28
29 class Specification(object):
30     """Specification of Presentation and analytics layer.
31
32     - based on specification specified in the specification YAML file
33     - presentation and analytics layer is model driven
34     """
35
36     # Tags are used in specification YAML file and replaced while the file is
37     # parsed.
38     TAG_OPENER = "{"
39     TAG_CLOSER = "}"
40
41     def __init__(self, cfg_file):
42         """Initialization.
43
44         :param cfg_file: File handler for the specification YAML file.
45         :type cfg_file: BinaryIO
46         """
47         self._cfg_file = cfg_file
48         self._cfg_yaml = None
49
50         self._specification = {"environment": dict(),
51                                "configuration": dict(),
52                                "debug": dict(),
53                                "static": dict(),
54                                "input": dict(),
55                                "output": dict(),
56                                "tables": list(),
57                                "plots": list(),
58                                "files": list(),
59                                "cpta": dict()}
60
61     @property
62     def specification(self):
63         """Getter - specification.
64
65         :returns: Specification.
66         :rtype: dict
67         """
68         return self._specification
69
70     @property
71     def environment(self):
72         """Getter - environment.
73
74         :returns: Environment specification.
75         :rtype: dict
76         """
77         return self._specification["environment"]
78
79     @property
80     def configuration(self):
81         """Getter - configuration.
82
83         :returns: Configuration of PAL.
84         :rtype: dict
85         """
86         return self._specification["configuration"]
87
88     @property
89     def static(self):
90         """Getter - static content.
91
92         :returns: Static content specification.
93         :rtype: dict
94         """
95         return self._specification["static"]
96
97     @property
98     def debug(self):
99         """Getter - debug
100
101         :returns: Debug specification
102         :rtype: dict
103         """
104         return self._specification["debug"]
105
106     @property
107     def is_debug(self):
108         """Getter - debug mode
109
110         :returns: True if debug mode is on, otherwise False.
111         :rtype: bool
112         """
113
114         try:
115             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
116                 return True
117             else:
118                 return False
119         except KeyError:
120             return False
121
122     @property
123     def input(self):
124         """Getter - specification - inputs.
125         - jobs and builds.
126
127         :returns: Inputs.
128         :rtype: dict
129         """
130         return self._specification["input"]
131
132     @property
133     def builds(self):
134         """Getter - builds defined in specification.
135
136         :returns: Builds defined in the specification.
137         :rtype: dict
138         """
139         return self.input["builds"]
140
141     @property
142     def output(self):
143         """Getter - specification - output formats and versions to be generated.
144         - formats: html, pdf
145         - versions: full, ...
146
147         :returns: Outputs to be generated.
148         :rtype: dict
149         """
150         return self._specification["output"]
151
152     @property
153     def tables(self):
154         """Getter - tables to be generated.
155
156         :returns: List of specifications of tables to be generated.
157         :rtype: list
158         """
159         return self._specification["tables"]
160
161     @property
162     def plots(self):
163         """Getter - plots to be generated.
164
165         :returns: List of specifications of plots to be generated.
166         :rtype: list
167         """
168         return self._specification["plots"]
169
170     @property
171     def files(self):
172         """Getter - files to be generated.
173
174         :returns: List of specifications of files to be generated.
175         :rtype: list
176         """
177         return self._specification["files"]
178
179     @property
180     def cpta(self):
181         """Getter - Continuous Performance Trending and Analysis to be
182         generated.
183
184         :returns: List of specifications of Continuous Performance Trending and
185         Analysis to be generated.
186         :rtype: list
187         """
188         return self._specification["cpta"]
189
190     def set_input_state(self, job, build_nr, state):
191         """Set the state of input
192
193         :param job:
194         :param build_nr:
195         :param state:
196         :return:
197         """
198
199         try:
200             for build in self._specification["input"]["builds"][job]:
201                 if build["build"] == build_nr:
202                     build["status"] = state
203                     break
204             else:
205                 raise PresentationError("Build '{}' is not defined for job '{}'"
206                                         " in specification file.".
207                                         format(build_nr, job))
208         except KeyError:
209             raise PresentationError("Job '{}' and build '{}' is not defined in "
210                                     "specification file.".format(job, build_nr))
211
212     def set_input_file_name(self, job, build_nr, file_name):
213         """Set the state of input
214
215         :param job:
216         :param build_nr:
217         :param file_name:
218         :return:
219         """
220
221         try:
222             for build in self._specification["input"]["builds"][job]:
223                 if build["build"] == build_nr:
224                     build["file-name"] = file_name
225                     break
226             else:
227                 raise PresentationError("Build '{}' is not defined for job '{}'"
228                                         " in specification file.".
229                                         format(build_nr, job))
230         except KeyError:
231             raise PresentationError("Job '{}' and build '{}' is not defined in "
232                                     "specification file.".format(job, build_nr))
233
234     def _get_build_number(self, job, build_type):
235         """Get the number of the job defined by its name:
236          - lastSuccessfulBuild
237          - lastCompletedBuild
238
239         :param job: Job name.
240         :param build_type: Build type:
241          - lastSuccessfulBuild
242          - lastCompletedBuild
243         :type job" str
244         :raises PresentationError: If it is not possible to get the build
245         number.
246         :returns: The build number.
247         :rtype: int
248         """
249
250         # defined as a range <start, end>
251         if build_type == "lastSuccessfulBuild":
252             # defined as a range <start, lastSuccessfulBuild>
253             ret_code, build_nr, _ = get_last_successful_build_number(
254                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
255         elif build_type == "lastCompletedBuild":
256             # defined as a range <start, lastCompletedBuild>
257             ret_code, build_nr, _ = get_last_completed_build_number(
258                 self.environment["urls"]["URL[JENKINS,CSIT]"], job)
259         else:
260             raise PresentationError("Not supported build type: '{0}'".
261                                     format(build_type))
262         if ret_code != 0:
263             raise PresentationError("Not possible to get the number of the "
264                                     "build number.")
265         try:
266             build_nr = int(build_nr)
267             return build_nr
268         except ValueError as err:
269             raise PresentationError("Not possible to get the number of the "
270                                     "build number.\nReason: {0}".format(err))
271
272     def _get_type_index(self, item_type):
273         """Get index of item type (environment, input, output, ...) in
274         specification YAML file.
275
276         :param item_type: Item type: Top level items in specification YAML file,
277         e.g.: environment, input, output.
278         :type item_type: str
279         :returns: Index of the given item type.
280         :rtype: int
281         """
282
283         index = 0
284         for item in self._cfg_yaml:
285             if item["type"] == item_type:
286                 return index
287             index += 1
288         return None
289
290     def _find_tag(self, text):
291         """Find the first tag in the given text. The tag is enclosed by the
292         TAG_OPENER and TAG_CLOSER.
293
294         :param text: Text to be searched.
295         :type text: str
296         :returns: The tag, or None if not found.
297         :rtype: str
298         """
299         try:
300             start = text.index(self.TAG_OPENER)
301             end = text.index(self.TAG_CLOSER, start + 1) + 1
302             return text[start:end]
303         except ValueError:
304             return None
305
306     def _replace_tags(self, data, src_data=None):
307         """Replace tag(s) in the data by their values.
308
309         :param data: The data where the tags will be replaced by their values.
310         :param src_data: Data where the tags are defined. It is dictionary where
311         the key is the tag and the value is the tag value. If not given, 'data'
312         is used instead.
313         :type data: str or dict
314         :type src_data: dict
315         :returns: Data with the tags replaced.
316         :rtype: str or dict
317         :raises: PresentationError if it is not possible to replace the tag or
318         the data is not the supported data type (str, dict).
319         """
320
321         if src_data is None:
322             src_data = data
323
324         if isinstance(data, str):
325             tag = self._find_tag(data)
326             if tag is not None:
327                 data = data.replace(tag, src_data[tag[1:-1]])
328
329         elif isinstance(data, dict):
330             counter = 0
331             for key, value in data.items():
332                 tag = self._find_tag(value)
333                 if tag is not None:
334                     try:
335                         data[key] = value.replace(tag, src_data[tag[1:-1]])
336                         counter += 1
337                     except KeyError:
338                         raise PresentationError("Not possible to replace the "
339                                                 "tag '{}'".format(tag))
340             if counter:
341                 self._replace_tags(data, src_data)
342         else:
343             raise PresentationError("Replace tags: Not supported data type.")
344
345         return data
346
347     def _parse_env(self):
348         """Parse environment specification in the specification YAML file.
349         """
350
351         logging.info("Parsing specification file: environment ...")
352
353         idx = self._get_type_index("environment")
354         if idx is None:
355             return None
356
357         try:
358             self._specification["environment"]["configuration"] = \
359                 self._cfg_yaml[idx]["configuration"]
360         except KeyError:
361             self._specification["environment"]["configuration"] = None
362
363         try:
364             self._specification["environment"]["paths"] = \
365                 self._replace_tags(self._cfg_yaml[idx]["paths"])
366         except KeyError:
367             self._specification["environment"]["paths"] = None
368
369         try:
370             self._specification["environment"]["urls"] = \
371                 self._replace_tags(self._cfg_yaml[idx]["urls"])
372         except KeyError:
373             self._specification["environment"]["urls"] = None
374
375         try:
376             self._specification["environment"]["make-dirs"] = \
377                 self._cfg_yaml[idx]["make-dirs"]
378         except KeyError:
379             self._specification["environment"]["make-dirs"] = None
380
381         try:
382             self._specification["environment"]["remove-dirs"] = \
383                 self._cfg_yaml[idx]["remove-dirs"]
384         except KeyError:
385             self._specification["environment"]["remove-dirs"] = None
386
387         try:
388             self._specification["environment"]["build-dirs"] = \
389                 self._cfg_yaml[idx]["build-dirs"]
390         except KeyError:
391             self._specification["environment"]["build-dirs"] = None
392
393         logging.info("Done.")
394
395     def _parse_configuration(self):
396         """Parse configuration of PAL in the specification YAML file.
397         """
398
399         logging.info("Parsing specification file: configuration ...")
400
401         idx = self._get_type_index("configuration")
402         if idx is None:
403             logging.warning("No configuration information in the specification "
404                             "file.")
405             return None
406
407         try:
408             self._specification["configuration"] = self._cfg_yaml[idx]
409
410         except KeyError:
411             raise PresentationError("No configuration defined.")
412
413         # Data sets: Replace ranges by lists
414         for set_name, data_set in self.configuration["data-sets"].items():
415             for job, builds in data_set.items():
416                 if builds:
417                     if isinstance(builds, dict):
418                         build_nr = builds.get("end", None)
419                         try:
420                             build_nr = int(build_nr)
421                         except ValueError:
422                             # defined as a range <start, build_type>
423                             build_nr = self._get_build_number(job, build_nr)
424                         builds = [x for x in range(builds["start"], build_nr+1)]
425                         self.configuration["data-sets"][set_name][job] = builds
426         logging.info("Done.")
427
428     def _parse_debug(self):
429         """Parse debug specification in the specification YAML file.
430         """
431
432         if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
433             return None
434
435         logging.info("Parsing specification file: debug ...")
436
437         idx = self._get_type_index("debug")
438         if idx is None:
439             self.environment["configuration"]["CFG[DEBUG]"] = 0
440             return None
441
442         try:
443             for key, value in self._cfg_yaml[idx]["general"].items():
444                 self._specification["debug"][key] = value
445
446             self._specification["input"]["builds"] = dict()
447             for job, builds in self._cfg_yaml[idx]["builds"].items():
448                 if builds:
449                     self._specification["input"]["builds"][job] = list()
450                     for build in builds:
451                         self._specification["input"]["builds"][job].\
452                             append({"build": build["build"],
453                                     "status": "downloaded",
454                                     "file-name": self._replace_tags(
455                                         build["file"],
456                                         self.environment["paths"])})
457                 else:
458                     logging.warning("No build is defined for the job '{}'. "
459                                     "Trying to continue without it.".
460                                     format(job))
461
462         except KeyError:
463             raise PresentationError("No data to process.")
464
465     def _parse_input(self):
466         """Parse input specification in the specification YAML file.
467
468         :raises: PresentationError if there are no data to process.
469         """
470
471         logging.info("Parsing specification file: input ...")
472
473         idx = self._get_type_index("input")
474         if idx is None:
475             raise PresentationError("No data to process.")
476
477         try:
478             for key, value in self._cfg_yaml[idx]["general"].items():
479                 self._specification["input"][key] = value
480             self._specification["input"]["builds"] = dict()
481
482             for job, builds in self._cfg_yaml[idx]["builds"].items():
483                 if builds:
484                     if isinstance(builds, dict):
485                         build_nr = builds.get("end", None)
486                         try:
487                             build_nr = int(build_nr)
488                         except ValueError:
489                             # defined as a range <start, build_type>
490                             build_nr = self._get_build_number(job, build_nr)
491                         builds = [x for x in range(builds["start"], build_nr+1)]
492                     self._specification["input"]["builds"][job] = list()
493                     for build in builds:
494                         self._specification["input"]["builds"][job]. \
495                             append({"build": build, "status": None})
496
497                 else:
498                     logging.warning("No build is defined for the job '{}'. "
499                                     "Trying to continue without it.".
500                                     format(job))
501         except KeyError:
502             raise PresentationError("No data to process.")
503
504         logging.info("Done.")
505
506     def _parse_output(self):
507         """Parse output specification in the specification YAML file.
508
509         :raises: PresentationError if there is no output defined.
510         """
511
512         logging.info("Parsing specification file: output ...")
513
514         idx = self._get_type_index("output")
515         if idx is None:
516             raise PresentationError("No output defined.")
517
518         try:
519             self._specification["output"] = self._cfg_yaml[idx]
520         except (KeyError, IndexError):
521             raise PresentationError("No output defined.")
522
523         logging.info("Done.")
524
525     def _parse_static(self):
526         """Parse specification of the static content in the specification YAML
527         file.
528         """
529
530         logging.info("Parsing specification file: static content ...")
531
532         idx = self._get_type_index("static")
533         if idx is None:
534             logging.warning("No static content specified.")
535
536         for key, value in self._cfg_yaml[idx].items():
537             if isinstance(value, str):
538                 try:
539                     self._cfg_yaml[idx][key] = self._replace_tags(
540                         value, self._specification["environment"]["paths"])
541                 except KeyError:
542                     pass
543
544         self._specification["static"] = self._cfg_yaml[idx]
545
546         logging.info("Done.")
547
548     def _parse_elements(self):
549         """Parse elements (tables, plots) specification in the specification
550         YAML file.
551         """
552
553         logging.info("Parsing specification file: elements ...")
554
555         count = 1
556         for element in self._cfg_yaml:
557             try:
558                 element["output-file"] = self._replace_tags(
559                     element["output-file"],
560                     self._specification["environment"]["paths"])
561             except KeyError:
562                 pass
563
564             # add data sets to the elements:
565             if isinstance(element.get("data", None), str):
566                 data_set = element["data"]
567                 try:
568                     element["data"] = self.configuration["data-sets"][data_set]
569                 except KeyError:
570                     raise PresentationError("Data set {0} is not defined in "
571                                             "the configuration section.".
572                                             format(data_set))
573
574             if element["type"] == "table":
575                 logging.info("  {:3d} Processing a table ...".format(count))
576                 try:
577                     element["template"] = self._replace_tags(
578                         element["template"],
579                         self._specification["environment"]["paths"])
580                 except KeyError:
581                     pass
582                 self._specification["tables"].append(element)
583                 count += 1
584
585             elif element["type"] == "plot":
586                 logging.info("  {:3d} Processing a plot ...".format(count))
587
588                 # Add layout to the plots:
589                 layout = element["layout"].get("layout", None)
590                 if layout is not None:
591                     element["layout"].pop("layout")
592                     try:
593                         for key, val in (self.configuration["plot-layouts"]
594                                          [layout].items()):
595                             element["layout"][key] = val
596                     except KeyError:
597                         raise PresentationError("Layout {0} is not defined in "
598                                                 "the configuration section.".
599                                                 format(layout))
600                 self._specification["plots"].append(element)
601                 count += 1
602
603             elif element["type"] == "file":
604                 logging.info("  {:3d} Processing a file ...".format(count))
605                 try:
606                     element["dir-tables"] = self._replace_tags(
607                         element["dir-tables"],
608                         self._specification["environment"]["paths"])
609                 except KeyError:
610                     pass
611                 self._specification["files"].append(element)
612                 count += 1
613
614             elif element["type"] == "cpta":
615                 logging.info("  {:3d} Processing Continuous Performance "
616                              "Trending and Analysis ...".format(count))
617
618                 for plot in element["plots"]:
619                     # Add layout to the plots:
620                     layout = plot.get("layout", None)
621                     if layout is not None:
622                         try:
623                             plot["layout"] = \
624                                 self.configuration["plot-layouts"][layout]
625                         except KeyError:
626                             raise PresentationError(
627                                 "Layout {0} is not defined in the "
628                                 "configuration section.".format(layout))
629                     # Add data sets:
630                     if isinstance(plot.get("data", None), str):
631                         data_set = plot["data"]
632                         try:
633                             plot["data"] = \
634                                 self.configuration["data-sets"][data_set]
635                         except KeyError:
636                             raise PresentationError(
637                                 "Data set {0} is not defined in "
638                                 "the configuration section.".
639                                 format(data_set))
640                 self._specification["cpta"] = element
641                 count += 1
642
643         logging.info("Done.")
644
645     def read_specification(self):
646         """Parse specification in the specification YAML file.
647
648         :raises: PresentationError if an error occurred while parsing the
649         specification file.
650         """
651         try:
652             self._cfg_yaml = load(self._cfg_file)
653         except YAMLError as err:
654             raise PresentationError(msg="An error occurred while parsing the "
655                                         "specification file.",
656                                     details=str(err))
657
658         self._parse_env()
659         self._parse_configuration()
660         self._parse_debug()
661         if not self.debug:
662             self._parse_input()
663         self._parse_output()
664         self._parse_static()
665         self._parse_elements()
666
667         logging.debug("Specification: \n{}".
668                       format(pformat(self._specification)))

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.