ec663f99d9aa4ba69d358e6f1428163a2aa8b741
[csit.git] / resources / tools / presentation / specification_parser.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Specification
15
16 Parsing of the specification YAML file.
17 """
18
19
20 import logging
21 from yaml import load, YAMLError
22 from pprint import pformat
23
24 from errors import PresentationError
25
26
27 class Specification(object):
28     """Specification of Presentation and analytics layer.
29
30     - based on specification specified in the specification YAML file
31     - presentation and analytics layer is model driven
32     """
33
34     # Tags are used in specification YAML file and replaced while the file is
35     # parsed.
36     TAG_OPENER = "{"
37     TAG_CLOSER = "}"
38
39     def __init__(self, cfg_file):
40         """Initialization.
41
42         :param cfg_file: File handler for the specification YAML file.
43         :type cfg_file: BinaryIO
44         """
45         self._cfg_file = cfg_file
46         self._cfg_yaml = None
47
48         self._specification = {"environment": dict(),
49                                "debug": dict(),
50                                "static": dict(),
51                                "input": dict(),
52                                "output": dict(),
53                                "tables": list(),
54                                "plots": list(),
55                                "files": list()}
56
57     @property
58     def specification(self):
59         """Getter - specification.
60
61         :returns: Specification.
62         :rtype: dict
63         """
64         return self._specification
65
66     @property
67     def environment(self):
68         """Getter - environment.
69
70         :returns: Environment specification.
71         :rtype: dict
72         """
73         return self._specification["environment"]
74
75     @property
76     def static(self):
77         """Getter - static content.
78
79         :returns: Static content specification.
80         :rtype: dict
81         """
82         return self._specification["static"]
83
84     @property
85     def debug(self):
86         """Getter - debug
87
88         :returns: Debug specification
89         :rtype: dict
90         """
91         return self._specification["debug"]
92
93     @property
94     def is_debug(self):
95         """Getter - debug mode
96
97         :returns: True if debug mode is on, otherwise False.
98         :rtype: bool
99         """
100
101         try:
102             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
103                 return True
104             else:
105                 return False
106         except KeyError:
107             return False
108
109     @property
110     def input(self):
111         """Getter - specification - inputs.
112         - jobs and builds.
113
114         :returns: Inputs.
115         :rtype: dict
116         """
117         return self._specification["input"]
118
119     @property
120     def builds(self):
121         """Getter - builds defined in specification.
122
123         :returns: Builds defined in the specification.
124         :rtype: dict
125         """
126         return self.input["builds"]
127
128     @property
129     def output(self):
130         """Getter - specification - output formats and versions to be generated.
131         - formats: html, pdf
132         - versions: full, ...
133
134         :returns: Outputs to be generated.
135         :rtype: dict
136         """
137         return self._specification["output"]
138
139     @property
140     def tables(self):
141         """Getter - tables to be generated.
142
143         :returns: List of specifications of tables to be generated.
144         :rtype: list
145         """
146         return self._specification["tables"]
147
148     @property
149     def plots(self):
150         """Getter - plots to be generated.
151
152         :returns: List of specifications of plots to be generated.
153         :rtype: list
154         """
155         return self._specification["plots"]
156
157     @property
158     def files(self):
159         """Getter - files to be generated.
160
161         :returns: List of specifications of files to be generated.
162         :rtype: list
163         """
164         return self._specification["files"]
165
166     def set_input_state(self, job, build_nr, state):
167         """Set the state of input
168
169         :param job:
170         :param build_nr:
171         :param state:
172         :return:
173         """
174
175         try:
176             for build in self._specification["input"]["builds"][job]:
177                 if build["build"] == build_nr:
178                     build["status"] = state
179                     break
180             else:
181                 raise PresentationError("Build '{}' is not defined for job '{}'"
182                                         " in specification file.".
183                                         format(build_nr, job))
184         except KeyError:
185             raise PresentationError("Job '{}' and build '{}' is not defined in "
186                                     "specification file.".format(job, build_nr))
187
188     def set_input_file_name(self, job, build_nr, file_name):
189         """Set the state of input
190
191         :param job:
192         :param build_nr:
193         :param file_name:
194         :return:
195         """
196
197         try:
198             for build in self._specification["input"]["builds"][job]:
199                 if build["build"] == build_nr:
200                     build["file-name"] = file_name
201                     break
202             else:
203                 raise PresentationError("Build '{}' is not defined for job '{}'"
204                                         " in specification file.".
205                                         format(build_nr, job))
206         except KeyError:
207             raise PresentationError("Job '{}' and build '{}' is not defined in "
208                                     "specification file.".format(job, build_nr))
209
210     def _get_type_index(self, item_type):
211         """Get index of item type (environment, input, output, ...) in
212         specification YAML file.
213
214         :param item_type: Item type: Top level items in specification YAML file,
215         e.g.: environment, input, output.
216         :type item_type: str
217         :returns: Index of the given item type.
218         :rtype: int
219         """
220
221         index = 0
222         for item in self._cfg_yaml:
223             if item["type"] == item_type:
224                 return index
225             index += 1
226         return None
227
228     def _find_tag(self, text):
229         """Find the first tag in the given text. The tag is enclosed by the
230         TAG_OPENER and TAG_CLOSER.
231
232         :param text: Text to be searched.
233         :type text: str
234         :returns: The tag, or None if not found.
235         :rtype: str
236         """
237         try:
238             start = text.index(self.TAG_OPENER)
239             end = text.index(self.TAG_CLOSER, start + 1) + 1
240             return text[start:end]
241         except ValueError:
242             return None
243
244     def _replace_tags(self, data, src_data=None):
245         """Replace tag(s) in the data by their values.
246
247         :param data: The data where the tags will be replaced by their values.
248         :param src_data: Data where the tags are defined. It is dictionary where
249         the key is the tag and the value is the tag value. If not given, 'data'
250         is used instead.
251         :type data: str or dict
252         :type src_data: dict
253         :returns: Data with the tags replaced.
254         :rtype: str or dict
255         :raises: PresentationError if it is not possible to replace the tag or
256         the data is not the supported data type (str, dict).
257         """
258
259         if src_data is None:
260             src_data = data
261
262         if isinstance(data, str):
263             tag = self._find_tag(data)
264             if tag is not None:
265                 data = data.replace(tag, src_data[tag[1:-1]])
266
267         elif isinstance(data, dict):
268             counter = 0
269             for key, value in data.items():
270                 tag = self._find_tag(value)
271                 if tag is not None:
272                     try:
273                         data[key] = value.replace(tag, src_data[tag[1:-1]])
274                         counter += 1
275                     except KeyError:
276                         raise PresentationError("Not possible to replace the "
277                                                 "tag '{}'".format(tag))
278             if counter:
279                 self._replace_tags(data, src_data)
280         else:
281             raise PresentationError("Replace tags: Not supported data type.")
282
283         return data
284
285     def _parse_env(self):
286         """Parse environment specification in the specification YAML file.
287         """
288
289         logging.info("Parsing specification file: environment ...")
290
291         idx = self._get_type_index("environment")
292         if idx is None:
293             return None
294
295         try:
296             self._specification["environment"]["configuration"] = \
297                 self._cfg_yaml[idx]["configuration"]
298         except KeyError:
299             self._specification["environment"]["configuration"] = None
300
301         try:
302             self._specification["environment"]["paths"] = \
303                 self._replace_tags(self._cfg_yaml[idx]["paths"])
304         except KeyError:
305             self._specification["environment"]["paths"] = None
306
307         try:
308             self._specification["environment"]["urls"] = \
309                 self._replace_tags(self._cfg_yaml[idx]["urls"])
310         except KeyError:
311             self._specification["environment"]["urls"] = None
312
313         try:
314             self._specification["environment"]["make-dirs"] = \
315                 self._cfg_yaml[idx]["make-dirs"]
316         except KeyError:
317             self._specification["environment"]["make-dirs"] = None
318
319         try:
320             self._specification["environment"]["remove-dirs"] = \
321                 self._cfg_yaml[idx]["remove-dirs"]
322         except KeyError:
323             self._specification["environment"]["remove-dirs"] = None
324
325         try:
326             self._specification["environment"]["build-dirs"] = \
327                 self._cfg_yaml[idx]["build-dirs"]
328         except KeyError:
329             self._specification["environment"]["build-dirs"] = None
330
331         logging.info("Done.")
332
333     def _parse_debug(self):
334         """Parse debug specification in the specification YAML file.
335         """
336
337         if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
338             return None
339
340         logging.info("Parsing specification file: debug ...")
341
342         idx = self._get_type_index("debug")
343         if idx is None:
344             self.environment["configuration"]["CFG[DEBUG]"] = 0
345             return None
346
347         try:
348             for key, value in self._cfg_yaml[idx]["general"].items():
349                 self._specification["debug"][key] = value
350
351             self._specification["input"]["builds"] = dict()
352             for job, builds in self._cfg_yaml[idx]["builds"].items():
353                 if builds:
354                     self._specification["input"]["builds"][job] = list()
355                     for build in builds:
356                         self._specification["input"]["builds"][job].\
357                             append({"build": build["build"],
358                                     "status": "downloaded",
359                                     "file-name": self._replace_tags(
360                                         build["file"],
361                                         self.environment["paths"])})
362                 else:
363                     logging.warning("No build is defined for the job '{}'. "
364                                     "Trying to continue without it.".
365                                     format(job))
366
367         except KeyError:
368             raise PresentationError("No data to process.")
369
370     def _parse_input(self):
371         """Parse input specification in the specification YAML file.
372
373         :raises: PresentationError if there are no data to process.
374         """
375
376         logging.info("Parsing specification file: input ...")
377
378         idx = self._get_type_index("input")
379         if idx is None:
380             raise PresentationError("No data to process.")
381
382         try:
383             for key, value in self._cfg_yaml[idx]["general"].items():
384                 self._specification["input"][key] = value
385             self._specification["input"]["builds"] = dict()
386             for job, builds in self._cfg_yaml[idx]["builds"].items():
387                 if builds:
388                     self._specification["input"]["builds"][job] = list()
389                     for build in builds:
390                         self._specification["input"]["builds"][job].\
391                             append({"build": build, "status": None})
392                 else:
393                     logging.warning("No build is defined for the job '{}'. "
394                                     "Trying to continue without it.".
395                                     format(job))
396         except KeyError:
397             raise PresentationError("No data to process.")
398
399         logging.info("Done.")
400
401     def _parse_output(self):
402         """Parse output specification in the specification YAML file.
403
404         :raises: PresentationError if there is no output defined.
405         """
406
407         logging.info("Parsing specification file: output ...")
408
409         idx = self._get_type_index("output")
410         if idx is None:
411             raise PresentationError("No output defined.")
412
413         try:
414             self._specification["output"] = self._cfg_yaml[idx]["format"]
415         except KeyError:
416             raise PresentationError("No output defined.")
417
418         logging.info("Done.")
419
420     def _parse_static(self):
421         """Parse specification of the static content in the specification YAML
422         file.
423         """
424
425         logging.info("Parsing specification file: static content ...")
426
427         idx = self._get_type_index("static")
428         if idx is None:
429             logging.warning("No static content specified.")
430
431         for key, value in self._cfg_yaml[idx].items():
432             if isinstance(value, str):
433                 try:
434                     self._cfg_yaml[idx][key] = self._replace_tags(
435                         value, self._specification["environment"]["paths"])
436                 except KeyError:
437                     pass
438
439         self._specification["static"] = self._cfg_yaml[idx]
440
441         logging.info("Done.")
442
443     def _parse_elements(self):
444         """Parse elements (tables, plots) specification in the specification
445         YAML file.
446         """
447
448         logging.info("Parsing specification file: elements ...")
449
450         count = 1
451         for element in self._cfg_yaml:
452             try:
453                 element["output-file"] = self._replace_tags(
454                     element["output-file"],
455                     self._specification["environment"]["paths"])
456             except KeyError:
457                 pass
458             if element["type"] == "table":
459                 logging.info("  {:3d} Processing a table ...".format(count))
460                 try:
461                     element["template"] = self._replace_tags(
462                         element["template"],
463                         self._specification["environment"]["paths"])
464                 except KeyError:
465                     pass
466                 self._specification["tables"].append(element)
467                 count += 1
468             elif element["type"] == "plot":
469                 logging.info("  {:3d} Processing a plot ...".format(count))
470                 self._specification["plots"].append(element)
471                 count += 1
472             elif element["type"] == "file":
473                 logging.info("  {:3d} Processing a file ...".format(count))
474                 try:
475                     element["dir-tables"] = self._replace_tags(
476                         element["dir-tables"],
477                         self._specification["environment"]["paths"])
478                 except KeyError:
479                     pass
480                 self._specification["files"].append(element)
481                 count += 1
482
483         logging.info("Done.")
484
485     def read_specification(self):
486         """Parse specification in the specification YAML file.
487
488         :raises: PresentationError if an error occurred while parsing the
489         specification file.
490         """
491         try:
492             self._cfg_yaml = load(self._cfg_file)
493         except YAMLError as err:
494             raise PresentationError(msg="An error occurred while parsing the "
495                                         "specification file.",
496                                     details=str(err))
497
498         self._parse_env()
499         self._parse_debug()
500         if not self.debug:
501             self._parse_input()
502         self._parse_output()
503         self._parse_static()
504         self._parse_elements()
505
506         logging.debug("Specification: \n{}".
507                       format(pformat(self._specification)))

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.