PAL: Implement data sets for specification file.
[csit.git] / resources / tools / presentation / specification_parser.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Specification
15
16 Parsing of the specification YAML file.
17 """
18
19
20 import logging
21 from yaml import load, YAMLError
22 from pprint import pformat
23
24 from errors import PresentationError
25
26
27 class Specification(object):
28     """Specification of Presentation and analytics layer.
29
30     - based on specification specified in the specification YAML file
31     - presentation and analytics layer is model driven
32     """
33
34     # Tags are used in specification YAML file and replaced while the file is
35     # parsed.
36     TAG_OPENER = "{"
37     TAG_CLOSER = "}"
38
39     def __init__(self, cfg_file):
40         """Initialization.
41
42         :param cfg_file: File handler for the specification YAML file.
43         :type cfg_file: BinaryIO
44         """
45         self._cfg_file = cfg_file
46         self._cfg_yaml = None
47
48         self._specification = {"environment": dict(),
49                                "configuration": dict(),
50                                "debug": dict(),
51                                "static": dict(),
52                                "input": dict(),
53                                "output": dict(),
54                                "tables": list(),
55                                "plots": list(),
56                                "files": list()}
57
58     @property
59     def specification(self):
60         """Getter - specification.
61
62         :returns: Specification.
63         :rtype: dict
64         """
65         return self._specification
66
67     @property
68     def environment(self):
69         """Getter - environment.
70
71         :returns: Environment specification.
72         :rtype: dict
73         """
74         return self._specification["environment"]
75
76     @property
77     def configuration(self):
78         """Getter - configuration.
79
80         :returns: Configuration of PAL.
81         :rtype: dict
82         """
83         return self._specification["configuration"]
84
85     @property
86     def static(self):
87         """Getter - static content.
88
89         :returns: Static content specification.
90         :rtype: dict
91         """
92         return self._specification["static"]
93
94     @property
95     def debug(self):
96         """Getter - debug
97
98         :returns: Debug specification
99         :rtype: dict
100         """
101         return self._specification["debug"]
102
103     @property
104     def is_debug(self):
105         """Getter - debug mode
106
107         :returns: True if debug mode is on, otherwise False.
108         :rtype: bool
109         """
110
111         try:
112             if self.environment["configuration"]["CFG[DEBUG]"] == 1:
113                 return True
114             else:
115                 return False
116         except KeyError:
117             return False
118
119     @property
120     def input(self):
121         """Getter - specification - inputs.
122         - jobs and builds.
123
124         :returns: Inputs.
125         :rtype: dict
126         """
127         return self._specification["input"]
128
129     @property
130     def builds(self):
131         """Getter - builds defined in specification.
132
133         :returns: Builds defined in the specification.
134         :rtype: dict
135         """
136         return self.input["builds"]
137
138     @property
139     def output(self):
140         """Getter - specification - output formats and versions to be generated.
141         - formats: html, pdf
142         - versions: full, ...
143
144         :returns: Outputs to be generated.
145         :rtype: dict
146         """
147         return self._specification["output"]
148
149     @property
150     def tables(self):
151         """Getter - tables to be generated.
152
153         :returns: List of specifications of tables to be generated.
154         :rtype: list
155         """
156         return self._specification["tables"]
157
158     @property
159     def plots(self):
160         """Getter - plots to be generated.
161
162         :returns: List of specifications of plots to be generated.
163         :rtype: list
164         """
165         return self._specification["plots"]
166
167     @property
168     def files(self):
169         """Getter - files to be generated.
170
171         :returns: List of specifications of files to be generated.
172         :rtype: list
173         """
174         return self._specification["files"]
175
176     def set_input_state(self, job, build_nr, state):
177         """Set the state of input
178
179         :param job:
180         :param build_nr:
181         :param state:
182         :return:
183         """
184
185         try:
186             for build in self._specification["input"]["builds"][job]:
187                 if build["build"] == build_nr:
188                     build["status"] = state
189                     break
190             else:
191                 raise PresentationError("Build '{}' is not defined for job '{}'"
192                                         " in specification file.".
193                                         format(build_nr, job))
194         except KeyError:
195             raise PresentationError("Job '{}' and build '{}' is not defined in "
196                                     "specification file.".format(job, build_nr))
197
198     def set_input_file_name(self, job, build_nr, file_name):
199         """Set the state of input
200
201         :param job:
202         :param build_nr:
203         :param file_name:
204         :return:
205         """
206
207         try:
208             for build in self._specification["input"]["builds"][job]:
209                 if build["build"] == build_nr:
210                     build["file-name"] = file_name
211                     break
212             else:
213                 raise PresentationError("Build '{}' is not defined for job '{}'"
214                                         " in specification file.".
215                                         format(build_nr, job))
216         except KeyError:
217             raise PresentationError("Job '{}' and build '{}' is not defined in "
218                                     "specification file.".format(job, build_nr))
219
220     def _get_type_index(self, item_type):
221         """Get index of item type (environment, input, output, ...) in
222         specification YAML file.
223
224         :param item_type: Item type: Top level items in specification YAML file,
225         e.g.: environment, input, output.
226         :type item_type: str
227         :returns: Index of the given item type.
228         :rtype: int
229         """
230
231         index = 0
232         for item in self._cfg_yaml:
233             if item["type"] == item_type:
234                 return index
235             index += 1
236         return None
237
238     def _find_tag(self, text):
239         """Find the first tag in the given text. The tag is enclosed by the
240         TAG_OPENER and TAG_CLOSER.
241
242         :param text: Text to be searched.
243         :type text: str
244         :returns: The tag, or None if not found.
245         :rtype: str
246         """
247         try:
248             start = text.index(self.TAG_OPENER)
249             end = text.index(self.TAG_CLOSER, start + 1) + 1
250             return text[start:end]
251         except ValueError:
252             return None
253
254     def _replace_tags(self, data, src_data=None):
255         """Replace tag(s) in the data by their values.
256
257         :param data: The data where the tags will be replaced by their values.
258         :param src_data: Data where the tags are defined. It is dictionary where
259         the key is the tag and the value is the tag value. If not given, 'data'
260         is used instead.
261         :type data: str or dict
262         :type src_data: dict
263         :returns: Data with the tags replaced.
264         :rtype: str or dict
265         :raises: PresentationError if it is not possible to replace the tag or
266         the data is not the supported data type (str, dict).
267         """
268
269         if src_data is None:
270             src_data = data
271
272         if isinstance(data, str):
273             tag = self._find_tag(data)
274             if tag is not None:
275                 data = data.replace(tag, src_data[tag[1:-1]])
276
277         elif isinstance(data, dict):
278             counter = 0
279             for key, value in data.items():
280                 tag = self._find_tag(value)
281                 if tag is not None:
282                     try:
283                         data[key] = value.replace(tag, src_data[tag[1:-1]])
284                         counter += 1
285                     except KeyError:
286                         raise PresentationError("Not possible to replace the "
287                                                 "tag '{}'".format(tag))
288             if counter:
289                 self._replace_tags(data, src_data)
290         else:
291             raise PresentationError("Replace tags: Not supported data type.")
292
293         return data
294
295     def _parse_env(self):
296         """Parse environment specification in the specification YAML file.
297         """
298
299         logging.info("Parsing specification file: environment ...")
300
301         idx = self._get_type_index("environment")
302         if idx is None:
303             return None
304
305         try:
306             self._specification["environment"]["configuration"] = \
307                 self._cfg_yaml[idx]["configuration"]
308         except KeyError:
309             self._specification["environment"]["configuration"] = None
310
311         try:
312             self._specification["environment"]["paths"] = \
313                 self._replace_tags(self._cfg_yaml[idx]["paths"])
314         except KeyError:
315             self._specification["environment"]["paths"] = None
316
317         try:
318             self._specification["environment"]["urls"] = \
319                 self._replace_tags(self._cfg_yaml[idx]["urls"])
320         except KeyError:
321             self._specification["environment"]["urls"] = None
322
323         try:
324             self._specification["environment"]["make-dirs"] = \
325                 self._cfg_yaml[idx]["make-dirs"]
326         except KeyError:
327             self._specification["environment"]["make-dirs"] = None
328
329         try:
330             self._specification["environment"]["remove-dirs"] = \
331                 self._cfg_yaml[idx]["remove-dirs"]
332         except KeyError:
333             self._specification["environment"]["remove-dirs"] = None
334
335         try:
336             self._specification["environment"]["build-dirs"] = \
337                 self._cfg_yaml[idx]["build-dirs"]
338         except KeyError:
339             self._specification["environment"]["build-dirs"] = None
340
341         logging.info("Done.")
342
343     def _parse_configuration(self):
344         """Parse configuration of PAL in the specification YAML file.
345         """
346
347         logging.info("Parsing specification file: configuration ...")
348
349         idx = self._get_type_index("configuration")
350         if idx is None:
351             logging.warning("No configuration information in the specification "
352                             "file.")
353             return None
354
355         try:
356             self._specification["configuration"] = self._cfg_yaml[idx]
357         except KeyError:
358             raise PresentationError("No configuration defined.")
359
360         logging.info("Done.")
361
362     def _parse_debug(self):
363         """Parse debug specification in the specification YAML file.
364         """
365
366         if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
367             return None
368
369         logging.info("Parsing specification file: debug ...")
370
371         idx = self._get_type_index("debug")
372         if idx is None:
373             self.environment["configuration"]["CFG[DEBUG]"] = 0
374             return None
375
376         try:
377             for key, value in self._cfg_yaml[idx]["general"].items():
378                 self._specification["debug"][key] = value
379
380             self._specification["input"]["builds"] = dict()
381             for job, builds in self._cfg_yaml[idx]["builds"].items():
382                 if builds:
383                     self._specification["input"]["builds"][job] = list()
384                     for build in builds:
385                         self._specification["input"]["builds"][job].\
386                             append({"build": build["build"],
387                                     "status": "downloaded",
388                                     "file-name": self._replace_tags(
389                                         build["file"],
390                                         self.environment["paths"])})
391                 else:
392                     logging.warning("No build is defined for the job '{}'. "
393                                     "Trying to continue without it.".
394                                     format(job))
395
396         except KeyError:
397             raise PresentationError("No data to process.")
398
399     def _parse_input(self):
400         """Parse input specification in the specification YAML file.
401
402         :raises: PresentationError if there are no data to process.
403         """
404
405         logging.info("Parsing specification file: input ...")
406
407         idx = self._get_type_index("input")
408         if idx is None:
409             raise PresentationError("No data to process.")
410
411         try:
412             for key, value in self._cfg_yaml[idx]["general"].items():
413                 self._specification["input"][key] = value
414             self._specification["input"]["builds"] = dict()
415             for job, builds in self._cfg_yaml[idx]["builds"].items():
416                 if builds:
417                     self._specification["input"]["builds"][job] = list()
418                     for build in builds:
419                         self._specification["input"]["builds"][job].\
420                             append({"build": build, "status": None})
421                 else:
422                     logging.warning("No build is defined for the job '{}'. "
423                                     "Trying to continue without it.".
424                                     format(job))
425         except KeyError:
426             raise PresentationError("No data to process.")
427
428         logging.info("Done.")
429
430     def _parse_output(self):
431         """Parse output specification in the specification YAML file.
432
433         :raises: PresentationError if there is no output defined.
434         """
435
436         logging.info("Parsing specification file: output ...")
437
438         idx = self._get_type_index("output")
439         if idx is None:
440             raise PresentationError("No output defined.")
441
442         try:
443             self._specification["output"] = self._cfg_yaml[idx]["format"]
444         except KeyError:
445             raise PresentationError("No output defined.")
446
447         logging.info("Done.")
448
449     def _parse_static(self):
450         """Parse specification of the static content in the specification YAML
451         file.
452         """
453
454         logging.info("Parsing specification file: static content ...")
455
456         idx = self._get_type_index("static")
457         if idx is None:
458             logging.warning("No static content specified.")
459
460         for key, value in self._cfg_yaml[idx].items():
461             if isinstance(value, str):
462                 try:
463                     self._cfg_yaml[idx][key] = self._replace_tags(
464                         value, self._specification["environment"]["paths"])
465                 except KeyError:
466                     pass
467
468         self._specification["static"] = self._cfg_yaml[idx]
469
470         logging.info("Done.")
471
472     def _parse_elements(self):
473         """Parse elements (tables, plots) specification in the specification
474         YAML file.
475         """
476
477         logging.info("Parsing specification file: elements ...")
478
479         count = 1
480         for element in self._cfg_yaml:
481             try:
482                 element["output-file"] = self._replace_tags(
483                     element["output-file"],
484                     self._specification["environment"]["paths"])
485             except KeyError:
486                 pass
487
488             # add data sets to the elements:
489             if isinstance(element["data"], str):
490                 data_set = element["data"]
491                 try:
492                     element["data"] = self.configuration["data-sets"][data_set]
493                 except KeyError:
494                     raise PresentationError("Data set {0} is not defined in "
495                                             "the configuration section.".
496                                             format(data_set))
497
498             if element["type"] == "table":
499                 logging.info("  {:3d} Processing a table ...".format(count))
500                 try:
501                     element["template"] = self._replace_tags(
502                         element["template"],
503                         self._specification["environment"]["paths"])
504                 except KeyError:
505                     pass
506                 self._specification["tables"].append(element)
507                 count += 1
508
509             elif element["type"] == "plot":
510                 logging.info("  {:3d} Processing a plot ...".format(count))
511
512                 # Add layout to the plots:
513                 layout = element["layout"].get("layout", None)
514                 if layout is not None:
515                     element["layout"].pop("layout")
516                     try:
517                         for key, val in (self.configuration["plot-layouts"]
518                                          [layout]):
519                             element["layout"][key] = val
520                     except KeyError:
521                         raise PresentationError("Layout {0} is not defined in "
522                                                 "the configuration section.".
523                                                 format(layout))
524                 self._specification["plots"].append(element)
525                 count += 1
526
527             elif element["type"] == "file":
528                 logging.info("  {:3d} Processing a file ...".format(count))
529                 try:
530                     element["dir-tables"] = self._replace_tags(
531                         element["dir-tables"],
532                         self._specification["environment"]["paths"])
533                 except KeyError:
534                     pass
535                 self._specification["files"].append(element)
536                 count += 1
537
538         logging.info("Done.")
539
540     def read_specification(self):
541         """Parse specification in the specification YAML file.
542
543         :raises: PresentationError if an error occurred while parsing the
544         specification file.
545         """
546         try:
547             self._cfg_yaml = load(self._cfg_file)
548         except YAMLError as err:
549             raise PresentationError(msg="An error occurred while parsing the "
550                                         "specification file.",
551                                     details=str(err))
552
553         self._parse_env()
554         self._parse_configuration()
555         self._parse_debug()
556         if not self.debug:
557             self._parse_input()
558         self._parse_output()
559         self._parse_static()
560         self._parse_elements()
561         print(self.tables)
562         print(self.files)
563         print(self.plots)
564
565         logging.debug("Specification: \n{}".
566                       format(pformat(self._specification)))

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.