1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 Parsing of the specification YAML file.
21 from yaml import load, YAMLError
22 from pprint import pformat
24 from errors import PresentationError
27 class Specification(object):
28 """Specification of Presentation and analytics layer.
30 - based on specification specified in the specification YAML file
31 - presentation and analytics layer is model driven
34 # Tags are used in specification YAML file and replaced while the file is
39 def __init__(self, cfg_file):
42 :param cfg_file: File handler for the specification YAML file.
43 :type cfg_file: BinaryIO
45 self._cfg_file = cfg_file
48 self._specification = {"environment": dict(),
49 "configuration": dict(),
59 def specification(self):
60 """Getter - specification.
62 :returns: Specification.
65 return self._specification
68 def environment(self):
69 """Getter - environment.
71 :returns: Environment specification.
74 return self._specification["environment"]
77 def configuration(self):
78 """Getter - configuration.
80 :returns: Configuration of PAL.
83 return self._specification["configuration"]
87 """Getter - static content.
89 :returns: Static content specification.
92 return self._specification["static"]
98 :returns: Debug specification
101 return self._specification["debug"]
105 """Getter - debug mode
107 :returns: True if debug mode is on, otherwise False.
112 if self.environment["configuration"]["CFG[DEBUG]"] == 1:
121 """Getter - specification - inputs.
127 return self._specification["input"]
131 """Getter - builds defined in specification.
133 :returns: Builds defined in the specification.
136 return self.input["builds"]
140 """Getter - specification - output formats and versions to be generated.
142 - versions: full, ...
144 :returns: Outputs to be generated.
147 return self._specification["output"]
151 """Getter - tables to be generated.
153 :returns: List of specifications of tables to be generated.
156 return self._specification["tables"]
160 """Getter - plots to be generated.
162 :returns: List of specifications of plots to be generated.
165 return self._specification["plots"]
169 """Getter - files to be generated.
171 :returns: List of specifications of files to be generated.
174 return self._specification["files"]
176 def set_input_state(self, job, build_nr, state):
177 """Set the state of input
186 for build in self._specification["input"]["builds"][job]:
187 if build["build"] == build_nr:
188 build["status"] = state
191 raise PresentationError("Build '{}' is not defined for job '{}'"
192 " in specification file.".
193 format(build_nr, job))
195 raise PresentationError("Job '{}' and build '{}' is not defined in "
196 "specification file.".format(job, build_nr))
198 def set_input_file_name(self, job, build_nr, file_name):
199 """Set the state of input
208 for build in self._specification["input"]["builds"][job]:
209 if build["build"] == build_nr:
210 build["file-name"] = file_name
213 raise PresentationError("Build '{}' is not defined for job '{}'"
214 " in specification file.".
215 format(build_nr, job))
217 raise PresentationError("Job '{}' and build '{}' is not defined in "
218 "specification file.".format(job, build_nr))
220 def _get_type_index(self, item_type):
221 """Get index of item type (environment, input, output, ...) in
222 specification YAML file.
224 :param item_type: Item type: Top level items in specification YAML file,
225 e.g.: environment, input, output.
227 :returns: Index of the given item type.
232 for item in self._cfg_yaml:
233 if item["type"] == item_type:
238 def _find_tag(self, text):
239 """Find the first tag in the given text. The tag is enclosed by the
240 TAG_OPENER and TAG_CLOSER.
242 :param text: Text to be searched.
244 :returns: The tag, or None if not found.
248 start = text.index(self.TAG_OPENER)
249 end = text.index(self.TAG_CLOSER, start + 1) + 1
250 return text[start:end]
254 def _replace_tags(self, data, src_data=None):
255 """Replace tag(s) in the data by their values.
257 :param data: The data where the tags will be replaced by their values.
258 :param src_data: Data where the tags are defined. It is dictionary where
259 the key is the tag and the value is the tag value. If not given, 'data'
261 :type data: str or dict
263 :returns: Data with the tags replaced.
265 :raises: PresentationError if it is not possible to replace the tag or
266 the data is not the supported data type (str, dict).
272 if isinstance(data, str):
273 tag = self._find_tag(data)
275 data = data.replace(tag, src_data[tag[1:-1]])
277 elif isinstance(data, dict):
279 for key, value in data.items():
280 tag = self._find_tag(value)
283 data[key] = value.replace(tag, src_data[tag[1:-1]])
286 raise PresentationError("Not possible to replace the "
287 "tag '{}'".format(tag))
289 self._replace_tags(data, src_data)
291 raise PresentationError("Replace tags: Not supported data type.")
295 def _parse_env(self):
296 """Parse environment specification in the specification YAML file.
299 logging.info("Parsing specification file: environment ...")
301 idx = self._get_type_index("environment")
306 self._specification["environment"]["configuration"] = \
307 self._cfg_yaml[idx]["configuration"]
309 self._specification["environment"]["configuration"] = None
312 self._specification["environment"]["paths"] = \
313 self._replace_tags(self._cfg_yaml[idx]["paths"])
315 self._specification["environment"]["paths"] = None
318 self._specification["environment"]["urls"] = \
319 self._replace_tags(self._cfg_yaml[idx]["urls"])
321 self._specification["environment"]["urls"] = None
324 self._specification["environment"]["make-dirs"] = \
325 self._cfg_yaml[idx]["make-dirs"]
327 self._specification["environment"]["make-dirs"] = None
330 self._specification["environment"]["remove-dirs"] = \
331 self._cfg_yaml[idx]["remove-dirs"]
333 self._specification["environment"]["remove-dirs"] = None
336 self._specification["environment"]["build-dirs"] = \
337 self._cfg_yaml[idx]["build-dirs"]
339 self._specification["environment"]["build-dirs"] = None
341 logging.info("Done.")
343 def _parse_configuration(self):
344 """Parse configuration of PAL in the specification YAML file.
347 logging.info("Parsing specification file: configuration ...")
349 idx = self._get_type_index("configuration")
351 logging.warning("No configuration information in the specification "
356 self._specification["configuration"] = self._cfg_yaml[idx]
358 raise PresentationError("No configuration defined.")
360 logging.info("Done.")
362 def _parse_debug(self):
363 """Parse debug specification in the specification YAML file.
366 if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
369 logging.info("Parsing specification file: debug ...")
371 idx = self._get_type_index("debug")
373 self.environment["configuration"]["CFG[DEBUG]"] = 0
377 for key, value in self._cfg_yaml[idx]["general"].items():
378 self._specification["debug"][key] = value
380 self._specification["input"]["builds"] = dict()
381 for job, builds in self._cfg_yaml[idx]["builds"].items():
383 self._specification["input"]["builds"][job] = list()
385 self._specification["input"]["builds"][job].\
386 append({"build": build["build"],
387 "status": "downloaded",
388 "file-name": self._replace_tags(
390 self.environment["paths"])})
392 logging.warning("No build is defined for the job '{}'. "
393 "Trying to continue without it.".
397 raise PresentationError("No data to process.")
399 def _parse_input(self):
400 """Parse input specification in the specification YAML file.
402 :raises: PresentationError if there are no data to process.
405 logging.info("Parsing specification file: input ...")
407 idx = self._get_type_index("input")
409 raise PresentationError("No data to process.")
412 for key, value in self._cfg_yaml[idx]["general"].items():
413 self._specification["input"][key] = value
414 self._specification["input"]["builds"] = dict()
415 for job, builds in self._cfg_yaml[idx]["builds"].items():
417 self._specification["input"]["builds"][job] = list()
419 self._specification["input"]["builds"][job].\
420 append({"build": build, "status": None})
422 logging.warning("No build is defined for the job '{}'. "
423 "Trying to continue without it.".
426 raise PresentationError("No data to process.")
428 logging.info("Done.")
430 def _parse_output(self):
431 """Parse output specification in the specification YAML file.
433 :raises: PresentationError if there is no output defined.
436 logging.info("Parsing specification file: output ...")
438 idx = self._get_type_index("output")
440 raise PresentationError("No output defined.")
443 self._specification["output"] = self._cfg_yaml[idx]["format"]
445 raise PresentationError("No output defined.")
447 logging.info("Done.")
449 def _parse_static(self):
450 """Parse specification of the static content in the specification YAML
454 logging.info("Parsing specification file: static content ...")
456 idx = self._get_type_index("static")
458 logging.warning("No static content specified.")
460 for key, value in self._cfg_yaml[idx].items():
461 if isinstance(value, str):
463 self._cfg_yaml[idx][key] = self._replace_tags(
464 value, self._specification["environment"]["paths"])
468 self._specification["static"] = self._cfg_yaml[idx]
470 logging.info("Done.")
472 def _parse_elements(self):
473 """Parse elements (tables, plots) specification in the specification
477 logging.info("Parsing specification file: elements ...")
480 for element in self._cfg_yaml:
482 element["output-file"] = self._replace_tags(
483 element["output-file"],
484 self._specification["environment"]["paths"])
488 # add data sets to the elements:
489 if isinstance(element.get("data", None), str):
490 data_set = element["data"]
492 element["data"] = self.configuration["data-sets"][data_set]
494 raise PresentationError("Data set {0} is not defined in "
495 "the configuration section.".
498 if element["type"] == "table":
499 logging.info(" {:3d} Processing a table ...".format(count))
501 element["template"] = self._replace_tags(
503 self._specification["environment"]["paths"])
506 self._specification["tables"].append(element)
509 elif element["type"] == "plot":
510 logging.info(" {:3d} Processing a plot ...".format(count))
512 # Add layout to the plots:
513 layout = element["layout"].get("layout", None)
514 if layout is not None:
515 element["layout"].pop("layout")
517 for key, val in (self.configuration["plot-layouts"]
519 element["layout"][key] = val
521 raise PresentationError("Layout {0} is not defined in "
522 "the configuration section.".
524 self._specification["plots"].append(element)
527 elif element["type"] == "file":
528 logging.info(" {:3d} Processing a file ...".format(count))
530 element["dir-tables"] = self._replace_tags(
531 element["dir-tables"],
532 self._specification["environment"]["paths"])
535 self._specification["files"].append(element)
538 logging.info("Done.")
540 def read_specification(self):
541 """Parse specification in the specification YAML file.
543 :raises: PresentationError if an error occurred while parsing the
547 self._cfg_yaml = load(self._cfg_file)
548 except YAMLError as err:
549 raise PresentationError(msg="An error occurred while parsing the "
550 "specification file.",
554 self._parse_configuration()
560 self._parse_elements()
562 logging.debug("Specification: \n{}".
563 format(pformat(self._specification)))