X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fspecification_parser.py;h=207507e3b65c39bf0c61b54efaf4b4baa6efcf98;hp=ec663f99d9aa4ba69d358e6f1428163a2aa8b741;hb=eb9c66ae07b3b3cf2146ac6b4a52e5ddd4424f44;hpb=eecad36d7d2275fa47fbcab40dbcf56108ab0a51 diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py index ec663f99d9..207507e3b6 100644 --- a/resources/tools/presentation/specification_parser.py +++ b/resources/tools/presentation/specification_parser.py @@ -22,6 +22,8 @@ from yaml import load, YAMLError from pprint import pformat from errors import PresentationError +from utils import get_last_successful_build_number +from utils import get_last_completed_build_number class Specification(object): @@ -46,13 +48,15 @@ class Specification(object): self._cfg_yaml = None self._specification = {"environment": dict(), + "configuration": dict(), "debug": dict(), "static": dict(), "input": dict(), "output": dict(), "tables": list(), "plots": list(), - "files": list()} + "files": list(), + "cpta": dict()} @property def specification(self): @@ -72,6 +76,15 @@ class Specification(object): """ return self._specification["environment"] + @property + def configuration(self): + """Getter - configuration. + + :returns: Configuration of PAL. + :rtype: dict + """ + return self._specification["configuration"] + @property def static(self): """Getter - static content. @@ -163,6 +176,17 @@ class Specification(object): """ return self._specification["files"] + @property + def cpta(self): + """Getter - Continuous Performance Trending and Analysis to be + generated. + + :returns: List of specifications of Continuous Performance Trending and + Analysis to be generated. + :rtype: list + """ + return self._specification["cpta"] + def set_input_state(self, job, build_nr, state): """Set the state of input @@ -207,6 +231,44 @@ class Specification(object): raise PresentationError("Job '{}' and build '{}' is not defined in " "specification file.".format(job, build_nr)) + def _get_build_number(self, job, build_type): + """Get the number of the job defined by its name: + - lastSuccessfulBuild + - lastCompletedBuild + + :param job: Job name. + :param build_type: Build type: + - lastSuccessfulBuild + - lastCompletedBuild + :type job" str + :raises PresentationError: If it is not possible to get the build + number. + :returns: The build number. + :rtype: int + """ + + # defined as a range + if build_type == "lastSuccessfulBuild": + # defined as a range + ret_code, build_nr, _ = get_last_successful_build_number( + self.environment["urls"]["URL[JENKINS,CSIT]"], job) + elif build_type == "lastCompletedBuild": + # defined as a range + ret_code, build_nr, _ = get_last_completed_build_number( + self.environment["urls"]["URL[JENKINS,CSIT]"], job) + else: + raise PresentationError("Not supported build type: '{0}'". + format(build_type)) + if ret_code != 0: + raise PresentationError("Not possible to get the number of the " + "build number.") + try: + build_nr = int(build_nr) + return build_nr + except ValueError as err: + raise PresentationError("Not possible to get the number of the " + "build number.\nReason: {0}".format(err)) + def _get_type_index(self, item_type): """Get index of item type (environment, input, output, ...) in specification YAML file. @@ -330,6 +392,39 @@ class Specification(object): logging.info("Done.") + def _parse_configuration(self): + """Parse configuration of PAL in the specification YAML file. + """ + + logging.info("Parsing specification file: configuration ...") + + idx = self._get_type_index("configuration") + if idx is None: + logging.warning("No configuration information in the specification " + "file.") + return None + + try: + self._specification["configuration"] = self._cfg_yaml[idx] + + except KeyError: + raise PresentationError("No configuration defined.") + + # Data sets: Replace ranges by lists + for set_name, data_set in self.configuration["data-sets"].items(): + for job, builds in data_set.items(): + if builds: + if isinstance(builds, dict): + build_nr = builds.get("end", None) + try: + build_nr = int(build_nr) + except ValueError: + # defined as a range + build_nr = self._get_build_number(job, build_nr) + builds = [x for x in range(builds["start"], build_nr+1)] + self.configuration["data-sets"][set_name][job] = builds + logging.info("Done.") + def _parse_debug(self): """Parse debug specification in the specification YAML file. """ @@ -383,12 +478,22 @@ class Specification(object): for key, value in self._cfg_yaml[idx]["general"].items(): self._specification["input"][key] = value self._specification["input"]["builds"] = dict() + for job, builds in self._cfg_yaml[idx]["builds"].items(): if builds: + if isinstance(builds, dict): + build_nr = builds.get("end", None) + try: + build_nr = int(build_nr) + except ValueError: + # defined as a range + build_nr = self._get_build_number(job, build_nr) + builds = [x for x in range(builds["start"], build_nr+1)] self._specification["input"]["builds"][job] = list() for build in builds: - self._specification["input"]["builds"][job].\ + self._specification["input"]["builds"][job]. \ append({"build": build, "status": None}) + else: logging.warning("No build is defined for the job '{}'. " "Trying to continue without it.". @@ -411,8 +516,8 @@ class Specification(object): raise PresentationError("No output defined.") try: - self._specification["output"] = self._cfg_yaml[idx]["format"] - except KeyError: + self._specification["output"] = self._cfg_yaml[idx] + except (KeyError, IndexError): raise PresentationError("No output defined.") logging.info("Done.") @@ -455,6 +560,17 @@ class Specification(object): self._specification["environment"]["paths"]) except KeyError: pass + + # add data sets to the elements: + if isinstance(element.get("data", None), str): + data_set = element["data"] + try: + element["data"] = self.configuration["data-sets"][data_set] + except KeyError: + raise PresentationError("Data set {0} is not defined in " + "the configuration section.". + format(data_set)) + if element["type"] == "table": logging.info(" {:3d} Processing a table ...".format(count)) try: @@ -465,10 +581,25 @@ class Specification(object): pass self._specification["tables"].append(element) count += 1 + elif element["type"] == "plot": logging.info(" {:3d} Processing a plot ...".format(count)) + + # Add layout to the plots: + layout = element["layout"].get("layout", None) + if layout is not None: + element["layout"].pop("layout") + try: + for key, val in (self.configuration["plot-layouts"] + [layout].items()): + element["layout"][key] = val + except KeyError: + raise PresentationError("Layout {0} is not defined in " + "the configuration section.". + format(layout)) self._specification["plots"].append(element) count += 1 + elif element["type"] == "file": logging.info(" {:3d} Processing a file ...".format(count)) try: @@ -480,6 +611,35 @@ class Specification(object): self._specification["files"].append(element) count += 1 + elif element["type"] == "cpta": + logging.info(" {:3d} Processing Continuous Performance " + "Trending and Analysis ...".format(count)) + + for plot in element["plots"]: + # Add layout to the plots: + layout = plot.get("layout", None) + if layout is not None: + try: + plot["layout"] = \ + self.configuration["plot-layouts"][layout] + except KeyError: + raise PresentationError( + "Layout {0} is not defined in the " + "configuration section.".format(layout)) + # Add data sets: + if isinstance(plot.get("data", None), str): + data_set = plot["data"] + try: + plot["data"] = \ + self.configuration["data-sets"][data_set] + except KeyError: + raise PresentationError( + "Data set {0} is not defined in " + "the configuration section.". + format(data_set)) + self._specification["cpta"] = element + count += 1 + logging.info("Done.") def read_specification(self): @@ -496,6 +656,7 @@ class Specification(object): details=str(err)) self._parse_env() + self._parse_configuration() self._parse_debug() if not self.debug: self._parse_input()