From: Tibor Frank Date: Wed, 10 Apr 2019 07:42:13 +0000 (+0200) Subject: Trending: Reduce input data X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=8d8c5c429ef242d3903f2ad3f4743fcac50d6253 Trending: Reduce input data Change-Id: Ia787036e4a717b9693df09fc75738369d6d39b57 Signed-off-by: Tibor Frank --- diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py index ce5d8035e9..b106d9589a 100644 --- a/resources/tools/presentation/generator_alerts.py +++ b/resources/tools/presentation/generator_alerts.py @@ -142,19 +142,8 @@ class Alerting(object): """ for alert, alert_data in self.alerts.iteritems(): - if alert_data["way"] == "email": - text, html = self._create_alert_message(alert_data) - conf = self.configs["email"] - self._send_email(server=conf["server"], - addr_from=conf["address-from"], - addr_to=conf["address-to"], - subject=alert_data["title"], - text=text, - html=html) - elif alert_data["way"] == "jenkins": + if alert_data["way"] == "jenkins": self._generate_email_body(alert_data) - # TODO: Remove when not needed - self._generate_files_for_jenkins(alert_data) else: raise AlertingError("Alert with way '{0}' is not implemented.". format(alert_data["way"])) @@ -253,9 +242,10 @@ class Alerting(object): directory = self.configs[alert["way"]]["output-dir"] failed_tests = OrderedDict() + file_path = "{0}/{1}.txt".format(directory, test_set) version = "" try: - with open("{0}/{1}.txt".format(directory, test_set), 'r') as f_txt: + with open(file_path, 'r') as f_txt: for idx, line in enumerate(f_txt): if idx == 0: build = line[:-1] @@ -281,8 +271,9 @@ class Alerting(object): failed_tests[name]["framesizes"].append(framesize) if cores not in failed_tests[name]["cores"]: failed_tests[name]["cores"].append(cores) - except IOError as err: - logging.error(repr(err)) + except IOError: + logging.error("No such file or directory: {file}". + format(file=file_path)) return None, None, None, None if sort: sorted_failed_tests = OrderedDict() @@ -312,6 +303,8 @@ class Alerting(object): build, version, nr, failed_tests = \ self._get_compressed_failed_tests(alert, test_set) if build is None: + text += "\n\nNo data for the test set '{set}'.\n".\ + format(set=test_set) continue text += ("\n\n{topo}-{arch}, " "{nr} tests failed, " @@ -363,24 +356,3 @@ class Alerting(object): except IOError: logging.error("Not possible to write the file '{0}.txt'.". format(file_name)) - - def _generate_files_for_jenkins(self, alert): - """Create the file which is used in the generated alert. - - # TODO: Remove when not needed. - - :param alert: Files are created for this alert. - :type alert: dict - """ - - config = self.configs[alert["way"]] - - zip_file = config.get("zip-output", None) - if zip_file: - logging.info("Writing the file '{0}/{1}' ...". - format(config["output-dir"], zip_file)) - execute_command("tar czvf {dir}/{zip} --directory={dir} " - "{input}.txt". - format(dir=config["output-dir"], - zip=zip_file, - input=config["output-file"])) diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index cdce5f98c0..6f9925fede 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -885,8 +885,14 @@ def table_last_failed_tests(table, input_data): for job, builds in table["data"].items(): for build in builds: build = str(build) + try: + version = input_data.metadata(job, build).get("version", "") + except KeyError: + logging.error("Data for {job}: {build} is not present.". + format(job=job, build=build)) + return tbl_list.append(build) - tbl_list.append(input_data.metadata(job, build).get("version", "")) + tbl_list.append(version) for tst_name, tst_data in data[job][build].iteritems(): if tst_data["status"] != "FAIL": continue diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index 9604da509a..cc6bb0145c 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -169,7 +169,7 @@ # 3n-hsw plot-performance-trending-all-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 487 + start: 501 end: "lastCompletedBuild" skip: - 593 @@ -185,7 +185,7 @@ plot-performance-trending-vpp-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 487 + start: 501 end: "lastCompletedBuild" skip: - 593 @@ -204,7 +204,7 @@ # 3n-skx plot-performance-trending-all-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 237 + start: 250 end: "lastCompletedBuild" skip: - 356 @@ -237,7 +237,7 @@ plot-performance-trending-vpp-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 237 + start: 250 end: "lastCompletedBuild" skip: - 356 @@ -273,7 +273,7 @@ # 2n-skx plot-performance-trending-all-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 237 + start: 250 end: "lastCompletedBuild" skip: - 347 @@ -291,13 +291,14 @@ - 375 - 380 - 381 + - 396 csit-dpdk-perf-mrr-weekly-master-2n-skx: start: 20 end: "lastCompletedBuild" plot-performance-trending-vpp-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 237 + start: 250 end: "lastCompletedBuild" skip: - 347 @@ -315,6 +316,7 @@ - 375 - 380 - 381 + - 396 plot-performance-trending-dpdk-2n-skx: csit-dpdk-perf-mrr-weekly-master-2n-skx: @@ -419,7 +421,7 @@ # 3n-hsw csit-vpp-perf-mrr-daily-master: - start: 487 + start: 501 end: "lastCompletedBuild" skip: - 593 @@ -435,7 +437,7 @@ # 3n-skx csit-vpp-perf-mrr-daily-master-3n-skx: - start: 237 + start: 250 end: "lastCompletedBuild" skip: - 356 @@ -468,7 +470,7 @@ # 2n-skx csit-vpp-perf-mrr-daily-master-2n-skx: - start: 237 + start: 250 end: "lastCompletedBuild" skip: - 347 @@ -486,6 +488,7 @@ - 375 - 380 - 381 + - 396 csit-vpp-perf-mrr-weekly-master-2n-skx: start: 1 end: "lastCompletedBuild"