From cee3ad0f9cc29ffc67d9c87c58920252671beb21 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Thu, 1 Aug 2019 12:24:08 +0200 Subject: [PATCH] Trending: Add multiprocessing, remove archiving Change-Id: I67cfde7dfc9b81fca3ae102b43f6defafe88f689 Signed-off-by: Tibor Frank --- resources/tools/presentation/input_data_files.py | 16 +- resources/tools/presentation/input_data_parser.py | 91 +++++--- resources/tools/presentation/requirements.txt | 1 - .../tools/presentation/specification_CPTA.yaml | 240 ++++++++++----------- 4 files changed, 192 insertions(+), 156 deletions(-) diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py index 892dfe2a07..15d7d02b7c 100644 --- a/resources/tools/presentation/input_data_files.py +++ b/resources/tools/presentation/input_data_files.py @@ -219,15 +219,15 @@ def download_and_unzip_data_file(spec, job, build, pid, log): else: return False - if spec.configuration.get("archive-inputs", True): - if spec.input["file-name"].endswith(".gz"): - if "docs.fd.io" in url: - execute_command("gzip --decompress --keep --force {0}". - format(new_name)) - else: - rename(new_name, new_name[:-3]) + if spec.input["file-name"].endswith(".gz"): + if "docs.fd.io" in url: + execute_command("gzip --decompress --keep --force {0}". + format(new_name)) + else: + rename(new_name, new_name[:-3]) + if spec.configuration.get("archive-inputs", True): execute_command("gzip --keep {0}".format(new_name[:-3])) - build["file-name"] = new_name[:-3] + build["file-name"] = new_name[:-3] if new_name.endswith(".zip"): if is_zipfile(new_name): diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index 37532c83d2..9c0e38073c 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -23,7 +23,6 @@ import multiprocessing import os import re import resource -import objgraph import pandas as pd import logging @@ -1189,10 +1188,13 @@ class InputData(object): return checker.data - def _download_and_parse_build(self, job, build, repeat, pid=10000): + def _download_and_parse_build(self, pid, data_queue, job, build, repeat): """Download and parse the input data file. :param pid: PID of the process executing this method. + :param data_queue: Shared memory between processes. Queue which keeps + the result data. This data is then read by the main process and used + in further processing. :param job: Name of the Jenkins job which generated the processed input file. :param build: Information about the Jenkins build which generated the @@ -1200,6 +1202,7 @@ class InputData(object): :param repeat: Repeat the download specified number of times if not successful. :type pid: int + :type data_queue: multiprocessing.Manager().Queue() :type job: str :type build: dict :type repeat: int @@ -1280,6 +1283,14 @@ class InputData(object): format(full_name, repr(err)))) logs.append(("INFO", " Done.")) + result = { + "data": data, + "state": state, + "job": job, + "build": build + } + data_queue.put(result) + for level, line in logs: if level == "INFO": logging.info(line) @@ -1292,7 +1303,8 @@ class InputData(object): elif level == "WARNING": logging.warning(line) - return {"data": data, "state": state, "job": job, "build": build} + logging.info("Memory allocation: {0:,d}MB".format( + resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000)) def download_and_parse_data(self, repeat=1): """Download the input data files, parse input data from input files and @@ -1305,36 +1317,67 @@ class InputData(object): logging.info("Downloading and parsing input files ...") + work_queue = multiprocessing.JoinableQueue() + manager = multiprocessing.Manager() + data_queue = manager.Queue() + cpus = multiprocessing.cpu_count() + + workers = list() + for cpu in range(cpus): + worker = Worker(work_queue, + data_queue, + self._download_and_parse_build) + worker.daemon = True + worker.start() + workers.append(worker) + os.system("taskset -p -c {0} {1} > /dev/null 2>&1". + format(cpu, worker.pid)) + for job, builds in self._cfg.builds.items(): for build in builds: + work_queue.put((job, build, repeat)) + + work_queue.join() + + logging.info("Done.") + logging.info("Collecting data:") + + while not data_queue.empty(): + result = data_queue.get() + + job = result["job"] + build_nr = result["build"]["build"] + logging.info(" {job}-{build}".format(job=job, build=build_nr)) - result = self._download_and_parse_build(job, build, repeat) - build_nr = result["build"]["build"] + if result["data"]: + data = result["data"] + build_data = pd.Series({ + "metadata": pd.Series( + data["metadata"].values(), + index=data["metadata"].keys()), + "suites": pd.Series(data["suites"].values(), + index=data["suites"].keys()), + "tests": pd.Series(data["tests"].values(), + index=data["tests"].keys())}) - if result["data"]: - data = result["data"] - build_data = pd.Series({ - "metadata": pd.Series( - data["metadata"].values(), - index=data["metadata"].keys()), - "suites": pd.Series(data["suites"].values(), - index=data["suites"].keys()), - "tests": pd.Series(data["tests"].values(), - index=data["tests"].keys())}) + if self._input_data.get(job, None) is None: + self._input_data[job] = pd.Series() + self._input_data[job][str(build_nr)] = build_data - if self._input_data.get(job, None) is None: - self._input_data[job] = pd.Series() - self._input_data[job][str(build_nr)] = build_data + self._cfg.set_input_file_name( + job, build_nr, result["build"]["file-name"]) - self._cfg.set_input_file_name( - job, build_nr, result["build"]["file-name"]) + self._cfg.set_input_state(job, build_nr, result["state"]) - self._cfg.set_input_state(job, build_nr, result["state"]) + logging.info("Memory allocation: {0:,d}MB".format( + resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000)) - logging.info("ru_maxrss = {0}".format( - resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)) + del data_queue - logging.info(objgraph.most_common_types()) + # Terminate all workers + for worker in workers: + worker.terminate() + worker.join() logging.info("Done.") diff --git a/resources/tools/presentation/requirements.txt b/resources/tools/presentation/requirements.txt index 6ada198b70..66a993df79 100644 --- a/resources/tools/presentation/requirements.txt +++ b/resources/tools/presentation/requirements.txt @@ -8,4 +8,3 @@ numpy==1.16.4 pandas==0.24.2 plotly==3.3.0 PTable==0.9.2 -objgraph==3.4.1 diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index c98a51ae28..4015ebc52c 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -171,23 +171,23 @@ # 3n-hsw plot-performance-trending-all-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 720 + start: 670 end: "lastCompletedBuild" skip: -# - 672 -# - 673 -# - 674 -# - 675 -# - 676 -# - 677 -# - 678 -# - 679 -# - 680 -# - 688 -# - 689 -# - 690 -# - 694 -# - 695 + - 672 + - 673 + - 674 + - 675 + - 676 + - 677 + - 678 + - 679 + - 680 + - 688 + - 689 + - 690 + - 694 + - 695 - 726 - 749 - 750 @@ -218,7 +218,6 @@ - 797 - 798 - 799 - - 800 # - 801 # - 802 # - 803 # @@ -236,23 +235,23 @@ plot-performance-trending-vpp-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 720 + start: 670 end: "lastCompletedBuild" skip: -# - 672 -# - 673 -# - 674 -# - 675 -# - 676 -# - 677 -# - 678 -# - 679 -# - 680 -# - 688 -# - 689 -# - 690 -# - 694 -# - 695 + - 672 + - 673 + - 674 + - 675 + - 676 + - 677 + - 678 + - 679 + - 680 + - 688 + - 689 + - 690 + - 694 + - 695 - 726 - 749 - 750 @@ -283,7 +282,6 @@ - 797 - 798 - 799 - - 800 # - 801 # - 802 # - 803 # @@ -304,20 +302,20 @@ # 3n-skx plot-performance-trending-all-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 450 + start: 420 end: "lastCompletedBuild" skip: -# - 423 -# - 424 -# - 425 -# - 426 -# - 427 -# - 428 -# - 429 -# - 430 -# - 440 -# - 444 -# - 446 + - 423 + - 424 + - 425 + - 426 + - 427 + - 428 + - 429 + - 430 + - 440 + - 444 + - 446 - 480 - 481 - 499 @@ -350,7 +348,6 @@ - 547 - 548 - 549 - - 550 # - 551 # - 552 # - 553 # @@ -368,20 +365,20 @@ plot-performance-trending-vpp-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 450 + start: 420 end: "lastCompletedBuild" skip: -# - 423 -# - 424 -# - 425 -# - 426 -# - 427 -# - 428 -# - 429 -# - 430 -# - 440 -# - 444 -# - 446 + - 423 + - 424 + - 425 + - 426 + - 427 + - 428 + - 429 + - 430 + - 440 + - 444 + - 446 - 480 - 481 - 499 @@ -414,7 +411,6 @@ - 547 - 548 - 549 - - 550 # - 551 # - 552 # - 553 # @@ -435,20 +431,20 @@ # 2n-skx plot-performance-trending-all-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 450 + start: 420 end: "lastCompletedBuild" skip: -# - 423 -# - 424 -# - 425 -# - 426 -# - 427 -# - 428 -# - 429 -# - 430 -# - 431 -# - 441 -# - 444 + - 423 + - 424 + - 425 + - 426 + - 427 + - 428 + - 429 + - 430 + - 431 + - 441 + - 444 - 477 - 500 - 501 @@ -482,20 +478,20 @@ plot-performance-trending-vpp-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 450 + start: 420 end: "lastCompletedBuild" skip: -# - 423 -# - 424 -# - 425 -# - 426 -# - 427 -# - 428 -# - 429 -# - 430 -# - 431 -# - 441 -# - 444 + - 423 + - 424 + - 425 + - 426 + - 427 + - 428 + - 429 + - 430 + - 431 + - 441 + - 444 - 477 - 500 - 501 @@ -626,23 +622,23 @@ # 3n-hsw csit-vpp-perf-mrr-daily-master: - start: 720 # 670 + start: 670 end: "lastCompletedBuild" skip: -# - 672 -# - 673 -# - 674 -# - 675 -# - 676 -# - 677 -# - 678 -# - 679 -# - 680 -# - 688 -# - 689 -# - 690 -# - 694 -# - 695 + - 672 + - 673 + - 674 + - 675 + - 676 + - 677 + - 678 + - 679 + - 680 + - 688 + - 689 + - 690 + - 694 + - 695 - 726 - 749 - 750 @@ -673,7 +669,6 @@ - 797 - 798 - 799 - - 800 # - 801 # - 802 # - 803 # @@ -691,20 +686,20 @@ # 3n-skx csit-vpp-perf-mrr-daily-master-3n-skx: - start: 450 # 420 + start: 420 end: "lastCompletedBuild" skip: -# - 423 -# - 424 -# - 425 -# - 426 -# - 427 -# - 428 -# - 429 -# - 430 -# - 440 -# - 444 -# - 446 + - 423 + - 424 + - 425 + - 426 + - 427 + - 428 + - 429 + - 430 + - 440 + - 444 + - 446 - 480 - 481 - 499 @@ -737,7 +732,6 @@ - 547 - 548 - 549 - - 550 # - 551 # - 552 # - 553 # @@ -755,20 +749,20 @@ # 2n-skx csit-vpp-perf-mrr-daily-master-2n-skx: - start: 450 + start: 420 end: "lastCompletedBuild" skip: -# - 423 -# - 424 -# - 425 -# - 426 -# - 427 -# - 428 -# - 429 -# - 430 -# - 431 -# - 441 -# - 444 + - 423 + - 424 + - 425 + - 426 + - 427 + - 428 + - 429 + - 430 + - 431 + - 441 + - 444 - 477 - 500 - 501 -- 2.16.6