X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Futils.py;h=ab86bafdd7c4574046010f02571576afc07f86f3;hb=372eab0eac428149d547b2d6eb2ce43cd0d750f6;hp=966d7f558b17ce554ef5bdc999641db01bc4cde8;hpb=efdcf6470f6e15dcc918c70e5a61d10e10653f1e;p=csit.git diff --git a/resources/tools/presentation/utils.py b/resources/tools/presentation/utils.py index 966d7f558b..ab86bafdd7 100644 --- a/resources/tools/presentation/utils.py +++ b/resources/tools/presentation/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -21,7 +21,7 @@ import logging from os import walk, makedirs, environ from os.path import join, isdir -from shutil import copy, Error +from shutil import move, Error from math import sqrt from errors import PresentationError @@ -68,35 +68,69 @@ def relative_change(nr1, nr2): return float(((nr2 - nr1) / nr1) * 100) -def find_outliers(input_data, outlier_const=1.5): +def remove_outliers(input_list, outlier_const=1.5, window=14): + """Return list with outliers removed, using split_outliers. + + :param input_list: Data from which the outliers will be removed. + :param outlier_const: Outlier constant. + :param window: How many preceding values to take into account. + :type input_list: list of floats + :type outlier_const: float + :type window: int + :returns: The input list without outliers. + :rtype: list of floats + """ + + data = np.array(input_list) + upper_quartile = np.percentile(data, 75) + lower_quartile = np.percentile(data, 25) + iqr = (upper_quartile - lower_quartile) * outlier_const + quartile_set = (lower_quartile - iqr, upper_quartile + iqr) + result_lst = list() + for y in input_list: + if quartile_set[0] <= y <= quartile_set[1]: + result_lst.append(y) + return result_lst + + +def split_outliers(input_series, outlier_const=1.5, window=14): """Go through the input data and generate two pandas series: - - input data without outliers + - input data with outliers replaced by NAN - outliers. The function uses IQR to detect outliers. - :param input_data: Data to be examined for outliers. + :param input_series: Data to be examined for outliers. :param outlier_const: Outlier constant. - :type input_data: pandas.Series + :param window: How many preceding values to take into account. + :type input_series: pandas.Series :type outlier_const: float - :returns: Tuple: input data with outliers removed; Outliers. - :rtype: tuple (trimmed_data, outliers) + :type window: int + :returns: Input data with NAN outliers and Outliers. + :rtype: (pandas.Series, pandas.Series) """ - upper_quartile = input_data.quantile(q=0.75) - lower_quartile = input_data.quantile(q=0.25) - iqr = (upper_quartile - lower_quartile) * outlier_const - low = lower_quartile - iqr - high = upper_quartile + iqr + list_data = list(input_series.items()) + head_size = min(window, len(list_data)) + head_list = list_data[:head_size] trimmed_data = pd.Series() outliers = pd.Series() - for item in input_data.items(): - item_pd = pd.Series([item[1], ], index=[item[0], ]) - if low <= item[1] <= high: + for item_x, item_y in head_list: + item_pd = pd.Series([item_y, ], index=[item_x, ]) + trimmed_data = trimmed_data.append(item_pd) + for index, (item_x, item_y) in list(enumerate(list_data))[head_size:]: + y_rolling_list = [y for (x, y) in list_data[index - head_size:index]] + y_rolling_array = np.array(y_rolling_list) + q1 = np.percentile(y_rolling_array, 25) + q3 = np.percentile(y_rolling_array, 75) + iqr = (q3 - q1) * outlier_const + low = q1 - iqr + item_pd = pd.Series([item_y, ], index=[item_x, ]) + if low <= item_y: trimmed_data = trimmed_data.append(item_pd) else: - trimmed_data = trimmed_data.append(pd.Series([np.nan, ], - index=[item[0], ])) outliers = outliers.append(item_pd) + nan_pd = pd.Series([np.nan, ], index=[item_x, ]) + trimmed_data = trimmed_data.append(nan_pd) return trimmed_data, outliers @@ -106,7 +140,7 @@ def get_files(path, extension=None, full_path=True): :param path: Path to files. :param extension: Extension of files to process. If it is the empty string, - all files will be processed. + all files will be processed. :param full_path: If True, the files with full path are generated. :type path: str :type extension: str @@ -164,20 +198,25 @@ def execute_command(cmd): stdout, stderr = proc.communicate() - logging.info(stdout) - logging.info(stderr) + if stdout: + logging.info(stdout) + if stderr: + logging.info(stderr) if proc.returncode != 0: logging.error(" Command execution failed.") return proc.returncode, stdout, stderr -def get_last_build_number(jenkins_url, job_name): - """ +def get_last_successful_build_number(jenkins_url, job_name): + """Get the number of the last successful build of the given job. - :param jenkins_url: - :param job_name: - :return: + :param jenkins_url: Jenkins URL. + :param job_name: Job name. + :type jenkins_url: str + :type job_name: str + :returns: The build number as a string. + :rtype: str """ url = "{}/{}/lastSuccessfulBuild/buildNumber".format(jenkins_url, job_name) @@ -186,6 +225,23 @@ def get_last_build_number(jenkins_url, job_name): return execute_command(cmd) +def get_last_completed_build_number(jenkins_url, job_name): + """Get the number of the last completed build of the given job. + + :param jenkins_url: Jenkins URL. + :param job_name: Job name. + :type jenkins_url: str + :type job_name: str + :returns: The build number as a string. + :rtype: str + """ + + url = "{}/{}/lastCompletedBuild/buildNumber".format(jenkins_url, job_name) + cmd = "wget -qO- {url}".format(url=url) + + return execute_command(cmd) + + def archive_input_data(spec): """Archive the report. @@ -196,10 +252,7 @@ def archive_input_data(spec): logging.info(" Archiving the input data files ...") - if spec.is_debug: - extension = spec.debug["input-format"] - else: - extension = spec.input["file-format"] + extension = spec.input["file-format"] data_files = get_files(spec.environment["paths"]["DIR[WORKING,DATA]"], extension=extension) dst = spec.environment["paths"]["DIR[STATIC,ARCH]"] @@ -210,8 +263,8 @@ def archive_input_data(spec): makedirs(dst) for data_file in data_files: - logging.info(" Copying the file: {0} ...".format(data_file)) - copy(data_file, dst) + logging.info(" Moving the file: {0} ...".format(data_file)) + move(data_file, dst) except (Error, OSError) as err: raise PresentationError("Not possible to archive the input data.",