X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;ds=sidebyside;f=resources%2Ftools%2Fpresentation%2Futils.py;h=88baf9592857f309d5c9c7d58661195ff0a4c547;hb=fde192bc8a6ba1a4e2b4cf5b3297fb076efd58fc;hp=2fbf70cadcf6c648bd47a1e31fbbef7616de145c;hpb=7cfdac0cf07e3a7d9a8b53b7621f8b7500fd1515;p=csit.git diff --git a/resources/tools/presentation/utils.py b/resources/tools/presentation/utils.py index 2fbf70cadc..88baf95928 100644 --- a/resources/tools/presentation/utils.py +++ b/resources/tools/presentation/utils.py @@ -81,15 +81,16 @@ def remove_outliers(input_list, outlier_const=1.5, window=14): :rtype: list of floats """ - input_series = pd.Series() - for index, value in enumerate(input_list): - item_pd = pd.Series([value, ], index=[index, ]) - input_series.append(item_pd) - output_series, _ = split_outliers(input_series, outlier_const=outlier_const, - window=window) - output_list = [y for x, y in output_series.items() if not np.isnan(y)] - - return output_list + data = np.array(input_list) + upper_quartile = np.percentile(data, 75) + lower_quartile = np.percentile(data, 25) + iqr = (upper_quartile - lower_quartile) * outlier_const + quartile_set = (lower_quartile - iqr, upper_quartile + iqr) + result_lst = list() + for y in data.tolist(): + if quartile_set[0] <= y <= quartile_set[1]: + result_lst.append(y) + return result_lst def split_outliers(input_series, outlier_const=1.5, window=14): @@ -122,9 +123,9 @@ def split_outliers(input_series, outlier_const=1.5, window=14): q1 = np.percentile(y_rolling_array, 25) q3 = np.percentile(y_rolling_array, 75) iqr = (q3 - q1) * outlier_const - low, high = q1 - iqr, q3 + iqr + low = q1 - iqr item_pd = pd.Series([item_y, ], index=[item_x, ]) - if low <= item_y <= high: + if low <= item_y: trimmed_data = trimmed_data.append(item_pd) else: outliers = outliers.append(item_pd) @@ -139,7 +140,7 @@ def get_files(path, extension=None, full_path=True): :param path: Path to files. :param extension: Extension of files to process. If it is the empty string, - all files will be processed. + all files will be processed. :param full_path: If True, the files with full path are generated. :type path: str :type extension: str