1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """General purpose utilities.
22 from os import walk, makedirs, environ
23 from os.path import join, isdir
24 from shutil import copy, Error
27 from errors import PresentationError
31 """Calculate mean value from the items.
33 :param items: Mean value is calculated from these items.
39 return float(sum(items)) / len(items)
43 """Calculate stdev from the items.
45 :param items: Stdev is calculated from these items.
52 variance = [(x - avg) ** 2 for x in items]
53 stddev = sqrt(mean(variance))
57 def relative_change(nr1, nr2):
58 """Compute relative change of two values.
60 :param nr1: The first number.
61 :param nr2: The second number.
64 :returns: Relative change of nr1.
68 return float(((nr2 - nr1) / nr1) * 100)
71 def find_outliers(input_data, outlier_const=1.5):
72 """Go through the input data and generate two pandas series:
73 - input data without outliers
75 The function uses IQR to detect outliers.
77 :param input_data: Data to be examined for outliers.
78 :param outlier_const: Outlier constant.
79 :type input_data: pandas.Series
80 :type outlier_const: float
81 :returns: Tuple: input data with outliers removed; Outliers.
82 :rtype: tuple (trimmed_data, outliers)
85 upper_quartile = input_data.quantile(q=0.75)
86 lower_quartile = input_data.quantile(q=0.25)
87 iqr = (upper_quartile - lower_quartile) * outlier_const
88 low = lower_quartile - iqr
89 high = upper_quartile + iqr
90 trimmed_data = pd.Series()
91 outliers = pd.Series()
92 for item in input_data.items():
93 item_pd = pd.Series([item[1], ], index=[item[0], ])
94 if low <= item[1] <= high:
95 trimmed_data = trimmed_data.append(item_pd)
97 trimmed_data = trimmed_data.append(pd.Series([np.nan, ],
99 outliers = outliers.append(item_pd)
101 return trimmed_data, outliers
104 def get_files(path, extension=None, full_path=True):
105 """Generates the list of files to process.
107 :param path: Path to files.
108 :param extension: Extension of files to process. If it is the empty string,
109 all files will be processed.
110 :param full_path: If True, the files with full path are generated.
113 :type full_path: bool
114 :returns: List of files to process.
119 for root, _, files in walk(path):
120 for filename in files:
122 if filename.endswith(extension):
124 file_list.append(join(root, filename))
126 file_list.append(filename)
128 file_list.append(join(root, filename))
133 def get_rst_title_char(level):
134 """Return character used for the given title level in rst files.
136 :param level: Level of the title.
138 :returns: Character used for the given title level in rst files.
141 chars = ('=', '-', '`', "'", '.', '~', '*', '+', '^')
142 if level < len(chars):
148 def execute_command(cmd):
149 """Execute the command in a subprocess and log the stdout and stderr.
151 :param cmd: Command to execute.
153 :returns: Return code of the executed command.
158 proc = subprocess.Popen(
160 stdout=subprocess.PIPE,
161 stderr=subprocess.PIPE,
165 stdout, stderr = proc.communicate()
170 if proc.returncode != 0:
171 logging.error(" Command execution failed.")
172 return proc.returncode, stdout, stderr
175 def get_last_build_number(jenkins_url, job_name):
183 url = "{}/{}/lastSuccessfulBuild/buildNumber".format(jenkins_url, job_name)
184 cmd = "wget -qO- {url}".format(url=url)
186 return execute_command(cmd)
189 def archive_input_data(spec):
190 """Archive the report.
192 :param spec: Specification read from the specification file.
193 :type spec: Specification
194 :raises PresentationError: If it is not possible to archive the input data.
197 logging.info(" Archiving the input data files ...")
200 extension = spec.debug["input-format"]
202 extension = spec.input["file-format"]
203 data_files = get_files(spec.environment["paths"]["DIR[WORKING,DATA]"],
205 dst = spec.environment["paths"]["DIR[STATIC,ARCH]"]
206 logging.info(" Destination: {0}".format(dst))
212 for data_file in data_files:
213 logging.info(" Copying the file: {0} ...".format(data_file))
216 except (Error, OSError) as err:
217 raise PresentationError("Not possible to archive the input data.",
220 logging.info(" Done.")