1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """General purpose utilities.
17 import multiprocessing
24 from os import walk, makedirs, environ
25 from os.path import join, isdir
26 from shutil import move, Error
29 from errors import PresentationError
30 from jumpavg.BitCountingClassifier import BitCountingClassifier
34 """Calculate mean value from the items.
36 :param items: Mean value is calculated from these items.
42 return float(sum(items)) / len(items)
46 """Calculate stdev from the items.
48 :param items: Stdev is calculated from these items.
55 variance = [(x - avg) ** 2 for x in items]
56 stddev = sqrt(mean(variance))
60 def relative_change(nr1, nr2):
61 """Compute relative change of two values.
63 :param nr1: The first number.
64 :param nr2: The second number.
67 :returns: Relative change of nr1.
71 return float(((nr2 - nr1) / nr1) * 100)
74 def get_files(path, extension=None, full_path=True):
75 """Generates the list of files to process.
77 :param path: Path to files.
78 :param extension: Extension of files to process. If it is the empty string,
79 all files will be processed.
80 :param full_path: If True, the files with full path are generated.
84 :returns: List of files to process.
89 for root, _, files in walk(path):
90 for filename in files:
92 if filename.endswith(extension):
94 file_list.append(join(root, filename))
96 file_list.append(filename)
98 file_list.append(join(root, filename))
103 def get_rst_title_char(level):
104 """Return character used for the given title level in rst files.
106 :param level: Level of the title.
108 :returns: Character used for the given title level in rst files.
111 chars = ('=', '-', '`', "'", '.', '~', '*', '+', '^')
112 if level < len(chars):
118 def execute_command(cmd):
119 """Execute the command in a subprocess and log the stdout and stderr.
121 :param cmd: Command to execute.
123 :returns: Return code of the executed command.
128 proc = subprocess.Popen(
130 stdout=subprocess.PIPE,
131 stderr=subprocess.PIPE,
135 stdout, stderr = proc.communicate()
142 if proc.returncode != 0:
143 logging.error(" Command execution failed.")
144 return proc.returncode, stdout, stderr
147 def get_last_successful_build_number(jenkins_url, job_name):
148 """Get the number of the last successful build of the given job.
150 :param jenkins_url: Jenkins URL.
151 :param job_name: Job name.
152 :type jenkins_url: str
154 :returns: The build number as a string.
158 url = "{}/{}/lastSuccessfulBuild/buildNumber".format(jenkins_url, job_name)
159 cmd = "wget -qO- {url}".format(url=url)
161 return execute_command(cmd)
164 def get_last_completed_build_number(jenkins_url, job_name):
165 """Get the number of the last completed build of the given job.
167 :param jenkins_url: Jenkins URL.
168 :param job_name: Job name.
169 :type jenkins_url: str
171 :returns: The build number as a string.
175 url = "{}/{}/lastCompletedBuild/buildNumber".format(jenkins_url, job_name)
176 cmd = "wget -qO- {url}".format(url=url)
178 return execute_command(cmd)
181 def archive_input_data(spec):
182 """Archive the report.
184 :param spec: Specification read from the specification file.
185 :type spec: Specification
186 :raises PresentationError: If it is not possible to archive the input data.
189 logging.info(" Archiving the input data files ...")
191 extension = spec.input["file-format"]
192 data_files = get_files(spec.environment["paths"]["DIR[WORKING,DATA]"],
194 dst = spec.environment["paths"]["DIR[STATIC,ARCH]"]
195 logging.info(" Destination: {0}".format(dst))
201 for data_file in data_files:
202 logging.info(" Moving the file: {0} ...".format(data_file))
205 except (Error, OSError) as err:
206 raise PresentationError("Not possible to archive the input data.",
209 logging.info(" Done.")
212 def classify_anomalies(data):
213 """Process the data and return anomalies and trending values.
215 Gather data into groups with average as trend value.
216 Decorate values within groups to be normal,
217 the first value of changed average as a regression, or a progression.
219 :param data: Full data set with unavailable samples replaced by nan.
220 :type data: OrderedDict
221 :returns: Classification and trend values
222 :rtype: 2-tuple, list of strings and list of floats
224 # Nan mean something went wrong.
225 # Use 0.0 to cause that being reported as a severe regression.
226 bare_data = [0.0 if np.isnan(sample.avg) else sample
227 for _, sample in data.iteritems()]
228 # TODO: Put analogous iterator into jumpavg library.
229 groups = BitCountingClassifier().classify(bare_data)
230 groups.reverse() # Just to use .pop() for FIFO.
236 for _, sample in data.iteritems():
237 if np.isnan(sample.avg):
238 classification.append("outlier")
239 avgs.append(sample.avg)
241 if values_left < 1 or active_group is None:
243 while values_left < 1: # Ignore empty groups (should not happen).
244 active_group = groups.pop()
245 values_left = len(active_group.values)
246 avg = active_group.metadata.avg
247 classification.append(active_group.metadata.classification)
251 classification.append("normal")
254 return classification, avgs
257 def convert_csv_to_pretty_txt(csv_file, txt_file):
258 """Convert the given csv table to pretty text table.
260 :param csv_file: The path to the input csv file.
261 :param txt_file: The path to the output pretty text file.
267 with open(csv_file, 'rb') as csv_file:
268 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
269 for row in csv_content:
270 if txt_table is None:
271 txt_table = prettytable.PrettyTable(row)
273 txt_table.add_row(row)
274 txt_table.align["Test case"] = "l"
276 with open(txt_file, "w") as txt_file:
277 txt_file.write(str(txt_table))
280 class Worker(multiprocessing.Process):
281 """Worker class used to process tasks in separate parallel processes.
284 def __init__(self, work_queue, data_queue, func):
287 :param work_queue: Queue with items to process.
288 :param data_queue: Shared memory between processes. Queue which keeps
289 the result data. This data is then read by the main process and used
290 in further processing.
291 :param func: Function which is executed by the worker.
292 :type work_queue: multiprocessing.JoinableQueue
293 :type data_queue: multiprocessing.Manager().Queue()
294 :type func: Callable object
296 super(Worker, self).__init__()
297 self._work_queue = work_queue
298 self._data_queue = data_queue
302 """Method representing the process's activity.
307 self.process(self._work_queue.get())
309 self._work_queue.task_done()
311 def process(self, item_to_process):
312 """Method executed by the runner.
314 :param item_to_process: Data to be processed by the function.
315 :type item_to_process: tuple
317 self._func(self.pid, self._data_queue, *item_to_process)