1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Script for determining whether per-patch perf test votes -1.
16 This script assumes there exist two text files with processed BMRR results,
17 located at hardcoded relative paths (subdirs thereof), having several lines
18 of json-parseable lists of float values, corresponding to testcase results.
19 This script then uses jumpavg library to determine whether there was
20 a regression, progression or no change for each testcase.
21 If number of tests does not match, or there was a regression,
22 this script votes -1 (by exiting with code 1), otherwise it votes +1 (exit 0).
28 from jumpavg.BitCountingMetadataFactory import BitCountingMetadataFactory
29 from jumpavg.BitCountingClassifier import BitCountingClassifier
33 """Return middle two quartiles, hoping to reduce influence of outliers.
35 :param value_list: List to pick subset from.
36 :type value_list: list of float
37 :returns: New list containing middle values.
40 tmp = sorted(value_list)
42 ret = tmp[3*eight:-eight]
46 parent_iterations = list()
47 new_iterations = list()
53 filename = "csit_parent/{iter}/results.txt".format(iter=iteration)
55 with open(filename) as parent_file:
56 parent_lines = parent_file.readlines()
59 num_lines = len(parent_lines)
60 filename = "csit_current/{iter}/results.txt".format(iter=iteration)
61 with open(filename) as new_file:
62 new_lines = new_file.readlines()
63 if num_lines != len(new_lines):
64 print "Number of tests does not match within iteration", iteration
68 elif num_tests != num_lines:
69 print "Number of tests does not match previous at iteration", iteration
71 parent_iterations.append(parent_lines)
72 new_iterations.append(new_lines)
73 classifier = BitCountingClassifier()
75 for test_index in range(num_tests):
77 parent_values = list()
79 for iteration_index in range(len(parent_iterations)):
81 json.loads(parent_iterations[iteration_index][test_index]))
83 json.loads(new_iterations[iteration_index][test_index]))
84 print "TRACE pre-hack parent: {p}".format(p=parent_values)
85 print "TRACE pre-hack current: {n}".format(n=new_values)
86 parent_values = hack(parent_values)
87 new_values = hack(new_values)
88 parent_max = BitCountingMetadataFactory.find_max_value(parent_values)
89 new_max = BitCountingMetadataFactory.find_max_value(new_values)
90 val_max = max(val_max, parent_max, new_max)
91 factory = BitCountingMetadataFactory(val_max)
92 parent_stats = factory.from_data(parent_values)
93 new_factory = BitCountingMetadataFactory(val_max, parent_stats.avg)
94 new_stats = new_factory.from_data(new_values)
95 print "TRACE parent: {p}".format(p=parent_values)
96 print "TRACE current: {n}".format(n=new_values)
97 print "DEBUG parent: {p}".format(p=parent_stats)
98 print "DEBUG current: {n}".format(n=new_stats)
99 common_max = max(parent_stats.avg, new_stats.avg)
100 difference = (new_stats.avg - parent_stats.avg) / common_max
101 print "DEBUG difference: {d}%".format(d=100 * difference)
102 classified_list = classifier.classify([parent_stats, new_stats])
103 if len(classified_list) < 2:
104 print "Test test_index {test_index}: normal (no anomaly)".format(
105 test_index=test_index)
107 anomaly = classified_list[1].metadata.classification
108 if anomaly == "regression":
109 print "Test test_index {test_index}: anomaly regression".format(
110 test_index=test_index)
113 print "Test test_index {test_index}: anomaly {anomaly}".format(
114 test_index=test_index, anomaly=anomaly)
115 print "DEBUG exit code {code}".format(code=exit_code)