1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Script for determining whether per-patch perf test votes -1.
16 This script assumes there exist two text files with processed BMRR results,
17 located at hardcoded relative paths (subdirs thereof), having several lines
18 of json-parseable lists of float values, corresponding to testcase results.
19 This script then uses jumpavg library to determine whether there was
20 a regression, progression or no change for each testcase.
21 If number of tests does not match, or there was a regression,
22 this script votes -1 (by exiting with code 1), otherwise it votes +1 (exit 0).
28 from jumpavg.BitCountingMetadataFactory import BitCountingMetadataFactory
29 from jumpavg.BitCountingClassifier import BitCountingClassifier
33 """Return middle two quartiles, hoping to reduce influence of outliers.
35 Currently "middle two" is "all", but that can change in future.
37 :param value_list: List to pick subset from.
38 :type value_list: list of float
39 :returns: New list containing middle values.
42 tmp = sorted(value_list)
44 ret = tmp[3*eight:-eight]
48 parent_iterations = list()
49 current_iterations = list()
54 current_lines = list()
55 filename = "csit_parent/{iter}/results.txt".format(iter=iteration)
57 with open(filename) as parent_file:
58 parent_lines = parent_file.readlines()
61 num_lines = len(parent_lines)
62 filename = "csit_current/{iter}/results.txt".format(iter=iteration)
63 with open(filename) as current_file:
64 current_lines = current_file.readlines()
65 if num_lines != len(current_lines):
66 print "Number of tests does not match within iteration", iteration
70 elif num_tests != num_lines:
71 print "Number of tests does not match previous at iteration", iteration
73 parent_iterations.append(parent_lines)
74 current_iterations.append(current_lines)
75 classifier = BitCountingClassifier()
77 for test_index in range(num_tests):
79 parent_values = list()
80 current_values = list()
81 for iteration_index in range(len(parent_iterations)):
83 json.loads(parent_iterations[iteration_index][test_index]))
84 current_values.extend(
85 json.loads(current_iterations[iteration_index][test_index]))
86 print "Time-ordered MRR values for parent build: {p}".format(
88 print "Time-ordered MRR values for current build: {c}".format(
90 parent_values = hack(parent_values)
91 current_values = hack(current_values)
92 parent_max = BitCountingMetadataFactory.find_max_value(parent_values)
93 current_max = BitCountingMetadataFactory.find_max_value(current_values)
94 val_max = max(val_max, parent_max, current_max)
95 factory = BitCountingMetadataFactory(val_max)
96 parent_stats = factory.from_data(parent_values)
97 current_factory = BitCountingMetadataFactory(val_max, parent_stats.avg)
98 current_stats = current_factory.from_data(current_values)
99 both_stats = factory.from_data(parent_values + current_values)
100 print "Value-ordered MRR values for parent build: {p}".format(
102 print "Value-ordered MRR values for current build: {c}".format(
104 difference = (current_stats.avg - parent_stats.avg) / parent_stats.avg
105 print "Difference of averages relative to parent: {d}%".format(
107 print "Jumpavg representation of parent group: {p}".format(
109 print "Jumpavg representation of current group: {c}".format(
111 print "Jumpavg representation of both as one group: {b}".format(
113 bits = parent_stats.bits + current_stats.bits - both_stats.bits
114 compared = "longer" if bits >= 0 else "shorter"
115 print "Separate groups are {cmp} than single group by {bit} bits".format(
116 cmp=compared, bit=abs(bits))
117 classified_list = classifier.classify([parent_stats, current_stats])
118 if len(classified_list) < 2:
119 print "Test test_index {test_index}: normal (no anomaly)".format(
120 test_index=test_index)
122 anomaly = classified_list[1].metadata.classification
123 if anomaly == "regression":
124 print "Test test_index {test_index}: anomaly regression".format(
125 test_index=test_index)
128 print "Test test_index {test_index}: anomaly {anomaly}".format(
129 test_index=test_index, anomaly=anomaly)
130 print "Exit code {code}".format(code=exit_code)