-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
"""Script for determining whether per-patch perf test votes -1.
This script assumes there exist two text files with processed BMRR results,
-located at hardcoded relative paths, having several lines
+located at hardcoded relative paths (subdirs thereof), having several lines
of json-parseable lists of float values, corresponding to testcase results.
This script then uses jumpavg library to determine whether there was
a regression, progression or no change for each testcase.
def hack(value_list):
"""Return middle two quartiles, hoping to reduce influence of outliers.
+ Currently "middle two" is "all", but that can change in future.
+
:param value_list: List to pick subset from.
:type value_list: list of float
:returns: New list containing middle values.
:rtype: list of float
"""
tmp = sorted(value_list)
- quarter = len(tmp) / 4
- ret = tmp[quarter:-quarter]
- return ret
+ eight = len(tmp) / 8
+ ret = tmp[3*eight:-eight]
+ return tmp # ret
+
-parent_lines = list()
-new_lines = list()
-with open("csit_parent/results.txt") as parent_file:
- parent_lines = parent_file.readlines()
-with open("csit_new/results.txt") as new_file:
- new_lines = new_file.readlines()
-if len(parent_lines) != len(new_lines):
- print "Number of passed tests does not match!"
- sys.exit(1)
+iteration = -1
+parent_iterations = list()
+current_iterations = list()
+num_tests = None
+while 1:
+ iteration += 1
+ parent_lines = list()
+ current_lines = list()
+ filename = f"csit_parent/{iteration}/results.txt"
+ try:
+ with open(filename) as parent_file:
+ parent_lines = parent_file.readlines()
+ except IOError:
+ break
+ num_lines = len(parent_lines)
+ filename = f"csit_current/{iteration}/results.txt"
+ with open(filename) as current_file:
+ current_lines = current_file.readlines()
+ if num_lines != len(current_lines):
+ print(f"Number of tests does not match within iteration {iteration}")
+ sys.exit(1)
+ if num_tests is None:
+ num_tests = num_lines
+ elif num_tests != num_lines:
+ print(
+ f"Number of tests does not match previous at iteration {iteration}"
+ )
+ sys.exit(1)
+ parent_iterations.append(parent_lines)
+ current_iterations.append(current_lines)
classifier = BitCountingClassifier()
-num_tests = len(parent_lines)
exit_code = 0
-for index in range(num_tests):
- parent_values = hack(json.loads(parent_lines[index]))
- new_values = hack(json.loads(new_lines[index]))
+for test_index in range(num_tests):
+ val_max = 1.0
+ parent_values = list()
+ current_values = list()
+ for iteration_index in range(len(parent_iterations)):
+ parent_values.extend(
+ json.loads(parent_iterations[iteration_index][test_index])
+ )
+ current_values.extend(
+ json.loads(current_iterations[iteration_index][test_index])
+ )
+ print(f"Time-ordered MRR values for parent build: {parent_values}")
+ print(f"Time-ordered MRR values for current build: {current_values}")
+ parent_values = hack(parent_values)
+ current_values = hack(current_values)
parent_max = BitCountingMetadataFactory.find_max_value(parent_values)
- new_max = BitCountingMetadataFactory.find_max_value(new_values)
- cmax = max(parent_max, new_max)
- factory = BitCountingMetadataFactory(cmax)
+ current_max = BitCountingMetadataFactory.find_max_value(current_values)
+ val_max = max(val_max, parent_max, current_max)
+ factory = BitCountingMetadataFactory(val_max)
parent_stats = factory.from_data(parent_values)
- factory = BitCountingMetadataFactory(cmax, parent_stats.avg)
- new_stats = factory.from_data(new_values)
- print "DEBUG parent: {p}".format(p=parent_stats)
- print "DEBUG new: {n}".format(n=new_stats)
- common_max = max(parent_stats.avg, new_stats.avg)
- difference = (new_stats.avg - parent_stats.avg) / common_max
- print "DEBUG difference: {d}%".format(d=100 * difference)
- classified_list = classifier.classify([parent_stats, new_stats])
+ current_factory = BitCountingMetadataFactory(val_max, parent_stats.avg)
+ current_stats = current_factory.from_data(current_values)
+ both_stats = factory.from_data(parent_values + current_values)
+ print(f"Value-ordered MRR values for parent build: {parent_values}")
+ print(f"Value-ordered MRR values for current build: {current_values}")
+ difference = (current_stats.avg - parent_stats.avg) / parent_stats.avg
+ print(f"Difference of averages relative to parent: {100 * difference}%")
+ print(f"Jumpavg representation of parent group: {parent_stats}")
+ print(f"Jumpavg representation of current group: {current_stats}")
+ print(f"Jumpavg representation of both as one group: {both_stats}")
+ bits = parent_stats.bits + current_stats.bits - both_stats.bits
+ compared = u"longer" if bits >= 0 else u"shorter"
+ print(
+ f"Separate groups are {compared} than single group by {abs(bits)} bits"
+ )
+ classified_list = classifier.classify([parent_stats, current_stats])
if len(classified_list) < 2:
- print "Test index {index}: normal (no anomaly)".format(
- index=index)
+ print(f"Test test_index {test_index}: normal (no anomaly)")
continue
anomaly = classified_list[1].metadata.classification
- if anomaly == "regression":
- print "Test index {index}: anomaly regression".format(index=index)
+ if anomaly == u"regression":
+ print(f"Test test_index {test_index}: anomaly regression")
exit_code = 1
continue
- print "Test index {index}: anomaly {anomaly}".format(
- index=index, anomaly=anomaly)
-print "DEBUG exit code {code}".format(code=exit_code)
+ print(f"Test test_index {test_index}: anomaly {anomaly}")
+print(f"Exit code {exit_code}")
sys.exit(exit_code)
Code Review / csit.git / blobdiff