3 # Copyright (c) 2017 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 """Generate csv files for the chapter "CSIT Release Notes" from json files
17 generated by Jenkins' jobs.
20 from sys import exit as sys_exit
22 from os.path import join
24 from argparse import ArgumentParser, RawDescriptionHelpFormatter
29 EXT_TMPL = ".template"
33 def get_files(path, extension):
34 """Generates the list of files to process.
36 :param path: Path to files.
37 :param extension: Extension of files to process. If it is the empty string,
38 all files will be processed.
41 :returns: List of files to process.
46 for root, _, files in walk(path):
47 for filename in files:
49 if filename.endswith(extension):
50 file_list.append(join(root, filename))
52 file_list.append(join(root, filename))
58 """Parse arguments from cmd line.
60 :returns: Parsed arguments.
64 parser = ArgumentParser(description=__doc__,
65 formatter_class=RawDescriptionHelpFormatter)
66 parser.add_argument("-i", "--input",
68 help="Input folder with data files.")
69 parser.add_argument("-o", "--output",
71 help="Output folder with csv files and templates for "
73 return parser.parse_args()
76 def calculate_stats(data):
77 """Calculate statistics:
81 :param data: Data to process.
83 :returns: Average and standard deviation.
91 """Calculate average from the items.
93 :param items: Average is calculated from these items.
98 return float(sum(items)) / len(items)
101 variance = [(x - avg) ** 2 for x in data]
102 stdev = sqrt(average(variance))
107 def write_line_to_file(file_handler, item):
108 """Write a line to the csv file.
110 :param file_handler: File handler for the csv file. It must be open for
112 :param item: Item to be written to the file.
113 :type file_handler: BinaryIO
117 mean = "" if item["mean"] is None else "{:.1f}".format(item["mean"])
118 stdev = "" if item["stdev"] is None else "{:.1f}".format(item["stdev"])
119 change = "" if item["change"] is None else "{:.0f}%".format(item["change"])
120 file_handler.write("{},{},{},{}\n".format(item["old"], mean, stdev, change))
124 """Main function to generate csv files for the chapter "CSIT Release Notes"
125 from json files generated by Jenkins' jobs.
130 json_files = get_files(args.input, EXT_JSON)
131 tmpl_files = get_files(args.output, EXT_TMPL)
133 if len(json_files) == 0:
134 print("No json data to process.")
137 if len(tmpl_files) == 0:
138 print("No template files to process.")
141 # Get information from template files
143 for tmpl_file in tmpl_files:
144 with open(tmpl_file, mode='r') as file_handler:
145 for line in file_handler:
146 line_list = line.split(',')
150 "type": line_list[0].rsplit("-", 1)[-1],
151 "old": ",".join(line_list[1:])[:-1],
152 "last_old": line_list[-1][:-1],
160 # Update existing data with the new information from json files
161 for json_file in json_files:
162 with open(json_file) as file_handler:
163 tests_data = load(file_handler)
164 for item in csv_data:
166 rate = tests_data["data"][item["ID"]]["throughput"]["value"]
167 item["rates"].append(rate)
172 for item in csv_data:
173 mean, stdev = calculate_stats(item["rates"])
175 mean = float(mean) / 1000000
176 old = float(item["last_old"]) if item["last_old"] else None
178 item["change"] = ((round(mean, 1) - round(old, 1)) / round(old, 1))\
179 * 100 if old else None
180 item["stdev"] = stdev / 1000000
182 # Sort the list, key = change
183 csv_data.sort(key=lambda data: data["change"], reverse=True)
186 for tmpl_file in tmpl_files:
187 csv_file = tmpl_file.replace(EXT_TMPL, EXT_CSV)
188 with open(csv_file, "w") as file_handler:
189 for item in csv_data:
190 if "pdr_" in csv_file \
191 and "_others" not in csv_file \
192 and item["type"] == "pdrdisc" \
193 and item["change"] >= 9.5:
194 write_line_to_file(file_handler, item)
195 elif "pdr_" in csv_file \
196 and "_others" in csv_file \
197 and item["type"] == "pdrdisc" \
198 and item["change"] < 9.5:
199 write_line_to_file(file_handler, item)
200 elif "ndr_" in csv_file \
201 and "_others" not in csv_file \
202 and item["type"] == "ndrdisc" \
203 and item["change"] >= 9.5:
204 write_line_to_file(file_handler, item)
205 elif "ndr_" in csv_file \
206 and "_others" in csv_file \
207 and item["type"] == "ndrdisc" \
208 and item["change"] < 9.5:
209 write_line_to_file(file_handler, item)
212 if __name__ == "__main__":