X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=9e7ada640f6aab4d748232209d366ebcc64e4320;hb=2d6410d275210acd535aae6873221ab0759e5d1e;hp=6de5d46681486273d96d3a3e10a92900ef343485;hpb=2c06090c7b919783a91d3ca9ca8c8319b5c35b07;p=csit.git diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 6de5d46681..9e7ada640f 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -34,7 +34,7 @@ from pal_utils import mean, stdev, relative_change, classify_anomalies, \ convert_csv_to_pretty_txt, relative_change_stdev -REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*') +REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)') def generate_tables(spec, data): @@ -47,7 +47,6 @@ def generate_tables(spec, data): """ generator = { - u"table_details": table_details, u"table_merged_details": table_merged_details, u"table_perf_comparison": table_perf_comparison, u"table_perf_comparison_nic": table_perf_comparison_nic, @@ -97,7 +96,14 @@ def table_oper_data_html(table, input_data): if data.empty: return data = input_data.merge_data(data) - data.sort_index(inplace=True) + + sort_tests = table.get(u"sort", None) + if sort_tests and sort_tests in (u"ascending", u"descending"): + args = dict( + inplace=True, + ascending=True if sort_tests == u"ascending" else False + ) + data.sort_index(**args) suites = input_data.filter_data( table, @@ -267,80 +273,6 @@ def table_oper_data_html(table, input_data): logging.info(u" Done.") -def table_details(table, input_data): - """Generate the table(s) with algorithm: table_detailed_test_results - specified in the specification file. - - :param table: Table to generate. - :param input_data: Data to process. - :type table: pandas.Series - :type input_data: InputData - """ - - logging.info(f" Generating the table {table.get(u'title', u'')} ...") - - # Transform the data - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) - data = input_data.filter_data(table) - - # Prepare the header of the tables - header = list() - for column in table[u"columns"]: - header.append( - u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""')) - ) - - # Generate the data for the table according to the model in the table - # specification - job = list(table[u"data"].keys())[0] - build = str(table[u"data"][job][0]) - try: - suites = input_data.suites(job, build) - except KeyError: - logging.error( - u" No data available. The table will not be generated." - ) - return - - for suite in suites.values: - # Generate data - suite_name = suite[u"name"] - table_lst = list() - for test in data[job][build].keys(): - if data[job][build][test][u"parent"] not in suite_name: - continue - row_lst = list() - for column in table[u"columns"]: - try: - col_data = str(data[job][build][test][column[ - u"data"].split(" ")[1]]).replace(u'"', u'""') - if column[u"data"].split(u" ")[1] in \ - (u"conf-history", u"show-run"): - col_data = col_data.replace(u" |br| ", u"", 1) - col_data = f" |prein| {col_data[:-5]} |preout| " - row_lst.append(f'"{col_data}"') - except KeyError: - row_lst.append(u"No data") - table_lst.append(row_lst) - - # Write the data to file - if table_lst: - file_name = ( - f"{table[u'output-file']}_{suite_name}" - f"{table[u'output-file-ext']}" - ) - logging.info(f" Writing file: {file_name}") - with open(file_name, u"wt") as file_handler: - file_handler.write(u",".join(header) + u"\n") - for item in table_lst: - file_handler.write(u",".join(item) + u"\n") - - logging.info(u" Done.") - - def table_merged_details(table, input_data): """Generate the table(s) with algorithm: table_merged_details specified in the specification file. @@ -359,12 +291,15 @@ def table_merged_details(table, input_data): ) data = input_data.filter_data(table, continue_on_error=True) data = input_data.merge_data(data) - data.sort_index(inplace=True) - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) + sort_tests = table.get(u"sort", None) + if sort_tests and sort_tests in (u"ascending", u"descending"): + args = dict( + inplace=True, + ascending=True if sort_tests == u"ascending" else False + ) + data.sort_index(**args) + suites = input_data.filter_data( table, continue_on_error=True, data_set=u"suites") suites = input_data.merge_data(suites) @@ -391,7 +326,17 @@ def table_merged_details(table, input_data): col_data = col_data.replace( u"No Data", u"Not Captured " ) - if column[u"data"].split(u" ")[1] in \ + if column[u"data"].split(u" ")[1] in (u"name", ): + if len(col_data) > 30: + col_data_lst = col_data.split(u"-") + half = int(len(col_data_lst) / 2) + col_data = f"{u'-'.join(col_data_lst[:half])}" \ + f"- |br| " \ + f"{u'-'.join(col_data_lst[half:])}" + col_data = f" |prein| {col_data} |preout| " + elif column[u"data"].split(u" ")[1] in (u"msg", ): + col_data = f" |prein| {col_data} |preout| " + elif column[u"data"].split(u" ")[1] in \ (u"conf-history", u"show-run"): col_data = col_data.replace(u" |br| ", u"", 1) col_data = f" |prein| {col_data[:-5]} |preout| " @@ -402,10 +347,7 @@ def table_merged_details(table, input_data): # Write the data to file if table_lst: - file_name = ( - f"{table[u'output-file']}_{suite_name}" - f"{table[u'output-file-ext']}" - ) + file_name = f"{table[u'output-file']}_{suite_name}.csv" logging.info(f" Writing file: {file_name}") with open(file_name, u"wt") as file_handler: file_handler.write(u",".join(header) + u"\n") @@ -670,7 +612,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: groups = re.search(REGEX_NIC, tst_data[u"parent"]) @@ -698,7 +642,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in rpl_data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -725,7 +671,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: groups = re.search(REGEX_NIC, tst_data[u"parent"]) @@ -755,7 +703,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in rpl_data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -783,7 +733,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: continue @@ -935,7 +887,9 @@ def table_perf_comparison_nic(table, input_data): if table[u"reference"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" @@ -964,7 +918,9 @@ def table_perf_comparison_nic(table, input_data): if table[u"reference"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -993,7 +949,9 @@ def table_perf_comparison_nic(table, input_data): if table[u"compare"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" @@ -1022,7 +980,9 @@ def table_perf_comparison_nic(table, input_data): if table[u"compare"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -1052,7 +1012,9 @@ def table_perf_comparison_nic(table, input_data): if item[u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: continue @@ -1594,7 +1556,7 @@ def _generate_url(testbed, test_name): elif u"dnv" in testbed or u"tsh" in testbed: driver = u"ixgbe" else: - driver = u"i40e" + driver = u"dpdk" if u"acl" in test_name or \ u"macip" in test_name or \