+ else:
+ tdata.text = item
+ try:
+ with open(table[u"output-file"], u'w') as html_file:
+ logging.info(f" Writing file: {table[u'output-file']}")
+ html_file.write(u".. raw:: html\n\n\t")
+ html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
+ html_file.write(u"\n\t<p><br><br></p>\n")
+ except KeyError:
+ logging.warning(u"The output file is not defined.")
+ return
+
+
+def table_last_failed_tests(table, input_data):
+ """Generate the table(s) with algorithm: table_last_failed_tests
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param input_data: Data to process.
+ :type table: pandas.Series
+ :type input_data: InputData
+ """
+
+ logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
+ # Transform the data
+ logging.info(
+ f" Creating the data set for the {table.get(u'type', u'')} "
+ f"{table.get(u'title', u'')}."
+ )
+
+ data = input_data.filter_data(table, continue_on_error=True)
+
+ if data is None or data.empty:
+ logging.warning(
+ f" No data for the {table.get(u'type', u'')} "
+ f"{table.get(u'title', u'')}."
+ )
+ return
+
+ tbl_list = list()
+ for job, builds in table[u"data"].items():
+ for build in builds:
+ build = str(build)
+ try:
+ version = input_data.metadata(job, build).get(u"version", u"")
+ except KeyError:
+ logging.error(f"Data for {job}: {build} is not present.")
+ return
+ tbl_list.append(build)
+ tbl_list.append(version)
+ failed_tests = list()
+ passed = 0
+ failed = 0
+ for tst_data in data[job][build].values:
+ if tst_data[u"status"] != u"FAIL":
+ passed += 1
+ continue
+ failed += 1
+ groups = re.search(REGEX_NIC, tst_data[u"parent"])
+ if not groups:
+ continue
+ nic = groups.group(0)
+ failed_tests.append(f"{nic}-{tst_data[u'name']}")
+ tbl_list.append(str(passed))
+ tbl_list.append(str(failed))
+ tbl_list.extend(failed_tests)
+
+ file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
+ logging.info(f" Writing file: {file_name}")
+ with open(file_name, u"wt") as file_handler:
+ for test in tbl_list:
+ file_handler.write(test + u'\n')
+
+
+def table_failed_tests(table, input_data):
+ """Generate the table(s) with algorithm: table_failed_tests
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param input_data: Data to process.
+ :type table: pandas.Series
+ :type input_data: InputData
+ """
+
+ logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
+ # Transform the data
+ logging.info(
+ f" Creating the data set for the {table.get(u'type', u'')} "
+ f"{table.get(u'title', u'')}."
+ )
+ data = input_data.filter_data(table, continue_on_error=True)
+
+ # Prepare the header of the tables
+ header = [
+ u"Test Case",
+ u"Failures [#]",
+ u"Last Failure [Time]",
+ u"Last Failure [VPP-Build-Id]",
+ u"Last Failure [CSIT-Job-Build-Id]"
+ ]
+
+ # Generate the data for the table according to the model in the table
+ # specification
+
+ now = dt.utcnow()
+ timeperiod = timedelta(int(table.get(u"window", 7)))
+
+ tbl_dict = dict()
+ for job, builds in table[u"data"].items():
+ for build in builds:
+ build = str(build)
+ for tst_name, tst_data in data[job][build].items():
+ if tst_name.lower() in table.get(u"ignore-list", list()):
+ continue
+ if tbl_dict.get(tst_name, None) is None:
+ groups = re.search(REGEX_NIC, tst_data[u"parent"])
+ if not groups:
+ continue
+ nic = groups.group(0)
+ tbl_dict[tst_name] = {
+ u"name": f"{nic}-{tst_data[u'name']}",
+ u"data": OrderedDict()
+ }
+ try:
+ generated = input_data.metadata(job, build).\
+ get(u"generated", u"")
+ if not generated:
+ continue
+ then = dt.strptime(generated, u"%Y%m%d %H:%M")
+ if (now - then) <= timeperiod:
+ tbl_dict[tst_name][u"data"][build] = (
+ tst_data[u"status"],
+ generated,
+ input_data.metadata(job, build).get(u"version",
+ u""),
+ build
+ )
+ except (TypeError, KeyError) as err:
+ logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
+
+ max_fails = 0
+ tbl_lst = list()
+ for tst_data in tbl_dict.values():
+ fails_nr = 0
+ fails_last_date = u""
+ fails_last_vpp = u""
+ fails_last_csit = u""
+ for val in tst_data[u"data"].values():
+ if val[0] == u"FAIL":
+ fails_nr += 1
+ fails_last_date = val[1]
+ fails_last_vpp = val[2]
+ fails_last_csit = val[3]
+ if fails_nr:
+ max_fails = fails_nr if fails_nr > max_fails else max_fails
+ tbl_lst.append(
+ [
+ tst_data[u"name"],
+ fails_nr,
+ fails_last_date,
+ fails_last_vpp,
+ f"mrr-daily-build-{fails_last_csit}"
+ ]
+ )
+
+ tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
+ tbl_sorted = list()
+ for nrf in range(max_fails, -1, -1):
+ tbl_fails = [item for item in tbl_lst if item[1] == nrf]
+ tbl_sorted.extend(tbl_fails)
+
+ file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
+ logging.info(f" Writing file: {file_name}")
+ with open(file_name, u"wt") as file_handler:
+ file_handler.write(u",".join(header) + u"\n")
+ for test in tbl_sorted:
+ file_handler.write(u",".join([str(item) for item in test]) + u'\n')
+
+ logging.info(f" Writing file: {table[u'output-file']}.txt")
+ convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
+
+
+def table_failed_tests_html(table, input_data):
+ """Generate the table(s) with algorithm: table_failed_tests_html
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param input_data: Data to process.
+ :type table: pandas.Series
+ :type input_data: InputData
+ """
+
+ _ = input_data
+
+ if not table.get(u"testbed", None):
+ logging.error(
+ f"The testbed is not defined for the table "
+ f"{table.get(u'title', u'')}."
+ )
+ return
+
+ logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
+ try:
+ with open(table[u"input-file"], u'rt') as csv_file:
+ csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
+ except KeyError:
+ logging.warning(u"The input file is not defined.")
+ return
+ except csv.Error as err:
+ logging.warning(
+ f"Not possible to process the file {table[u'input-file']}.\n"
+ f"{repr(err)}"
+ )
+ return