from ..utils.trigger import Trigger
from ..utils.url_processing import url_decode
from ..utils.utils import generate_options, gen_new_url, navbar_report, \
- filter_table_data, sort_table_data, show_iterative_graph_data, show_tooltip
+ filter_table_data, sort_table_data, show_iterative_graph_data, \
+ show_tooltip, get_topo_arch
from .tables import comparison_table
from ..report.graphs import graph_iterative
lst_job = row["job"].split("-")
dut = lst_job[1]
dver = f"{row['release']}-{row['dut_version']}"
- tbed = "-".join(lst_job[-2:])
+ tbed = get_topo_arch(lst_job)
lst_test_id = row["test_id"].split(".")
suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
:returns: A list of simple items.
:rtype: list
"""
- l_infra = sel["infra"].split("-")
+ l_infra = sel["infra"].rsplit("-", maxsplit=2)
selection = list()
for core in sel["core"]:
for fsize in sel["frmsize"]:
selection.append({
"dut": sel["dut"],
"dutver": sel["dutver"],
- "tbed": f"{l_infra[0]}-{l_infra[1]}",
- "nic": l_infra[2],
+ "tbed": l_infra[0],
+ "nic": l_infra[1],
"driver": l_infra[-1].replace("_", "-"),
"core": core,
"frmsize": fsize,
from ..utils.control_panel import ControlPanel
from ..utils.trigger import Trigger
from ..utils.utils import label, gen_new_url, generate_options, navbar_report, \
- show_tooltip
+ show_tooltip, get_topo_arch
from ..utils.url_processing import url_decode
from .tables import coverage_tables, select_coverage_data
lst_job = row["job"].split("-")
dut = lst_job[1]
d_ver = row["dut_version"]
- tbed = "-".join(lst_job[-2:])
+ tbed = get_topo_arch(lst_job)
lst_test_id = row["test_id"].split(".")
if dut == "dpdk":
area = "dpdk"
l_data = list()
# Filter data selected by the user.
- phy = selected["phy"].split("-")
- if len(phy) == 4:
- topo, arch, nic, drv = phy
+ phy = selected["phy"].rsplit("-", maxsplit=2)
+ if len(phy) == 3:
+ topo_arch, nic, drv = phy
drv_str = "" if drv == "dpdk" else drv.replace("_", "-")
else:
return l_data, None
(data["release"] == selected["rls"])
)])
df = df[
- (df.job.str.endswith(f"{topo}-{arch}")) &
+ (df.job.str.endswith(topo_arch)) &
(df.test_id.str.contains(
f"^.*\.{selected['area']}\..*{nic}.*{drv_str}.*$",
regex=True
from dash import Input, Output, State
from ..utils.constants import Constants as C
-from ..utils.utils import gen_new_url, navbar_trending
+from ..utils.utils import gen_new_url, navbar_trending, get_topo_arch
from ..utils.anomalies import classify_anomalies
from ..utils.url_processing import url_decode
from .tables import table_summary
d_job_info["dut"].append(lst_job[1])
d_job_info["ttype"].append(lst_job[3])
d_job_info["cadence"].append(lst_job[4])
- d_job_info["tbed"].append("-".join(lst_job[-2:]))
+ d_job_info["tbed"].append(get_topo_arch(lst_job))
self.job_info = pd.DataFrame.from_dict(d_job_info)
# Pre-process the data:
from ..utils.utils import get_color, get_hdrh_latencies
-def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
+def select_iterative_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
"""Select the data for graphs and tables from the provided data frame.
:param data: Data frame with data for graphs and tables.
:rtype: pandas.DataFrame
"""
- phy = itm["phy"].split("-")
- if len(phy) == 4:
- topo, arch, nic, drv = phy
+ phy = itm["phy"].rsplit("-", maxsplit=2)
+ if len(phy) == 3:
+ topo_arch, nic, drv = phy
if drv == "dpdk":
drv = ""
else:
(data["passed"] == True)
)]
- core = str() if itm["dut"] == "trex" else f"{itm['core']}"
+ core = str() if itm["dut"] == "trex" else itm["core"]
ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
regex_test = \
f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$"
df = df[
- (df.job.str.endswith(f"{topo}-{arch}")) &
+ (df.job.str.endswith(topo_arch)) &
(df.dut_version.str.contains(itm["dutver"].replace(".r", "-r").\
replace("rls", "release"))) &
(df.test_id.str.contains(regex_test, regex=True))
:rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
"""
- def get_y_values(data, y_data_max, param, norm_factor, release=str(),
+ def get_y_values(data, y_data_max, param, norm_factor,
remove_outliers=False):
- if param == "result_receive_rate_rate_values":
- if release in ("rls2402", "rls2406", "rls2410"):
- y_vals_raw = data["result_receive_rate_rate_avg"].to_list()
- else:
- y_vals_raw = data[param].to_list()[0]
- else:
- y_vals_raw = data[param].to_list()
- y_data = [(y * norm_factor) for y in y_vals_raw]
+ y_data = [(y * norm_factor) for y in data[param].to_list()]
if remove_outliers:
try:
if itm_data.empty:
continue
- phy = itm["phy"].split("-")
- topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
- norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
+ phy = itm["phy"].rsplit("-", maxsplit=2)
+ topo_arch = phy[0] if len(phy) == 3 else str()
+ norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY.get(topo_arch, 1.0)) \
if normalize else 1.0
if itm["area"] == "hoststack":
y_tput_max,
C.VALUE_ITER[ttype],
norm_factor,
- itm["rls"],
remove_outliers
)
)
}
- if itm["testtype"] == "mrr" and itm["rls"] == "rls2310":
- trial_run = "trial"
- metadata["csit-ref"] = (
- f"{itm_data['job'].to_list()[0]}/",
- f"{itm_data['build'].to_list()[0]}"
- )
- customdata = [{"metadata": metadata}, ] * nr_of_samples
- else:
- trial_run = "run"
- for _, row in itm_data.iterrows():
- metadata["csit-ref"] = f"{row['job']}/{row['build']}"
- try:
- metadata["hosts"] = ", ".join(row["hosts"])
- except (KeyError, TypeError):
- pass
- customdata.append({"metadata": deepcopy(metadata)})
+ for _, row in itm_data.iterrows():
+ metadata["csit-ref"] = f"{row['job']}/{row['build']}"
+ try:
+ metadata["hosts"] = ", ".join(row["hosts"])
+ except (KeyError, TypeError):
+ pass
+ customdata.append({"metadata": deepcopy(metadata)})
tput_kwargs = dict(
y=y_data,
name=(
f"{idx + 1}. "
f"({nr_of_samples:02d} "
- f"{trial_run}{'s' if nr_of_samples > 1 else ''}) "
+ f"run{'s' if nr_of_samples > 1 else ''}) "
f"{itm['id']}"
),
hoverinfo=u"y+name",
from ..utils.control_panel import ControlPanel
from ..utils.trigger import Trigger
from ..utils.utils import show_tooltip, label, sync_checklists, gen_new_url, \
- generate_options, get_list_group_items, navbar_report, \
+ generate_options, get_list_group_items, navbar_report, get_topo_arch, \
show_iterative_graph_data
from ..utils.url_processing import url_decode
from .graphs import graph_iterative, select_iterative_data
lst_job = row["job"].split("-")
dut = lst_job[1]
d_ver = row["dut_version"]
- tbed = "-".join(lst_job[-2:])
+ tbed = get_topo_arch(lst_job)
lst_test_id = row["test_id"].split(".")
if dut == "dpdk":
area = "dpdk"
from ..utils.trigger import Trigger
from ..utils.utils import gen_new_url, generate_options, navbar_trending, \
filter_table_data, sort_table_data, show_trending_graph_data, \
- show_iterative_graph_data, show_tooltip
+ show_iterative_graph_data, show_tooltip, get_topo_arch
from ..utils.url_processing import url_decode
from .tables import search_table
from ..coverage.tables import coverage_tables
l_id = row["test_id"].split(".")
suite = l_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
replace("2n-", "")
- tb = "-".join(row["job"].split("-")[-2:])
+ tb = get_topo_arch(row["job"].split("-"))
nic = suite.split("-")[0]
for driver in C.DRIVERS:
if driver in suite:
"release-dis": C.STYLE_DONT_DISPLAY,
"release-val": str(),
"help-dis": disabled,
- "help-val": "<testbed> <nic> <driver> " + \
+ "help-val": "<topo> <arch> <nic> <driver> " + \
"<framesize> <cores> <test>",
"search-dis": disabled,
"search-val": str()
ctrl_panel.set({
"release-val": trigger.value,
"help-dis": C.STYLE_DISPLAY,
- "help-val": "<DUT version> <testbed> <nic> " + \
+ "help-val": "<DUT version> <topo> <arch> <nic> " + \
"<driver> <framesize> <core> <test>",
"search-dis": C.STYLE_DISPLAY,
"search-val": str()
(data["dut_version"] == dutver) &
(data["release"] == rls)
)])
-
df = df[df.full_id.str.contains(
f".*{tb}.*{nic}.*{test_name}",
regex=True
import pandas as pd
from ..utils.constants import Constants as C
+from ..utils.utils import get_topo_arch
def select_search_data(data: pd.DataFrame, selection: list) -> pd.DataFrame:
l_id = row["test_id"].split(".")
suite = l_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
replace("2n-", "")
- l_tb.append("-".join(row["job"].split("-")[-2:]))
+ l_tb.append(get_topo_arch(row["job"].split("-")))
l_nic.append(suite.split("-")[0])
if selection["datatype"] != "trending":
l_dutver.append(row["dut_version"])
from ..utils.control_panel import ControlPanel
from ..utils.utils import show_tooltip, gen_new_url, get_ttypes, get_cadences, \
get_test_beds, get_job, generate_options, set_job_params, navbar_trending, \
- get_url_job, get_url_logs
+ get_url_job, get_url_logs, get_topo_arch
from ..utils.url_processing import url_decode
from .graphs import graph_statistics, select_data
"tbed": list()
}
for job in jobs:
- idx = -3 if "-x-" in job else -2
lst_job = job.split("-")
d_job_info["job"].append(job)
d_job_info["dut"].append(lst_job[1])
d_job_info["ttype"].append(lst_job[3])
d_job_info["cadence"].append(lst_job[4])
- d_job_info["tbed"].append("-".join(lst_job[idx:]))
+ d_job_info["tbed"].append(get_topo_arch(lst_job))
self._job_info = pd.DataFrame.from_dict(d_job_info)
self._default = set_job_params(self._job_info, d_job_info["job"][0])
:rtype: pandas.DataFrame
"""
- phy = itm["phy"].split("-")
- if len(phy) == 4:
- topo, arch, nic, drv = phy
+ phy = itm["phy"].rsplit("-", maxsplit=2)
+ if len(phy) == 3:
+ topo_arch, nic, drv = phy
if drv == "dpdk":
drv = ""
else:
(data["test_type"] == test_type) &
(data["passed"] == True)
)]
- df = df[df.job.str.endswith(f"{topo}-{arch}")]
- core = str() if itm["dut"] == "trex" else f"{itm['core']}"
+ df = df[df.job.str.endswith(topo_arch)]
+ core = str() if itm["dut"] == "trex" else itm["core"]
ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
df = df[df.test_id.str.contains(
f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
continue
if normalize:
- phy = itm["phy"].split("-")
- topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
+ phy = itm["phy"].rsplit("-", maxsplit=2)
+ topo_arch = phy[0] if len(phy) == 3 else str()
norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY.get(topo_arch, 1.0)) \
if topo_arch else 1.0
else:
from ..utils.trigger import Trigger
from ..utils.telemetry_data import TelemetryData
from ..utils.utils import show_tooltip, label, sync_checklists, gen_new_url, \
- generate_options, get_list_group_items, navbar_trending, \
+ generate_options, get_list_group_items, navbar_trending, get_topo_arch, \
show_trending_graph_data
from ..utils.url_processing import url_decode
from .graphs import graph_trending, select_trending_data, graph_tm_trending
for _, row in self._data[cols].drop_duplicates().iterrows():
lst_job = row["job"].split("-")
dut = lst_job[1]
- tbed = "-".join(lst_job[-2:])
+ tbed = get_topo_arch(lst_job)
lst_test = row["test_id"].split(".")
if dut == "dpdk":
area = "dpdk"
}
VALUE_ITER = {
- "mrr": "result_receive_rate_rate_values",
+ "mrr": "result_receive_rate_rate_avg",
"ndr": "result_ndr_lower_rate_value",
"pdr": "result_pdr_lower_rate_value",
"mrr-bandwidth": "result_receive_rate_bandwidth_avg",
for _, row in self._unique_metrics_labels.iterrows():
if _is_selected(row["labels"], selection):
lst_items.append(row.to_frame().T)
- self._selected_metrics_labels = \
- pd.concat(lst_items, ignore_index=True, axis=0, copy=False)
+ if len(lst_items) == 1:
+ self._selected_metrics_labels = lst_items[0]
+ elif len(lst_items) > 1:
+ self._selected_metrics_labels = \
+ pd.concat(lst_items, ignore_index=True, axis=0, copy=False)
return self._selected_metrics_labels
def select_tm_trending_data(
return str()
else:
return str()
+
+
+def get_topo_arch(lst_job: list) -> str:
+ """Get the topology and architecture string from the job name.
+
+ :param lst_job: The job name split into substrings.
+ :type lst_job: list
+ :returns: String with information about topology and architecture.
+ :rtype: str
+ """
+
+ if len(lst_job) < 2:
+ return str()
+
+ if "x" in lst_job[1:-1]:
+ # External testbeds.
+ # The structure of the name:
+ # <to be removed>-x-<topo>-<arch>-<other parts separated by dashes>
+ return "-".join(lst_job[(lst_job.index("x") + 1):])
+ else:
+ # Topology and architecture are the last two substrings.
+ return "-".join(lst_job[-2:])
Code Review / csit.git / commitdiff