C-Dash: Add soak tests

Change-Id: Ibe17b7dc2125fa6ace496897a7cf9115a5f2c736
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/csit.infra.dash/app/cdash/comparisons/layout.py b/csit.infra.dash/app/cdash/comparisons/layout.py
index 3aa3239..57a1c2a 100644 (file)
--- a/csit.infra.dash/app/cdash/comparisons/layout.py
+++ b/csit.infra.dash/app/cdash/comparisons/layout.py
@@ -171,6 +171,9 @@ class Layout:
             elif row["test_type"] == "hoststack" and row["tg_type"] == "ab":
                 if "CPS" not in tbs[dut][dver][infra]["ttype"]:
                     tbs[dut][dver][infra]["ttype"].extend(("CPS", "RPS", ))
+            elif row["test_type"] == "soak":
+                if "SOAK" not in tbs[dut][dver][infra]["ttype"]:
+                    tbs[dut][dver][infra]["ttype"].append("SOAK")
         self._tbs = tbs
 
         # Read from files:

diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2402_hoststack b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2402_hoststack
index 993d16c..1e9c708 100644 (file)
Binary files a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2402_hoststack and b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2402_hoststack differ

diff --git a/csit.infra.dash/app/cdash/data/_metadata/trending_soak b/csit.infra.dash/app/cdash/data/_metadata/trending_soak
new file mode 100644 (file)
index 0000000..4502ca4
Binary files /dev/null and b/csit.infra.dash/app/cdash/data/_metadata/trending_soak differ

diff --git a/csit.infra.dash/app/cdash/data/data.yaml b/csit.infra.dash/app/cdash/data/data.yaml
index 16f8dda..ed5fc0b 100644 (file)
--- a/csit.infra.dash/app/cdash/data/data.yaml
+++ b/csit.infra.dash/app/cdash/data/data.yaml
@@ -87,6 +87,27 @@
     - telemetry
     - test_id
     - version
+- data_type: trending
+  partition: test_type
+  partition_name: soak
+  path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/trending
+  schema: trending_soak
+  columns:
+    - job
+    - build
+    - dut_type
+    - dut_version
+    - hosts
+    - tg_type
+    - result_critical_rate_lower_bandwidth_unit
+    - result_critical_rate_lower_bandwidth_value
+    - result_critical_rate_lower_rate_unit
+    - result_critical_rate_lower_rate_value
+    - start_time
+    - passed
+    - telemetry
+    - test_id
+    - version
 - data_type: iterative
   partition: test_type
   partition_name: mrr
@@ -338,6 +359,7 @@
     - build
     - dut_type
     - dut_version
+    # - hosts
     - start_time
     - passed
     - test_id
@@ -361,6 +383,7 @@
     - build
     - dut_type
     - dut_version
+    # - hosts
     - start_time
     - passed
     - test_id
@@ -394,6 +417,7 @@
     - build
     - dut_type
     - dut_version
+    - hosts
     - tg_type
     - result_bandwidth_unit
     - result_bandwidth_value

diff --git a/csit.infra.dash/app/cdash/news/layout.py b/csit.infra.dash/app/cdash/news/layout.py
index b40db48..ba4fc85 100644 (file)
--- a/csit.infra.dash/app/cdash/news/layout.py
+++ b/csit.infra.dash/app/cdash/news/layout.py
@@ -182,20 +182,20 @@ class Layout:
                             _create_test_name(test).replace("-ndrpdr", "-pdr"),
                             x_axis[_get_rindex(anomalies, "regression")]
                         ))
-                else:  # mrr
-                    tst_data = tst_data.dropna(
-                        subset=["result_receive_rate_rate_avg", ]
-                    )
+                else:  # mrr, hoststack, soak
+                    if "soak" in test:
+                        val = "result_critical_rate_lower_rate_value"
+                    elif "hoststack" in test:
+                        val = "result_rate_value"
+                    else:  # mrr
+                        val = "result_receive_rate_rate_avg"
+                    tst_data = tst_data.dropna(subset=[val, ])
                     if tst_data.empty:
                         continue
                     x_axis = tst_data["start_time"].tolist()
                     try:
                         anomalies, _, _ = classify_anomalies({
-                            k: v for k, v in zip(
-                                x_axis,
-                                tst_data["result_receive_rate_rate_avg"].\
-                                    tolist()
-                            )
+                            k: v for k, v in zip(x_axis, tst_data[val].tolist())
                         })
                     except ValueError:
                         continue

diff --git a/csit.infra.dash/app/cdash/report/graphs.py b/csit.infra.dash/app/cdash/report/graphs.py
index 0627411..02202ea 100644 (file)
--- a/csit.infra.dash/app/cdash/report/graphs.py
+++ b/csit.infra.dash/app/cdash/report/graphs.py
@@ -51,6 +51,8 @@ def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
         test_type = "ndrpdr"
     elif itm["testtype"] == "mrr":
         test_type = "mrr"
+    elif itm["testtype"] == "soak":
+        test_type = "soak"
     elif itm["area"] == "hoststack":
         test_type = "hoststack"
     df = data.loc[(
@@ -190,6 +192,10 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
             trial_run = "run"
             for _, row in itm_data.iterrows():
                 metadata["csit-ref"] = f"{row['job']}/{row['build']}"
+                try:
+                    metadata["hosts"] = ", ".join(row["hosts"])
+                except (KeyError, TypeError):
+                    pass
                 customdata.append({"metadata": deepcopy(metadata)})
         tput_kwargs = dict(
             y=y_data,
@@ -207,7 +213,7 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
         )
         tput_traces.append(go.Box(**tput_kwargs))
 
-        if ttype in ("ndr", "pdr", "mrr"):
+        if ttype in C.TESTS_WITH_BANDWIDTH:
             y_band, y_band_max = get_y_values(
                 itm_data,
                 y_band_max,
@@ -237,7 +243,7 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
                 x_band.append(idx + 1)
                 band_traces.append(go.Box(**band_kwargs))
 
-        if ttype == "pdr":
+        if ttype in C.TESTS_WITH_LATENCY:
             y_lat, y_lat_max = get_y_values(
                 itm_data,
                 y_lat_max,
@@ -281,7 +287,7 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
         pl_tput["xaxis"]["ticktext"] = [str(i + 1) for i in range(len(sel))]
         pl_tput["yaxis"]["title"] = f"Throughput [{'|'.join(sorted(y_units))}]"
         if y_tput_max:
-            pl_tput["yaxis"]["range"] = [0, int(y_tput_max) + 2e6]
+            pl_tput["yaxis"]["range"] = [0, int(y_tput_max) * 1.1]
         fig_tput = go.Figure(data=tput_traces, layout=pl_tput)
 
     if band_traces:
@@ -291,7 +297,7 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
         pl_band["yaxis"]["title"] = \
             f"Bandwidth [{'|'.join(sorted(y_band_units))}]"
         if y_band_max:
-            pl_band["yaxis"]["range"] = [0, int(y_band_max) + 2e9]
+            pl_band["yaxis"]["range"] = [0, int(y_band_max) * 1.1]
         fig_band = go.Figure(data=band_traces, layout=pl_band)
 
     if lat_traces:

diff --git a/csit.infra.dash/app/cdash/report/layout.py b/csit.infra.dash/app/cdash/report/layout.py
index 1978e7a..f485894 100644 (file)
--- a/csit.infra.dash/app/cdash/report/layout.py
+++ b/csit.infra.dash/app/cdash/report/layout.py
@@ -166,10 +166,7 @@ class Layout:
                 tst_params["core"].append(core.upper())
             if framesize.upper() not in tst_params["frame-size"]:
                 tst_params["frame-size"].append(framesize.upper())
-            if row["test_type"] == "mrr":
-                if "MRR" not in tst_params["test-type"]:
-                    tst_params["test-type"].append("MRR")
-            elif row["test_type"] == "ndrpdr":
+            if row["test_type"] == "ndrpdr":
                 if "NDR" not in tst_params["test-type"]:
                     tst_params["test-type"].extend(("NDR", "PDR", ))
             elif row["test_type"] == "hoststack" and \
@@ -179,6 +176,9 @@ class Layout:
             elif row["test_type"] == "hoststack" and row["tg_type"] == "ab":
                 if "CPS" not in tst_params["test-type"]:
                     tst_params["test-type"].extend(("CPS", "RPS"))
+            else:  # MRR, SOAK
+                if row["test_type"].upper() not in tst_params["test-type"]:
+                    tst_params["test-type"].append(row["test_type"].upper())
         self._spec_tbs = tbs
 
         # Read from files:

diff --git a/csit.infra.dash/app/cdash/stats/layout.py b/csit.infra.dash/app/cdash/stats/layout.py
index 4e7b72e..655c61c 100644 (file)
--- a/csit.infra.dash/app/cdash/stats/layout.py
+++ b/csit.infra.dash/app/cdash/stats/layout.py
@@ -784,7 +784,7 @@ class Layout:
                             dbc.Badge(lst_itm[0]),
                             html.A(
                                 lst_itm[1],
-                                href=f"{C.URL_JENKINS}{lst_itm[1]}",
+                                href=f"{C.URL_LOGS}{lst_itm[1]}",
                                 target="_blank"
                             )
                         ])

diff --git a/csit.infra.dash/app/cdash/trending/graphs.py b/csit.infra.dash/app/cdash/trending/graphs.py
index ede3a06..1a507df 100644 (file)
--- a/csit.infra.dash/app/cdash/trending/graphs.py
+++ b/csit.infra.dash/app/cdash/trending/graphs.py
@@ -50,6 +50,8 @@ def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
         test_type = "ndrpdr"
     elif itm["testtype"] == "mrr":
         test_type = "mrr"
+    elif itm["testtype"] == "soak":
+        test_type = "soak"
     elif itm["area"] == "hoststack":
         test_type = "hoststack"
     df = data.loc[(
@@ -194,6 +196,24 @@ def graph_trending(
                     f"bandwidth [{row['result_bandwidth_unit']}]: "
                     f"{row['result_bandwidth_value'] * nf:,.0f}<br>"
                 )
+            elif ttype in ("soak", "soak-bandwidth"):
+                h_tput = (
+                    f"tput [{row['result_critical_rate_lower_rate_unit']}]: "
+                    f"{row['result_critical_rate_lower_rate_value'] * nf:,.0f}"
+                    "<br>"
+                )
+                if pd.notna(row["result_critical_rate_lower_bandwidth_value"]):
+                    bv = row['result_critical_rate_lower_bandwidth_value']
+                    h_band = (
+                        "bandwidth "
+                        f"[{row['result_critical_rate_lower_bandwidth_unit']}]:"
+                        f" {bv * nf:,.0f}"
+                        "<br>"
+                    )
+            try:
+                hosts = f"<br>hosts: {', '.join(row['hosts'])}"
+            except (KeyError, TypeError):
+                hosts = str()
             hover_itm = (
                 f"dut: {name_lst[0]}<br>"
                 f"infra: {'-'.join(name_lst[1:5])}<br>"
@@ -201,8 +221,8 @@ def graph_trending(
                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
                 f"{h_tput}{h_band}{h_lat}"
                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
-                f"csit-ref: {row['job']}/{row['build']}<br>"
-                f"hosts: {', '.join(row['hosts'])}"
+                f"csit-ref: {row['job']}/{row['build']}"
+                f"{hosts}"
             )
             hover.append(hover_itm)
             if ttype == "latency":
@@ -231,6 +251,10 @@ def graph_trending(
 
         hover_trend = list()
         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
+            try:
+                hosts = f"<br>hosts: {', '.join(row['hosts'])}"
+            except (KeyError, TypeError):
+                hosts = str()
             hover_itm = (
                 f"dut: {name_lst[0]}<br>"
                 f"infra: {'-'.join(name_lst[1:5])}<br>"
@@ -239,8 +263,8 @@ def graph_trending(
                 f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
                 f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
-                f"csit-ref: {row['job']}/{row['build']}<br>"
-                f"hosts: {', '.join(row['hosts'])}"
+                f"csit-ref: {row['job']}/{row['build']}"
+                f"{hosts}"
             )
             if ttype == "latency":
                 hover_itm = hover_itm.replace("[pps]", "[us]")
@@ -379,7 +403,7 @@ def graph_trending(
                 fig_tput = go.Figure()
             fig_tput.add_traces(traces)
 
-        if ttype in ("ndr", "pdr", "mrr", "hoststack-cps", "hoststack-rps"):
+        if ttype in C.TESTS_WITH_BANDWIDTH:
             traces, _ = _generate_trending_traces(
                 f"{ttype}-bandwidth",
                 itm["id"],
@@ -392,7 +416,7 @@ def graph_trending(
                     fig_band = go.Figure()
                 fig_band.add_traces(traces)
 
-        if ttype in ("pdr", "hoststack-cps", "hoststack-rps"):
+        if ttype in C.TESTS_WITH_LATENCY:
             traces, _ = _generate_trending_traces(
                 "latency" if ttype == "pdr" else "hoststack-latency",
                 itm["id"],

diff --git a/csit.infra.dash/app/cdash/trending/layout.py b/csit.infra.dash/app/cdash/trending/layout.py
index f6f96d7..da90ae2 100644 (file)
--- a/csit.infra.dash/app/cdash/trending/layout.py
+++ b/csit.infra.dash/app/cdash/trending/layout.py
@@ -158,10 +158,7 @@ class Layout:
                 tst_params["core"].append(core.upper())
             if framesize.upper() not in tst_params["frame-size"]:
                 tst_params["frame-size"].append(framesize.upper())
-            if row["test_type"] == "mrr":
-                if "MRR" not in tst_params["test-type"]:
-                    tst_params["test-type"].append("MRR")
-            elif row["test_type"] == "ndrpdr":
+            if row["test_type"] == "ndrpdr":
                 if "NDR" not in tst_params["test-type"]:
                     tst_params["test-type"].extend(("NDR", "PDR"))
             elif row["test_type"] == "hoststack":
@@ -171,6 +168,9 @@ class Layout:
                 elif row["tg_type"] == "ab":
                     if "CPS" not in tst_params["test-type"]:
                         tst_params["test-type"].extend(("CPS", "RPS"))
+            else:  # MRR, SOAK
+                if row["test_type"].upper() not in tst_params["test-type"]:
+                    tst_params["test-type"].append(row["test_type"].upper())
         self._spec_tbs = tbs
 
         # Read from files:

diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 5ed5a8c..8407664 100644 (file)
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -47,6 +47,9 @@ class Constants:
     # URL to Jenkins
     URL_JENKINS = "https://jenkins.fd.io/job/"
 
+    # URL to logs
+    URL_LOGS = "https://s3-logs.fd.io/vex-yul-rot-jenkins-1/"
+
     # URL to the documentation
     URL_DOC_TRENDING = "https://csit.fd.io/cdocs/methodology/trending/analysis/"
     URL_DOC_REL_NOTES = "https://csit.fd.io/cdocs/release_notes/current/"
@@ -242,7 +245,9 @@ class Constants:
         "hoststack-cps-bandwidth": "result_bandwidth_value",
         "hoststack-rps-bandwidth": "result_bandwidth_value",
         "hoststack-bps": "result_bandwidth_value",
-        "hoststack-latency": "result_latency_value"
+        "hoststack-latency": "result_latency_value",
+        "soak": "result_critical_rate_lower_rate_value",
+        "soak-bandwidth": "result_critical_rate_lower_bandwidth_value"
     }
 
     VALUE_ITER = {
@@ -255,8 +260,12 @@ class Constants:
         "latency": "result_latency_forward_pdr_50_avg",
         "hoststack-cps": "result_rate_value",
         "hoststack-rps": "result_rate_value",
+        "hoststack-cps-bandwidth": "result_bandwidth_value",
+        "hoststack-rps-bandwidth": "result_bandwidth_value",
         "hoststack-bps": "result_bandwidth_value",
-        "hoststack-latency": "result_latency_value"
+        "hoststack-latency": "result_latency_value",
+        "soak": "result_critical_rate_lower_rate_value",
+        "soak-bandwidth": "result_critical_rate_lower_bandwidth_value"
     }
 
     UNIT = {
@@ -272,9 +281,25 @@ class Constants:
         "hoststack-cps-bandwidth": "result_bandwidth_unit",
         "hoststack-rps-bandwidth": "result_bandwidth_unit",
         "hoststack-bps": "result_bandwidth_unit",
-        "hoststack-latency": "result_latency_unit"
+        "hoststack-latency": "result_latency_unit",
+        "soak": "result_critical_rate_lower_rate_unit",
+        "soak-bandwidth": "result_critical_rate_lower_bandwidth_unit"
     }
 
+    TESTS_WITH_BANDWIDTH = (
+        "ndr",
+        "pdr",
+        "mrr",
+        "hoststack-cps",
+        "hoststack-rps",
+        "soak"
+    )
+    TESTS_WITH_LATENCY = (
+        "pdr",
+        "hoststack-cps",
+        "hoststack-rps"
+    )
+
     # Latencies.
     LAT_HDRH = (  # Do not change the order
         "result_latency_forward_pdr_0_hdrh",

diff --git a/csit.infra.dash/app/cdash/utils/utils.py b/csit.infra.dash/app/cdash/utils/utils.py
index 692e45e..306b4f6 100644 (file)
--- a/csit.infra.dash/app/cdash/utils/utils.py
+++ b/csit.infra.dash/app/cdash/utils/utils.py
@@ -710,7 +710,7 @@ def show_trending_graph_data(
                 dbc.Badge(lst_itm[0]),
                 html.A(
                     lst_itm[1],
-                    href=f"{C.URL_JENKINS}{lst_itm[1]}",
+                    href=f"{C.URL_LOGS}{lst_itm[1]}",
                     target="_blank"
                 )
             ])
@@ -842,7 +842,7 @@ def show_iterative_graph_data(
                 continue
             list_group_item = dbc.ListGroupItem([
                 dbc.Badge(k),
-                html.A(v, href=f"{C.URL_JENKINS}{v}", target="_blank")
+                html.A(v, href=f"{C.URL_LOGS}{v}", target="_blank")
             ])
         else:
             list_group_item = dbc.ListGroupItem([dbc.Badge(k), v])