PAL: Process Hoststack data

Change-Id: Ib87ad83f3bbb7e71b51b91b9b3e8dc963a76a047
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/docs/report/index.html.template b/docs/report/index.html.template
index f6fcc76..6431290 100644 (file)
--- a/docs/report/index.html.template
+++ b/docs/report/index.html.template
@@ -22,13 +22,13 @@ CSIT-2001
     vpp_performance_tests/packet_latency/index
     vpp_performance_tests/reconf_tests/index
     vpp_performance_tests/nf_service_density/index
     vpp_performance_tests/packet_latency/index
     vpp_performance_tests/reconf_tests/index
     vpp_performance_tests/nf_service_density/index

+    vpp_performance_tests/hoststack_testing/index

     vpp_performance_tests/comparisons/index
     vpp_performance_tests/throughput_trending
     vpp_performance_tests/test_environment
     vpp_performance_tests/documentation/index
 
 ..
     vpp_performance_tests/comparisons/index
     vpp_performance_tests/throughput_trending
     vpp_performance_tests/test_environment
     vpp_performance_tests/documentation/index
 
 ..

-    vpp_performance_tests/hoststack_testing/index

     vpp_performance_tests/soak_tests/index
 
 .. toctree::
     vpp_performance_tests/soak_tests/index
 
 .. toctree::

diff --git a/docs/report/introduction/report_history.rst b/docs/report/introduction/report_history.rst
index b2199ed..e07fc53 100644 (file)
--- a/docs/report/introduction/report_history.rst
+++ b/docs/report/introduction/report_history.rst
@@ -23,6 +23,9 @@ FD.io CSIT-2001 Report history and per .[ww] revision changes are listed below.
 |                |                                                            |
 |                |    - 2n-clx                                                |
 |                |                                                            |
 |                |                                                            |
 |                |    - 2n-clx                                                |
 |                |                                                            |

+|                | 3. Added Hoststack tests                                   |
+|                |                                                            |
+|                | 4. Edited Test Methodology --> Hoststack Testing           |

 |                |                                                            |
 +----------------+------------------------------------------------------------+
 | .09            | 1. Added data:                                             |
 |                |                                                            |
 +----------------+------------------------------------------------------------+
 | .09            | 1. Added data:                                             |

diff --git a/docs/report/vpp_performance_tests/hoststack_testing/index.rst b/docs/report/vpp_performance_tests/hoststack_testing/index.rst
index 46390b5..fef315a 100644 (file)
--- a/docs/report/vpp_performance_tests/hoststack_testing/index.rst
+++ b/docs/report/vpp_performance_tests/hoststack_testing/index.rst
@@ -3,8 +3,8 @@ Hoststack Testing
 
 .. toctree::
 
 
 .. toctree::
 

-    http_server_performance/index
-
-..

     iperf3/index
     quic/index
     iperf3/index
     quic/index

+
+..
+    http_server_performance/index

diff --git a/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst b/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst
index b871f58..83c9d63 100644 (file)
--- a/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst
+++ b/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst
@@ -1,2 +1,75 @@
+
+.. raw:: latex
+
+    \clearpage
+
+.. raw:: html
+
+    <script type="text/javascript">
+
+        function getDocHeight(doc) {
+            doc = doc || document;
+            var body = doc.body, html = doc.documentElement;
+            var height = Math.max( body.scrollHeight, body.offsetHeight,
+                html.clientHeight, html.scrollHeight, html.offsetHeight );
+            return height;
+        }
+
+        function setIframeHeight(id) {
+            var ifrm = document.getElementById(id);
+            var doc = ifrm.contentDocument? ifrm.contentDocument:
+                ifrm.contentWindow.document;
+            ifrm.style.visibility = 'hidden';
+            ifrm.style.height = "10px"; // reset to minimal height ...
+            // IE opt. for bing/msn needs a bit added or scrollbar appears
+            ifrm.style.height = getDocHeight( doc ) + 4 + "px";
+            ifrm.style.visibility = 'visible';
+        }
+
+    </script>
+

 TCP/IP with iperf3
 TCP/IP with iperf3

-------------------
+~~~~~~~~~~~~~~~~~~
+
+.. todo::
+    Add introduction
+
+.. raw:: latex
+
+    \clearpage
+
+9000b-1t1c-xl710-base-scale
+---------------------------
+
+.. raw:: html
+
+    <iframe id="ifrm01" onload="setIframeHeight(this.id)" width="700" frameborder="0" scrolling="no" src="../../../_static/vpp/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps.html"></iframe>
+
+.. raw:: latex
+
+    \begin{figure}[H]
+        \centering
+            \graphicspath{{../../_build/_static/vpp/}}
+            \includegraphics[clip, trim=0cm 0cm 5cm 0cm, width=0.70\textwidth]{3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps}
+            \label{fig:3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps}
+    \end{figure}
+
+.. raw:: latex
+
+    \clearpage
+
+9000b-1t1c-xl710-nsim-base-scale
+--------------------------------
+
+.. raw:: html
+
+    <iframe id="ifrm02" onload="setIframeHeight(this.id)" width="700" frameborder="0" scrolling="no" src="../../../_static/vpp/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps.html"></iframe>
+
+.. raw:: latex
+
+    \begin{figure}[H]
+        \centering
+            \graphicspath{{../../_build/_static/vpp/}}
+            \includegraphics[clip, trim=0cm 0cm 5cm 0cm, width=0.70\textwidth]{3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps}
+            \label{fig:3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps}
+    \end{figure}

diff --git a/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst b/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst
index 9cf6bef..09a89a9 100644 (file)
--- a/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst
+++ b/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst
@@ -1,2 +1,55 @@
+
+.. raw:: latex
+
+    \clearpage
+
+.. raw:: html
+
+    <script type="text/javascript">
+
+        function getDocHeight(doc) {
+            doc = doc || document;
+            var body = doc.body, html = doc.documentElement;
+            var height = Math.max( body.scrollHeight, body.offsetHeight,
+                html.clientHeight, html.scrollHeight, html.offsetHeight );
+            return height;
+        }
+
+        function setIframeHeight(id) {
+            var ifrm = document.getElementById(id);
+            var doc = ifrm.contentDocument? ifrm.contentDocument:
+                ifrm.contentWindow.document;
+            ifrm.style.visibility = 'hidden';
+            ifrm.style.height = "10px"; // reset to minimal height ...
+            // IE opt. for bing/msn needs a bit added or scrollbar appears
+            ifrm.style.height = getDocHeight( doc ) + 4 + "px";
+            ifrm.style.visibility = 'visible';
+        }
+
+    </script>
+

 QUIC(picotls)/UDP/IP with vpp_echo
 QUIC(picotls)/UDP/IP with vpp_echo

-----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. todo::
+    Add introduction
+
+.. raw:: latex
+
+    \clearpage
+
+9000b-1t1c-xl710-base-scale
+---------------------------
+
+.. raw:: html
+
+    <iframe id="ifrm01" onload="setIframeHeight(this.id)" width="700" frameborder="0" scrolling="no" src="../../../_static/vpp/3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps.html"></iframe>
+
+.. raw:: latex
+
+    \begin{figure}[H]
+        \centering
+            \graphicspath{{../../_build/_static/vpp/}}
+            \includegraphics[clip, trim=0cm 0cm 5cm 0cm, width=0.70\textwidth]{3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps}
+            \label{fig:3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps}
+    \end{figure}

diff --git a/docs/report/vpp_performance_tests/index.rst b/docs/report/vpp_performance_tests/index.rst
index 36aed84..2ddf9c8 100644 (file)
--- a/docs/report/vpp_performance_tests/index.rst
+++ b/docs/report/vpp_performance_tests/index.rst
@@ -10,11 +10,11 @@ VPP Performance
     packet_latency/index
     reconf_tests/index
     nf_service_density/index
     packet_latency/index
     reconf_tests/index
     nf_service_density/index

+    hoststack_testing/index

     comparisons/index
     throughput_trending
     test_environment
     documentation/index
 
 ..
     comparisons/index
     throughput_trending
     test_environment
     documentation/index
 
 ..

-    hoststack_testing/index

     soak_tests/index
     soak_tests/index

diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py
index 5c96989..40993cd 100644 (file)
--- a/resources/tools/presentation/generator_plots.py
+++ b/resources/tools/presentation/generator_plots.py
@@ -622,13 +622,14 @@ def plot_perf_box_name(plot, input_data):
         f"{plot.get(u'title', u'')}."
     )
     data = input_data.filter_tests_by_name(
         f"{plot.get(u'title', u'')}."
     )
     data = input_data.filter_tests_by_name(

-        plot, params=[u"throughput", u"parent", u"tags", u"type"])
+        plot, params=[u"throughput", u"result", u"parent", u"tags", u"type"])

     if data is None:
         logging.error(u"No data.")
         return
 
     # Prepare the data for the plot
     y_vals = OrderedDict()
     if data is None:
         logging.error(u"No data.")
         return
 
     # Prepare the data for the plot
     y_vals = OrderedDict()

+    test_type = u""

     for job in data:
         for build in job:
             for test in build:
     for job in data:
         for build in job:
             for test in build:


@@ -639,13 +640,30 @@ def plot_perf_box_name(plot, input_data):
                             u"-pdr" in plot.get(u"title", u"").lower()):
                         y_vals[test[u"parent"]].\
                             append(test[u"throughput"][u"PDR"][u"LOWER"])
                             u"-pdr" in plot.get(u"title", u"").lower()):
                         y_vals[test[u"parent"]].\
                             append(test[u"throughput"][u"PDR"][u"LOWER"])

+                        test_type = u"NDRPDR"

                     elif (test[u"type"] in (u"NDRPDR", ) and
                           u"-ndr" in plot.get(u"title", u"").lower()):
                         y_vals[test[u"parent"]]. \
                             append(test[u"throughput"][u"NDR"][u"LOWER"])
                     elif (test[u"type"] in (u"NDRPDR", ) and
                           u"-ndr" in plot.get(u"title", u"").lower()):
                         y_vals[test[u"parent"]]. \
                             append(test[u"throughput"][u"NDR"][u"LOWER"])

+                        test_type = u"NDRPDR"

                     elif test[u"type"] in (u"SOAK", ):
                         y_vals[test[u"parent"]].\
                             append(test[u"throughput"][u"LOWER"])
                     elif test[u"type"] in (u"SOAK", ):
                         y_vals[test[u"parent"]].\
                             append(test[u"throughput"][u"LOWER"])

+                        test_type = u"SOAK"
+                    elif test[u"type"] in (u"HOSTSTACK", ):
+                        if u"LDPRELOAD" in test[u"tags"]:
+                            y_vals[test[u"parent"]].append(
+                                float(test[u"result"][u"bits_per_second"]) / 1e3
+                            )
+                        elif u"VPPECHO" in test[u"tags"]:
+                            y_vals[test[u"parent"]].append(
+                                (float(test[u"result"][u"client"][u"tx_data"])
+                                 * 8 / 1e3) /
+                                ((float(test[u"result"][u"client"][u"time"]) +
+                                  float(test[u"result"][u"server"][u"time"])) /
+                                 2)
+                            )
+                        test_type = u"HOSTSTACK"

                     else:
                         continue
                 except (KeyError, TypeError):
                     else:
                         continue
                 except (KeyError, TypeError):


@@ -696,7 +714,10 @@ def plot_perf_box_name(plot, input_data):
         # Create plot
         layout = deepcopy(plot[u"layout"])
         if layout.get(u"title", None):
         # Create plot
         layout = deepcopy(plot[u"layout"])
         if layout.get(u"title", None):

-            layout[u"title"] = f"<b>Throughput:</b> {layout[u'title']}"
+            if test_type in (u"HOSTSTACK", ):
+                layout[u"title"] = f"<b>Bandwidth:</b> {layout[u'title']}"
+            else:
+                layout[u"title"] = f"<b>Throughput:</b> {layout[u'title']}"

         if y_max:
             layout[u"yaxis"][u"range"] = [0, max(y_max)]
         plpl = plgo.Figure(data=traces, layout=layout)
         if y_max:
             layout[u"yaxis"][u"range"] = [0, max(y_max)]
         plpl = plgo.Figure(data=traces, layout=layout)

diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index f7869d8..c43d460 100644 (file)
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -29,6 +29,7 @@ from os import remove
 from datetime import datetime as dt
 from datetime import timedelta
 from json import loads
 from datetime import datetime as dt
 from datetime import timedelta
 from json import loads

+from json.decoder import JSONDecodeError

 
 import hdrh.histogram
 import hdrh.codec
 
 import hdrh.histogram
 import hdrh.codec


@@ -871,6 +872,40 @@ class ExecutionChecker(ResultVisitor):
 
         return latency, u"FAIL"
 
 
         return latency, u"FAIL"
 

+    @staticmethod
+    def _get_hoststack_data(msg, tags):
+        """Get data from the hoststack test message.
+
+        :param msg: The test message to be parsed.
+        :param tags: Test tags.
+        :type msg: str
+        :type tags: list
+        :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
+        :rtype: tuple(dict, str)
+        """
+        result = dict()
+        status = u"FAIL"
+
+        msg = msg.replace(u"'", u'"').replace(u" ", u"")
+        if u"LDPRELOAD" in tags:
+            try:
+                result = loads(msg)
+                status = u"PASS"
+            except JSONDecodeError:
+                pass
+        elif u"VPPECHO" in tags:
+            try:
+                msg_lst = msg.replace(u"}{", u"} {").split(u" ")
+                result = dict(
+                    client=loads(msg_lst[0]),
+                    server=loads(msg_lst[1])
+                )
+                status = u"PASS"
+            except (JSONDecodeError, IndexError):
+                pass
+
+        return result, status
+

     def visit_suite(self, suite):
         """Implements traversing through the suite and its direct children.
 
     def visit_suite(self, suite):
         """Implements traversing through the suite and its direct children.
 


@@ -1040,6 +1075,10 @@ class ExecutionChecker(ResultVisitor):
                 test_result[u"type"] = u"SOAK"
                 test_result[u"throughput"], test_result[u"status"] = \
                     self._get_plr_throughput(test.message)
                 test_result[u"type"] = u"SOAK"
                 test_result[u"throughput"], test_result[u"status"] = \
                     self._get_plr_throughput(test.message)

+            elif u"HOSTSTACK" in tags:
+                test_result[u"type"] = u"HOSTSTACK"
+                test_result[u"result"], test_result[u"status"] = \
+                    self._get_hoststack_data(test.message, tags)

             elif u"TCP" in tags:
                 test_result[u"type"] = u"TCP"
                 groups = re.search(self.REGEX_TCP, test.message)
             elif u"TCP" in tags:
                 test_result[u"type"] = u"TCP"
                 groups = re.search(self.REGEX_TCP, test.message)


@@ -1152,7 +1191,8 @@ class ExecutionChecker(ResultVisitor):
                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
             self._msg_type = u"test-show-runtime"
             self._sh_run_counter += 1
                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
             self._msg_type = u"test-show-runtime"
             self._sh_run_counter += 1

-        elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
+        elif test_kw.name.count(u"Install Dpdk Test On All Duts") and \
+                not self._version:

             self._msg_type = u"dpdk-version"
         else:
             return
             self._msg_type = u"dpdk-version"
         else:
             return

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 46e5ce1..c334f1a 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -247,6 +247,11 @@
         - 23  # NDRPDR sel
         - 24  # NDRPDR sel
 
         - 23  # NDRPDR sel
         - 24  # NDRPDR sel
 

+    plot-vpp-hoststack-3n-hsw:
+      csit-vpp-perf-verify-2001-3n-hsw:
+        - 87  # hoststack
+        - 88  # hoststack
+

     plot-dpdk-hdrh-lat-3n-hsw:
       csit-dpdk-perf-verify-2001-3n-hsw:
         - 16  # NDRPDR sel
     plot-dpdk-hdrh-lat-3n-hsw:
       csit-dpdk-perf-verify-2001-3n-hsw:
         - 16  # NDRPDR sel


@@ -2156,6 +2161,8 @@
       - 82  # NDRPDR full
       - 84  # NDRPDR full
       - 85  # NDRPDR full
       - 82  # NDRPDR full
       - 84  # NDRPDR full
       - 85  # NDRPDR full

+      - 87  # hoststack
+      - 88  # hoststack

 
 #    csit-vpp-perf-verify-1904-3n-skx:
 #      - 7   # MRR sel
 
 #    csit-vpp-perf-verify-1904-3n-skx:
 #      - 7   # MRR sel


@@ -5913,6 +5920,46 @@
 ###                                P L O T S                                 ###
 ################################################################################
 
 ###                                P L O T S                                 ###
 ################################################################################
 

+# Hoststack tests 3n-hsw
+
+- type: "plot"
+  title: "Bandwidth: 3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3"
+  algorithm: "plot_perf_box_name"
+  output-file: "{DIR[STATIC,VPP]}/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps"
+  data: "plot-vpp-hoststack-3n-hsw"
+  include:
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpbase-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpbase-ldpreload-iperf3-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpscale1Cl10S-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps"
+  layout:
+    title: "3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3"
+    layout: "plot-hoststack"
+
+- type: "plot"
+  title: "Bandwidth: 3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3"
+  algorithm: "plot_perf_box_name"
+  output-file: "{DIR[STATIC,VPP]}/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps"
+  data: "plot-vpp-hoststack-3n-hsw"
+  include:
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpbase-Nsim-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpscale1Cl10S-Nsim-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps"
+  layout:
+    title: "3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3"
+    layout: "plot-hoststack"
+
+- type: "plot"
+  title: "Bandwidth: 3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho"
+  algorithm: "plot_perf_box_name"
+  output-file: "{DIR[STATIC,VPP]}/3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps"
+  data: "plot-vpp-hoststack-3n-hsw"
+  include:
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicbase-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicbase-vppecho-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicscale1Cl10S-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicscale1cl10s-vppecho-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicscale10Cl1S-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicscale10cl1s-vppecho-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicscale10Cl10S-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicscale10cl10s-vppecho-bps"
+  layout:
+    title: "3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho"
+    layout: "plot-hoststack"
+

 ## Plots VPP HTTP Server Performance
 #- type: "plot"
 #  title: "VPP HTTP Server Performance"
 ## Plots VPP HTTP Server Performance
 #- type: "plot"
 #  title: "VPP HTTP Server Performance"