CSIT-1350: Add new data to 1810 report

Change-Id: I1c82729c2abc7e5d3699f254629da6a9095d3464
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py
index f406539..f936056 100644 (file)
--- a/resources/tools/presentation/generator_plots.py
+++ b/resources/tools/presentation/generator_plots.py
@@ -107,7 +107,7 @@ def plot_performance_box(plot, input_data):
         y_sorted = OrderedDict()
         y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()}
         for tag in order:
-            logging.info(tag)
+            logging.debug(tag)
             for suite, tags in y_tags_l.items():
                 if "not " in tag:
                     tag = tag.split(" ")[-1]
@@ -119,9 +119,9 @@ def plot_performance_box(plot, input_data):
                 try:
                     y_sorted[suite] = y_vals.pop(suite)
                     y_tags_l.pop(suite)
-                    logging.info(suite)
+                    logging.debug(suite)
                 except KeyError as err:
-                    logging.error("Not found: {0}".format(err))
+                    logging.error("Not found: {0}".format(repr(err)))
                 finally:
                     break
     else:
@@ -144,7 +144,7 @@ def plot_performance_box(plot, input_data):
     for i, col in enumerate(df.columns):
         name = "{0}. {1}".format(i + 1, col.lower().replace('-ndrpdrdisc', '').
                                  replace('-ndrpdr', ''))
-        logging.info(name)
+        logging.debug(name)
         traces.append(plgo.Box(x=[str(i + 1) + '.'] * len(df[col]),
                                y=[y / 1000000 if y else None for y in df[col]],
                                name=name,
@@ -152,7 +152,7 @@ def plot_performance_box(plot, input_data):
         try:
             val_max = max(df[col])
         except ValueError as err:
-            logging.error(err)
+            logging.error(repr(err))
             continue
         if val_max:
             y_max.append(int(val_max / 1000000) + 1)
@@ -175,7 +175,7 @@ def plot_performance_box(plot, input_data):
                                             plot["output-file-type"]))
     except PlotlyError as err:
         logging.error("   Finished with error: {}".
-                      format(str(err).replace("\n", " ")))
+                      format(repr(err).replace("\n", " ")))
         return
 
 
@@ -204,6 +204,11 @@ def plot_latency_error_bars(plot, input_data):
     for job in data:
         for build in job:
             for test in build:
+                try:
+                    logging.debug("test['latency']: {0}\n".
+                                 format(test["latency"]))
+                except ValueError as err:
+                    logging.warning(repr(err))
                 if y_tmp_vals.get(test["parent"], None) is None:
                     y_tmp_vals[test["parent"]] = [
                         list(),  # direction1, min
@@ -221,6 +226,8 @@ def plot_latency_error_bars(plot, input_data):
                         elif "-ndr" in plot_title.lower():
                             ttype = "NDR"
                         else:
+                            logging.warning("Invalid test type: {0}".
+                                            format(test["type"]))
                             continue
                         y_tmp_vals[test["parent"]][0].append(
                             test["latency"][ttype]["direction1"]["min"])
@@ -235,17 +242,19 @@ def plot_latency_error_bars(plot, input_data):
                         y_tmp_vals[test["parent"]][5].append(
                             test["latency"][ttype]["direction2"]["max"])
                     else:
+                        logging.warning("Invalid test type: {0}".
+                                        format(test["type"]))
                         continue
-                except (KeyError, TypeError):
-                    pass
-
+                except (KeyError, TypeError) as err:
+                    logging.warning(repr(err))
+    logging.debug("y_tmp_vals: {0}\n".format(y_tmp_vals))
     # Sort the tests
     order = plot.get("sort", None)
     if order and y_tags:
         y_sorted = OrderedDict()
         y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()}
         for tag in order:
-            logging.info(tag)
+            logging.debug(tag)
             for suite, tags in y_tags_l.items():
                 if "not " in tag:
                     tag = tag.split(" ")[-1]
@@ -257,14 +266,15 @@ def plot_latency_error_bars(plot, input_data):
                 try:
                     y_sorted[suite] = y_tmp_vals.pop(suite)
                     y_tags_l.pop(suite)
-                    logging.info(suite)
+                    logging.debug(suite)
                 except KeyError as err:
-                    logging.error("Not found: {0}".format(err))
+                    logging.error("Not found: {0}".format(repr(err)))
                 finally:
                     break
     else:
         y_sorted = y_tmp_vals
 
+    logging.debug("y_sorted: {0}\n".format(y_sorted))
     x_vals = list()
     y_vals = list()
     y_mins = list()
@@ -280,6 +290,10 @@ def plot_latency_error_bars(plot, input_data):
         y_mins.append(mean(val[3]) if val[3] else None)
         y_maxs.append(mean(val[5]) if val[5] else None)
 
+    logging.debug("x_vals :{0}\n".format(x_vals))
+    logging.debug("y_vals :{0}\n".format(y_vals))
+    logging.debug("y_mins :{0}\n".format(y_mins))
+    logging.debug("y_maxs :{0}\n".format(y_maxs))
     traces = list()
     annotations = list()
 
@@ -306,6 +320,9 @@ def plot_latency_error_bars(plot, input_data):
             arrayminus = [y_vals[idx] - y_mins[idx], ]
         else:
             arrayminus = [None, ]
+        logging.debug("y_vals[{1}] :{0}\n".format(y_vals[idx], idx))
+        logging.debug("array :{0}\n".format(array))
+        logging.debug("arrayminus :{0}\n".format(arrayminus))
         traces.append(plgo.Scatter(
             x=[idx, ],
             y=[y_vals[idx], ],

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 6b162f1..f379218 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -320,7 +320,6 @@
       - 27  # NDRPDR sel
 
     plot-vpp-http-server-performance:
-      # TODO: Add data:
       csit-vpp-perf-verify-1810-3n-hsw:
       - 28  # WRK
       - 29  # WRK
@@ -330,6 +329,8 @@
       - 33  # WRK
       - 34  # WRK
       - 35  # WRK
+      - 36  # WRK
+      - 37  # WRK
 
     # VPP Functional
     vpp-func-results-ubuntu:
@@ -337,7 +338,6 @@
       - 44
 
     vpp-func-results-centos:
-      # TODO: Replace with 1810 data:
       csit-vpp-functional-1810-centos7-virl:
       - 44
 
@@ -392,9 +392,14 @@
       - 2   # NDRPDR sel
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
+      - 5   # NDRPDR sel
+      - 6   # NDRPDR sel
+      - 7   # NDRPDR sel
+      - 8   # NDRPDR sel
+      - 9   # NDRPDR sel
+      - 10  # NDRPDR sel
 
     dpdk-compare-testbeds:
-      # TODO: Add data
       csit-dpdk-perf-verify-1810-3n-skx:
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
@@ -404,11 +409,19 @@
       - 9   # NDRPDR sel
       - 10  # NDRPDR sel
       - 11  # NDRPDR sel
+      - 12  # NDRPDR sel
+      - 13  # NDRPDR sel
       csit-dpdk-perf-verify-1810-3n-hsw:
       - 1   # NDRPDR sel
       - 2   # NDRPDR sel
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
+      - 5   # NDRPDR sel
+      - 6   # NDRPDR sel
+      - 7   # NDRPDR sel
+      - 8   # NDRPDR sel
+      - 9   # NDRPDR sel
+      - 10  # NDRPDR sel
 
     dpdk-perf-results-3n-hsw:
       csit-dpdk-perf-verify-1810-3n-hsw:
@@ -423,15 +436,19 @@
       - 3
 
     plot-dpdk-throughput-latency-3n-hsw:
-      # TODO: Add data:
       csit-dpdk-perf-verify-1810-3n-hsw:
       - 1   # NDRPDR sel
       - 2   # NDRPDR sel
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
+      - 5   # NDRPDR sel
+      - 6   # NDRPDR sel
+      - 7   # NDRPDR sel
+      - 8   # NDRPDR sel
+      - 9   # NDRPDR sel
+      - 10  # NDRPDR sel
 
     plot-dpdk-throughput-latency-3n-skx:
-      # TODO: Add data:
       csit-dpdk-perf-verify-1810-3n-skx:
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
@@ -441,9 +458,10 @@
       - 9   # NDRPDR sel
       - 10  # NDRPDR sel
       - 11  # NDRPDR sel
+      - 12  # NDRPDR sel
+      - 13  # NDRPDR sel
 
     plot-dpdk-throughput-latency-2n-skx:
-      # TODO: Add data:
       csit-dpdk-perf-verify-1810-2n-skx:
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
@@ -860,6 +878,8 @@
     - 33  # WRK
     - 34  # WRK
     - 35  # WRK
+    - 36  # WRK
+    - 37  # WRK
     csit-vpp-perf-verify-1807-3n-skx:
     - 31  # MRR full
     - 28  # MRR sel
@@ -992,12 +1012,17 @@
     - 21
     - 22
     - 23
-    # TODO: Add data:
     csit-dpdk-perf-verify-1810-3n-hsw:
     - 1   # NDRPDR sel
     - 2   # NDRPDR sel
     - 3   # NDRPDR sel
     - 4   # NDRPDR sel
+    - 5   # NDRPDR sel
+    - 6   # NDRPDR sel
+    - 7   # NDRPDR sel
+    - 8   # NDRPDR sel
+    - 9   # NDRPDR sel
+    - 10  # NDRPDR sel
     csit-dpdk-perf-verify-1807-3n-skx:
     - 1
     - 2
@@ -1009,7 +1034,6 @@
     - 10
     - 11
     - 12
-    # TODO: Add data:
     csit-dpdk-perf-verify-1810-3n-skx:
     - 3   # NDRPDR sel
     - 4   # NDRPDR sel
@@ -1019,7 +1043,8 @@
     - 9   # NDRPDR sel
     - 10  # NDRPDR sel
     - 11  # NDRPDR sel
-    # TODO: Add data:
+    - 12  # NDRPDR sel
+    - 13  # NDRPDR sel
     csit-dpdk-perf-verify-1810-2n-skx:
     - 3   # NDRPDR sel
     - 4   # NDRPDR sel
@@ -2066,12 +2091,17 @@
   compare:
     title: "rls1810"
     data:
-      # TODO: Add data
       csit-dpdk-perf-verify-1810-3n-hsw:
       - 1   # NDRPDR sel
       - 2   # NDRPDR sel
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
+      - 5   # NDRPDR sel
+      - 6   # NDRPDR sel
+      - 7   # NDRPDR sel
+      - 8   # NDRPDR sel
+      - 9   # NDRPDR sel
+      - 10  # NDRPDR sel
   data: "dpdk-performance-changes"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'1T1C' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -2118,12 +2148,17 @@
   compare:
     title: "rls1810"
     data:
-      # TODO: Add data
       csit-dpdk-perf-verify-1810-3n-hsw:
       - 1   # NDRPDR sel
       - 2   # NDRPDR sel
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
+      - 5   # NDRPDR sel
+      - 6   # NDRPDR sel
+      - 7   # NDRPDR sel
+      - 8   # NDRPDR sel
+      - 9   # NDRPDR sel
+      - 10  # NDRPDR sel
   data: "dpdk-performance-changes"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T2C' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -2170,12 +2205,17 @@
   compare:
     title: "rls1810"
     data:
-      # TODO: Add data
       csit-dpdk-perf-verify-1810-3n-hsw:
       - 1   # NDRPDR sel
       - 2   # NDRPDR sel
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
+      - 5   # NDRPDR sel
+      - 6   # NDRPDR sel
+      - 7   # NDRPDR sel
+      - 8   # NDRPDR sel
+      - 9   # NDRPDR sel
+      - 10  # NDRPDR sel
   data: "dpdk-performance-changes"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'1T1C' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -2222,12 +2262,17 @@
   compare:
     title: "rls1810"
     data:
-      # TODO: Add data
       csit-dpdk-perf-verify-1810-3n-hsw:
       - 1   # NDRPDR sel
       - 2   # NDRPDR sel
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
+      - 5   # NDRPDR sel
+      - 6   # NDRPDR sel
+      - 7   # NDRPDR sel
+      - 8   # NDRPDR sel
+      - 9   # NDRPDR sel
+      - 10  # NDRPDR sel
   data: "dpdk-performance-changes"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T2C' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -2245,16 +2290,20 @@
   reference:
     title: "3-Node Hsw"
     data:
-      # TODO: Add data
       csit-dpdk-perf-verify-1810-3n-hsw:
       - 1   # NDRPDR sel
       - 2   # NDRPDR sel
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
+      - 5   # NDRPDR sel
+      - 6   # NDRPDR sel
+      - 7   # NDRPDR sel
+      - 8   # NDRPDR sel
+      - 9   # NDRPDR sel
+      - 10  # NDRPDR sel
   compare:
     title: "3-Node Skx"
     data:
-      # TODO: Add data
       csit-dpdk-perf-verify-1810-3n-skx:
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
@@ -2264,6 +2313,8 @@
       - 9   # NDRPDR sel
       - 10  # NDRPDR sel
       - 11  # NDRPDR sel
+      - 12  # NDRPDR sel
+      - 13  # NDRPDR sel
   data: "dpdk-compare-testbeds"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'NIC_Intel-X710' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -2281,16 +2332,20 @@
   reference:
     title: "3-Node Hsw"
     data:
-      # TODO: Add data
       csit-dpdk-perf-verify-1810-3n-hsw:
       - 1   # NDRPDR sel
       - 2   # NDRPDR sel
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
+      - 5   # NDRPDR sel
+      - 6   # NDRPDR sel
+      - 7   # NDRPDR sel
+      - 8   # NDRPDR sel
+      - 9   # NDRPDR sel
+      - 10  # NDRPDR sel
   compare:
     title: "3-Node Skx"
     data:
-      # TODO: Add data
       csit-dpdk-perf-verify-1810-3n-skx:
       - 3   # NDRPDR sel
       - 4   # NDRPDR sel
@@ -2300,6 +2355,8 @@
       - 9   # NDRPDR sel
       - 10  # NDRPDR sel
       - 11  # NDRPDR sel
+      - 12  # NDRPDR sel
+      - 13  # NDRPDR sel
   data: "dpdk-compare-testbeds"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'NIC_Intel-X710' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -17020,7 +17077,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17054,7 +17111,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17085,7 +17142,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17115,7 +17172,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17147,7 +17204,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17176,7 +17233,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17206,7 +17263,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17235,7 +17292,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17325,7 +17382,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17359,7 +17416,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17390,7 +17447,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17420,7 +17477,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17452,7 +17509,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17481,7 +17538,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17511,7 +17568,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17540,7 +17597,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17687,7 +17744,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17721,7 +17778,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17752,7 +17809,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17782,7 +17839,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17814,7 +17871,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17843,7 +17900,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17873,7 +17930,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -17902,7 +17959,7 @@
            not 'IPSEC' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -18109,7 +18166,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -18137,7 +18194,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -18227,7 +18284,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -18255,7 +18312,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -18345,7 +18402,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -18373,7 +18430,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -18758,7 +18815,7 @@
            'LISP' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort:
@@ -18783,7 +18840,7 @@
            'LISP' and
            not 'VHOST'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   - "tags"
   sort: