Report: Add ucode comparisom

Change-Id: If918806ba5bcda4d87aa2ea2ab4d2c676dc9670a
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/docs/report/vpp_performance_tests/comparisons/bad_good.rst b/docs/report/vpp_performance_tests/comparisons/bad_good.rst
new file mode 100644 (file)
index 0000000..5660fc6
--- /dev/null
+++ b/docs/report/vpp_performance_tests/comparisons/bad_good.rst
@@ -0,0 +1,97 @@
+
+.. _vpp_bad_good:
+
+The Bad and the Good
+--------------------
+
+.. note::
+
+    Test results have been generated by
+
+    - `FD.io test executor vpp performance job 2n-skx`_,
+    - `FD.io test executor vpp performance job 3n-skx`_,
+    - `FD.io test executor vpp performance job 2n-clx`_,
+
+    with RF result files csit-vpp-perf-|srelease|-\*.zip
+    `archived here <../../_static/archive/>`_.
+
+2n-skx
+~~~~~~
+
+NDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+  - `HTML 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-ndr.html>`_
+  - `HTML 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-ndr.html>`_
+  - `ASCII 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-ndr.txt>`_
+  - `ASCII 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-ndr.txt>`_
+  - `CSV 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-ndr.csv>`_
+  - `CSV 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-ndr.csv>`_
+
+PDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+  - `HTML 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-pdr.html>`_
+  - `HTML 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-pdr.html>`_
+  - `ASCII 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-pdr.txt>`_
+  - `ASCII 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-pdr.txt>`_
+  - `CSV 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-pdr.csv>`_
+  - `CSV 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-pdr.csv>`_
+
+3n-skx
+~~~~~~
+
+NDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+  - `HTML 2t1c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-ndr.html>`_
+  - `HTML 4t2c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-ndr.html>`_
+  - `ASCII 2t1c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-ndr.txt>`_
+  - `ASCII 4t2c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-ndr.txt>`_
+  - `CSV 2t1c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-ndr.csv>`_
+  - `CSV 4t2c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-ndr.csv>`_
+
+PDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+  - `HTML 2t1c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-pdr.html>`_
+  - `HTML 4t2c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-pdr.html>`_
+  - `ASCII 2t1c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-pdr.txt>`_
+  - `ASCII 4t2c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-pdr.txt>`_
+  - `CSV 2t1c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-pdr.csv>`_
+  - `CSV 4t2c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-pdr.csv>`_
+
+2n-skx
+~~~~~~
+
+NDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+  - `HTML 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-ndr.html>`_
+  - `HTML 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-ndr.html>`_
+  - `ASCII 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-ndr.txt>`_
+  - `ASCII 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-ndr.txt>`_
+  - `CSV 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-ndr.csv>`_
+  - `CSV 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-ndr.csv>`_
+
+PDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+  - `HTML 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-pdr.html>`_
+  - `HTML 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-pdr.html>`_
+  - `ASCII 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-pdr.txt>`_
+  - `ASCII 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-pdr.txt>`_
+  - `CSV 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-pdr.csv>`_
+  - `CSV 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-pdr.csv>`_

diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst
index ed8ce9f..7862a63 100644 (file)
--- a/docs/report/vpp_performance_tests/comparisons/index.rst
+++ b/docs/report/vpp_performance_tests/comparisons/index.rst
@@ -16,3 +16,4 @@ Comparisons
     3n-skx_vs_2n-skx_testbeds
     nic_comparison
     soak_vs_ndr
+    bad_good

diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py
index 51f3097..b6a3939 100644 (file)
--- a/resources/tools/presentation/generator_plots.py
+++ b/resources/tools/presentation/generator_plots.py
@@ -205,7 +205,11 @@ def plot_hdrh_lat_by_percentile(plot, input_data):
             params=[u"latency", u"throughput", u"parent", u"tags", u"type"]
         )[0][0]
     elif plot.get(u"filter", None):
-        data = input_data.filter_data(plot, continue_on_error=True)
+        data = input_data.filter_data(
+            plot,
+            params=[u"latency", u"throughput", u"parent", u"tags", u"type"],
+            continue_on_error=True
+        )
     else:
         job = list(plot[u"data"].keys())[0]
         build = str(plot[u"data"][job][0])

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 84a56cd..309f45c 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -564,6 +564,75 @@
 ###########################################################################
     # Tables (more than one data set)
 
+    # The GOOD and the BAD
+
+    vpp-performance-3n-skx-bad:
+      csit-vpp-perf-verify-2001-3n-skx:
+        - 5   # NDRPDR sel BAD
+        - 6   # NDRPDR sel BAD
+        - 8   # NDRPDR sel BAD
+        - 9   # NDRPDR sel BAD
+    vpp-performance-3n-skx-good:
+      csit-vpp-perf-verify-2001-3n-skx:
+        - 14  # NDRPDR sel
+        - 15  # NDRPDR sel
+    vpp-performance-3n-skx:
+      csit-vpp-perf-verify-2001-3n-skx:
+        - 5   # NDRPDR sel BAD
+        - 6   # NDRPDR sel BAD
+        - 8   # NDRPDR sel BAD
+        - 9   # NDRPDR sel BAD
+        - 14  # NDRPDR sel
+        - 15  # NDRPDR sel
+
+    vpp-performance-2n-skx-bad:
+      csit-vpp-perf-verify-2001-2n-skx:
+        - 6   # NDRPDR sel BAD
+        - 7   # NDRPDR sel BAD
+        - 10  # NDRPDR sel BAD
+        - 11  # NDRPDR sel BAD
+        - 14  # NDRPDR sel BAD
+    vpp-performance-2n-skx-good:
+      csit-vpp-perf-verify-2001-2n-skx:
+        - 21  # NDRPDR sel
+        - 22  # NDRPDR sel
+        - 23  # NDRPDR sel
+        - 26  # NDRPDR sel
+    vpp-performance-2n-skx:
+      csit-vpp-perf-verify-2001-2n-skx:
+        - 6   # NDRPDR sel BAD
+        - 7   # NDRPDR sel BAD
+        - 10  # NDRPDR sel BAD
+        - 11  # NDRPDR sel BAD
+        - 14  # NDRPDR sel BAD
+        - 21  # NDRPDR sel
+        - 22  # NDRPDR sel
+        - 23  # NDRPDR sel
+        - 26  # NDRPDR sel
+
+    vpp-performance-2n-clx-bad:
+      csit-vpp-perf-verify-2001-2n-clx:
+        - 6   # NDRPDR sel BAD
+        - 7   # NDRPDR sel BAD
+        - 8   # NDRPDR sel BAD
+        - 10  # NDRPDR sel BAD
+        - 11  # NDRPDR sel BAD
+    vpp-performance-2n-clx-good:
+      csit-vpp-perf-verify-2001-2n-clx:
+        - 22  # NDRPDR sel
+        - 23  # NDRPDR sel
+        - 24  # NDRPDR sel
+    vpp-performance-2n-clx:
+      csit-vpp-perf-verify-2001-2n-clx:
+        - 6   # NDRPDR sel BAD
+        - 7   # NDRPDR sel BAD
+        - 8   # NDRPDR sel BAD
+        - 10  # NDRPDR sel BAD
+        - 11  # NDRPDR sel BAD
+        - 22  # NDRPDR sel
+        - 23  # NDRPDR sel
+        - 24  # NDRPDR sel
+
     # Performance changes
 
     vpp-performance-changes-3n-hsw-h1:
@@ -1892,6 +1961,10 @@
       - 67  # NDRPDR sel vhost
       - 36  # MRR sel
     csit-vpp-perf-verify-2001-3n-skx:
+      - 5   # NDRPDR sel BAD
+      - 6   # NDRPDR sel BAD
+      - 8   # NDRPDR sel BAD
+      - 9   # NDRPDR sel BAD
       - 14  # NDRPDR sel
       - 15  # NDRPDR sel
 
@@ -1942,6 +2015,11 @@
       - 83  # SOAK
       - 86  # SOAK
     csit-vpp-perf-verify-2001-2n-skx:
+      - 6   # NDRPDR sel BAD
+      - 7   # NDRPDR sel BAD
+      - 10  # NDRPDR sel BAD
+      - 11  # NDRPDR sel BAD
+      - 14  # NDRPDR sel BAD
       - 21  # NDRPDR sel
       - 22  # NDRPDR sel
       - 23  # NDRPDR sel
@@ -1950,6 +2028,11 @@
       - 31  # NFV
 
     csit-vpp-perf-verify-2001-2n-clx:
+      - 6   # NDRPDR sel BAD
+      - 7   # NDRPDR sel BAD
+      - 8   # NDRPDR sel BAD
+      - 10  # NDRPDR sel BAD
+      - 11  # NDRPDR sel BAD
       - 22  # NDRPDR sel
       - 23  # NDRPDR sel
       - 24  # NDRPDR sel
@@ -2474,6 +2557,237 @@
   - "result"
   - "tags"
 
+################################################################################
+# The Bad and the good
+
+- type: "table"
+  title: "VPP Performance Comparison 3n-skx 2t1c pdr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-2t1c-pdr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-3n-skx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-3n-skx-good"
+  data: "vpp-performance-3n-skx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 3n-skx 4t2c pdr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-4t2c-pdr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-3n-skx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-3n-skx-good"
+  data: "vpp-performance-3n-skx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 3n-skx 2t1c ndr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-2t1c-ndr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-3n-skx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-3n-skx-good"
+  data: "vpp-performance-3n-skx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 3n-skx 4t2c ndr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-4t2c-ndr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-3n-skx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-3n-skx-good"
+  data: "vpp-performance-3n-skx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 2n-skx 2t1c pdr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-2t1c-pdr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-2n-skx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-2n-skx-good"
+  data: "vpp-performance-2n-skx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 2n-skx 4t2c pdr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-4t2c-pdr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-2n-skx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-2n-skx-good"
+  data: "vpp-performance-2n-skx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 2n-skx 2t1c ndr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-2t1c-ndr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-2n-skx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-2n-skx-good"
+  data: "vpp-performance-2n-skx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 2n-skx 4t2c ndr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-4t2c-ndr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-2n-skx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-2n-skx-good"
+  data: "vpp-performance-2n-skx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 2n-clx 2t1c pdr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-2t1c-pdr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-2n-clx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-2n-clx-good"
+  data: "vpp-performance-2n-clx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 2n-clx 4t2c pdr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-4t2c-pdr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-2n-clx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-2n-clx-good"
+  data: "vpp-performance-2n-clx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 2n-clx 2t1c ndr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-2t1c-ndr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-2n-clx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-2n-clx-good"
+  data: "vpp-performance-2n-clx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+- type: "table"
+  title: "VPP Performance Comparison 2n-clx 4t2c ndr"
+  algorithm: "table_perf_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-4t2c-ndr"
+  reference:
+    title: "The Bad"
+    data: "vpp-performance-2n-clx-bad"
+  compare:
+    title: "The Good"
+    data: "vpp-performance-2n-clx-good"
+  data: "vpp-performance-2n-clx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
 ################################################################################
 # VPP