CSIT-1208: Add new data to 1807 report

Change-Id: I93c6625d537bfef3e955e171db92e5a639ffc22e
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/docs/report/dpdk_performance_tests/csit_release_notes.rst b/docs/report/dpdk_performance_tests/csit_release_notes.rst
index 28e6a37..d073c49 100644 (file)
--- a/docs/report/dpdk_performance_tests/csit_release_notes.rst
+++ b/docs/report/dpdk_performance_tests/csit_release_notes.rst
@@ -73,6 +73,27 @@ formats:
     files csit-dpdk-perf-|srelease|-\*.zip
     `archived here <../_static/archive/>`_.
 
+MRR Changes
+~~~~~~~~~~~
+
+MRR small packet throughput changes between releases are available in a
+CSV and pretty ASCII formats:
+
+  - `csv format for 1t1c <../_static/dpdk/performance-changes-1t1c-mrr.csv>`_,
+  - `csv format for 2t2c <../_static/dpdk/performance-changes-2t2c-mrr.csv>`_,
+  - `csv format for 4t4c <../_static/dpdk/performance-changes-4t4c-mrr.csv>`_,
+  - `pretty ASCII format for 1t1c <../_static/dpdk/performance-changes-1t1c-mrr.txt>`_,
+  - `pretty ASCII format for 2t2c <../_static/dpdk/performance-changes-2t2c-mrr.txt>`_,
+  - `pretty ASCII format for 4t4c <../_static/dpdk/performance-changes-4t4c-mrr.txt>`_.
+
+.. note::
+
+    Test results have been generated by
+    `FD.io test executor dpdk performance job 3n-hsw`_,
+    with Robot Framework result
+    files csit-vpp-perf-|srelease|-\*.zip
+    `archived here <../_static/archive/>`_.
+
 Comparison Across Testbeds
 --------------------------
 
@@ -115,6 +136,24 @@ formats:
     files csit-dpdk-perf-|srelease|-\*.zip
     `archived here <../_static/archive/>`_.
 
+MRR Changes
+~~~~~~~~~~~
+
+MRR throughput changes between testbeds are available in a
+CSV and pretty ASCII formats:
+
+  - `csv format for mrr <../_static/dpdk/performance-compare-testbeds-3n-hsw-3n-skx-mrr.csv>`_,
+  - `pretty ASCII format for mrr <../_static/dpdk/performance-compare-testbeds-3n-hsw-3n-skx-mrr.txt>`_.
+
+.. note::
+
+    Test results have been generated by
+    `FD.io test executor dpdk performance job 3n-hsw`_ and
+    `FD.io test executor dpdk performance job 3n-skx`_
+    with Robot Framework result
+    files csit-vpp-perf-|srelease|-\*.zip
+    `archived here <../_static/archive/>`_.
+
 Known Issues
 ------------
 

diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst
index a2fb1c7..90058d3 100644 (file)
--- a/docs/report/vpp_performance_tests/csit_release_notes.rst
+++ b/docs/report/vpp_performance_tests/csit_release_notes.rst
@@ -121,9 +121,10 @@ CSV and pretty ASCII formats:
 .. note::
 
     Test results have been generated by
-    `FD.io test executor vpp mrr jobs <https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master/>`_
-    with Robot Framework result files csit-vpp-perf-mrr-daily-master__*__output.xml.gz
-    `archived here <https://docs.fd.io/csit/master/trending/_static/archive/>`_.
+    `FD.io test executor vpp performance job 3n-hsw`_,
+    with Robot Framework result
+    files csit-vpp-perf-|srelease|-\*.zip
+    `archived here <../_static/archive/>`_.
 
 Comparison Across Testbeds
 --------------------------

diff --git a/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst b/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst
index 2417ce3..2042eb6 100644 (file)
--- a/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst
+++ b/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst
@@ -33,3 +33,4 @@ threaded VPP configurations relative to 1-core configurations.
     container_memif
     container_orchestrated
     ipsec
+    vts

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 3295aec..83d5bc5 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -125,10 +125,13 @@
       - 31  # sel
       - 35  # sel
       # TODO: Add Data sources
-      # 1807
+      # csit-vpp-perf-verify-1807-3n-hsw:
+      # -   # NDRPDR sel
+
     vpp-compare-testbeds:
       # TODO: Add Data sources
-      # HSW
+      # csit-vpp-perf-verify-1807-3n-hsw:
+      # -   # NDRPDR sel
       csit-vpp-perf-1801-all:
       - 124  # sel
       - 127  # sel
@@ -143,25 +146,23 @@
       - 167  # sel
       - 172  # sel acl only
       # TODO: Add Data sources
-      # SKX
-      csit-vpp-perf-1804-all:
-      - 21  # sel
-      - 22  # sel
-      - 23  # sel
-      - 24  # sel
-      - 27  # sel
-      - 28  # sel
-      - 29  # sel
-      - 30  # sel
-      - 31  # sel
-      - 35  # sel
+      csit-vpp-perf-verify-1807-3n-skx:
+      - 33  # NDRPDR sel
+      - 35  # NDRPDR sel
+      - 37  # NDRPDR sel
+      - 38  # NDRPDR sel
+      - 39  # NDRPDR sel
+
     vpp-compare-testbeds-mrr:
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 9   # MRR sel
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-skx:
-      - 21  # MRR sel
+      - 28  # MRR sel
+      - 32  # MRR sel
+      - 36  # MRR sel
+      - 42  # MRR sel
 
     vpp-performance-changes-mrr:
       csit-vpp-perf-check-1801:
@@ -189,6 +190,7 @@
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 9   # MRR sel
+
     vpp-perf-results-3n-hsw:
       # TODO: Add Data sources
       # csit-vpp-perf-verify-1807-3n-hsw:
@@ -197,6 +199,7 @@
       - 20  # full
       - 25  # full
       - 49  # full
+
     vpp-perf-results-3n-skx:
       # TODO: Add Data sources
       # csit-vpp-perf-verify-1807-3n-skx:
@@ -205,26 +208,25 @@
       - 20  # full
       - 25  # full
       - 49  # full
+
     vpp-perf-results-2n-skx:
       # TODO: Add Data sources
-      # csit-vpp-perf-verify-1807-2n-skx:
-      csit-vpp-perf-1804-all:
-      - 19  # full
-      - 20  # full
-      - 25  # full
-      - 49  # full
+      csit-vpp-perf-verify-1807-2n-skx:
+      - 32  # NDRPDR full
+
     vpp-mrr-results-3n-hsw:
-      # TODO: Replace with full
       csit-vpp-perf-verify-1807-3n-hsw:
-      - 9   # MRR sel
+      - 8   # MRR full
+
     vpp-mrr-results-3n-skx:
-      # TODO: Replace with full
+      # TODO: Replace with better results
       csit-vpp-perf-verify-1807-3n-skx:
-      - 21  # MRR sel
+      - 31  # MRR full
+
     vpp-mrr-results-2n-skx:
-      # TODO: Replace with full
       csit-vpp-perf-verify-1807-2n-skx:
-      - 22  # MRR sel
+      - 27  # MRR full
+
     plot-throughput-speedup-analysis-3n-hsw:
       # TODO: Add Data sources
       # csit-vpp-perf-verify-1807-3n-hsw:
@@ -243,30 +245,30 @@
       - 30  # sel
       - 31  # sel
       - 35  # sel
+
     plot-throughput-speedup-analysis-3n-skx:
       # TODO: Add Data sources
-      # csit-vpp-perf-verify-1807-3n-skx:
-      csit-vpp-perf-1804-all:
-      - 19  # full
-      - 20  # full
-      - 25  # full
-      - 49  # full
-      - 21  # sel
-      - 22  # sel
-      - 23  # sel
-      - 24  # sel
-      - 27  # sel
-      - 28  # sel
-      - 29  # sel
-      - 30  # sel
-      - 31  # sel
-      - 35  # sel
+      csit-vpp-perf-verify-1807-3n-skx:
+      - 33  # NDRPDR sel
+      - 35  # NDRPDR sel
+      - 37  # NDRPDR sel
+      - 38  # NDRPDR sel
+      - 39  # NDRPDR sel
+
     plot-throughput-speedup-analysis-2n-skx:
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-2n-skx:
-      - 18  # NDRPDR sel
-      - 19  # NDRPDR sel
-      - 20  # NDRPDR sel
+      - 26  # NDRPDR sel
+      - 28  # NDRPDR sel
+      - 29  # NDRPDR sel
+      - 30  # NDRPDR sel
+      - 34  # NDRPDR sel
+      - 35  # NDRPDR sel
+      - 36  # NDRPDR sel
+      - 39  # NDRPDR sel
+      - 40  # NDRPDR sel
+      - 42  # NDRPDR sel
+
     plot-vpp-throughput-latency-3n-hsw:
       # TODO: Add Data sources
       # csit-vpp-perf-verify-1807-3n-hsw:
@@ -285,29 +287,29 @@
       - 30  # sel
       - 31  # sel
       - 35  # sel
+
     plot-vpp-throughput-latency-3n-skx:
       # TODO: Add Data sources
-      # csit-vpp-perf-verify-1807-3n-skx:
-      csit-vpp-perf-1804-all:
-      - 19  # full
-      - 20  # full
-      - 25  # full
-      - 49  # full
-      - 21  # sel
-      - 22  # sel
-      - 23  # sel
-      - 24  # sel
-      - 27  # sel
-      - 28  # sel
-      - 29  # sel
-      - 30  # sel
-      - 31  # sel
-      - 35  # sel
+      csit-vpp-perf-verify-1807-3n-skx:
+      - 33  # NDRPDR sel
+      - 35  # NDRPDR sel
+      - 37  # NDRPDR sel
+      - 38  # NDRPDR sel
+      - 39  # NDRPDR sel
+
     plot-vpp-throughput-latency-2n-skx:
       csit-vpp-perf-verify-1807-2n-skx:
-      - 18  # NDRPDR sel
-      - 19  # NDRPDR sel
-      - 20  # NDRPDR sel
+      - 26  # NDRPDR sel
+      - 28  # NDRPDR sel
+      - 29  # NDRPDR sel
+      - 30  # NDRPDR sel
+      - 34  # NDRPDR sel
+      - 35  # NDRPDR sel
+      - 36  # NDRPDR sel
+      - 39  # NDRPDR sel
+      - 40  # NDRPDR sel
+      - 42  # NDRPDR sel
+
     plot-vpp-http-server-performance:
       # TODO: Add Data sources
       csit-vpp-perf-1804-all:
@@ -327,6 +329,7 @@
       csit-vpp-functional-1807-ubuntu1604-virl:
       # TODO: Add Data sources
       - "lastSuccessfulBuild"
+
     vpp-func-results-centos:
       csit-vpp-functional-1807-centos7-virl:
       # TODO: Add Data sources
@@ -334,13 +337,15 @@
 
     # VPP Ligato
     ligato-perf-results-3n-hsw:
-      # TODO: Replace with full
+      # TODO: Replace with better results
       csit-ligato-perf-1807-all:
-      - 4   # sel
+      - 3   # full ?
+
     plot-ligato-throughput-speedup-analysis-3n-hsw:
       # TODO: Add Data sources
       csit-ligato-perf-1807-all:
       - 4   # sel
+
     plot-ligato-throughput-latency-3n-hsw:
       # TODO: Add Data sources
       csit-ligato-perf-1807-all:
@@ -370,6 +375,7 @@
       - 21
       - 22
       - 23
+
     dpdk-compare-testbeds:
       csit-dpdk-perf-verify-1807-3n-skx:
       - 1
@@ -393,15 +399,19 @@
       - 21
       - 22
       - 23
+
     dpdk-perf-results-3n-hsw:
       csit-dpdk-perf-verify-1807-3n-hsw:
       - 12
+
     dpdk-perf-results-3n-skx:
       csit-dpdk-perf-verify-1807-3n-skx:
       - 12
+
     dpdk-perf-results-2n-skx:
       csit-dpdk-perf-verify-1807-2n-skx:
       - 10
+
     plot-dpdk-throughput-latency-3n-hsw:
       csit-dpdk-perf-verify-1807-3n-hsw:
       - 11
@@ -414,6 +424,7 @@
       - 21
       - 22
       - 23
+
     plot-dpdk-throughput-latency-3n-skx:
       csit-dpdk-perf-verify-1807-3n-skx:
       - 1
@@ -426,6 +437,7 @@
       - 10
       - 11
       - 12
+
     plot-dpdk-throughput-latency-2n-skx:
       csit-dpdk-perf-verify-1807-2n-skx:
       - 1
@@ -712,17 +724,40 @@
     - 48  # wrk
     # TODO: Add Data sources
     csit-vpp-perf-verify-1807-3n-hsw:
+    - 8   # MRR full
     - 9   # MRR sel
     # TODO: Add Data sources
     csit-vpp-perf-verify-1807-3n-skx:
-    - 21  # MRR sel
+    - 31  # MRR full
+    - 28  # MRR sel
+    - 32  # MRR sel
+    - 36  # MRR sel
+    - 42  # MRR sel
+    - 33  # NDRPDR sel
+    - 35  # NDRPDR sel
+    - 37  # NDRPDR sel
+    - 38  # NDRPDR sel
+    - 39  # NDRPDR sel
     # TODO: Add Data sources
     csit-vpp-perf-verify-1807-2n-skx:
-    - 17  # MRR sel
-    - 18  # NDRPDR sel
-    - 19  # NDRPDR sel
-    - 20  # NDRPDR sel
-    - 22  # MRR sel
+    - 27  # MRR full
+    - 31  # MRR sel
+    - 33  # MRR sel
+    - 37  # MRR sel
+    - 38  # MRR sel
+    - 41  # MRR sel
+    - 43  # MRR sel
+    - 32  # NDRPDR full
+    - 26  # NDRPDR sel
+    - 28  # NDRPDR sel
+    - 29  # NDRPDR sel
+    - 30  # NDRPDR sel
+    - 34  # NDRPDR sel
+    - 35  # NDRPDR sel
+    - 36  # NDRPDR sel
+    - 39  # NDRPDR sel
+    - 40  # NDRPDR sel
+    - 42  # NDRPDR sel
     csit-vpp-perf-check-1801:
     - 3   # mrr
     - 4   # mrr
@@ -760,6 +795,7 @@
     - 14  # sel
     # TODO: Add Data sources
     csit-ligato-perf-1807-all:
+    - 3   # full ?
     - 4   # sel
     csit-dpdk-perf-1804-all:
     - 4
@@ -824,8 +860,8 @@
   format:
     html:
     - full
-#    pdf:
-#    - minimal
+    pdf:
+    - minimal
 
 ################################################################################
 ###                               T A B L E S                                ###
@@ -873,6 +909,7 @@
     title: "rls1807"
     data:
       # TODO: Add Data sources
+      # csit-vpp-perf-verify-1807-3n-hsw:
       csit-vpp-perf-1804-all:
       - 21  # sel
       - 22  # sel
@@ -932,6 +969,7 @@
     title: "rls1807"
     data:
       # TODO: Add Data sources
+      # csit-vpp-perf-verify-1807-3n-hsw:
       csit-vpp-perf-1804-all:
       - 21  # sel
       - 22  # sel
@@ -991,6 +1029,7 @@
     title: "rls1807"
     data:
       # TODO: Add Data sources
+      # csit-vpp-perf-verify-1807-3n-hsw:
       csit-vpp-perf-1804-all:
       - 21  # sel
       - 22  # sel
@@ -1050,6 +1089,7 @@
     title: "rls1807"
     data:
       # TODO: Add Data sources
+      # csit-vpp-perf-verify-1807-3n-hsw:
       csit-vpp-perf-1804-all:
       - 21  # sel
       - 22  # sel
@@ -1078,6 +1118,7 @@
     title: "3-Node Hsw"
     data:
       # TODO: Add Data sources
+      # csit-vpp-perf-verify-1807-3n-hsw:
       csit-vpp-perf-1801-all:
       - 124  # sel
       - 127  # sel
@@ -1095,17 +1136,12 @@
     title: "3-Node Skx"
     data:
       # TODO: Add Data sources
-      csit-vpp-perf-1804-all:
-      - 21  # sel
-      - 22  # sel
-      - 23  # sel
-      - 24  # sel
-      - 27  # sel
-      - 28  # sel
-      - 29  # sel
-      - 30  # sel
-      - 31  # sel
-      - 35  # sel
+      csit-vpp-perf-verify-1807-3n-skx:
+      - 33  # NDRPDR sel
+      - 35  # NDRPDR sel
+      - 37  # NDRPDR sel
+      - 38  # NDRPDR sel
+      - 39  # NDRPDR sel
   data: "vpp-compare-testbeds"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'NIC_Intel-X710' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -1123,6 +1159,7 @@
     title: "3-Node Hsw"
     data:
       # TODO: Add Data sources
+      # csit-vpp-perf-verify-1807-3n-hsw:
       csit-vpp-perf-1801-all:
       - 124  # sel
       - 127  # sel
@@ -1140,17 +1177,12 @@
     title: "3-Node Skx"
     data:
       # TODO: Add Data sources
-      csit-vpp-perf-1804-all:
-      - 21  # sel
-      - 22  # sel
-      - 23  # sel
-      - 24  # sel
-      - 27  # sel
-      - 28  # sel
-      - 29  # sel
-      - 30  # sel
-      - 31  # sel
-      - 35  # sel
+      csit-vpp-perf-verify-1807-3n-skx:
+      - 33  # NDRPDR sel
+      - 35  # NDRPDR sel
+      - 37  # NDRPDR sel
+      - 38  # NDRPDR sel
+      - 39  # NDRPDR sel
   data: "vpp-compare-testbeds"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'NIC_Intel-X710' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -1319,14 +1351,17 @@
     data:
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-skx:
-      - 21  # MRR sel
+      - 28  # MRR sel
+      - 32  # MRR sel
+      - 36  # MRR sel
+      - 42  # MRR sel
   data: "vpp-compare-testbeds-mrr"
   include-tests: "MRR" # "PDR" | "NDR" | "MRR"
   filter: "'NIC_Intel-X710' and MRR" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
   parameters:
   - "name"
   - "parent"
-  - "throughput"
+  - "result"
 
 -
   type: "table"
@@ -2033,6 +2068,47 @@
   - "parent"
   - "throughput"
 
+-
+  type: "table"
+  title: "DPDK Comparison Across Testbeds 3n-hsw to 3n-skx mrr"
+  algorithm: "table_performance_comparison"
+  output-file: "{DIR[STATIC,DPDK]}/performance-compare-testbeds-3n-hsw-3n-skx-mrr"
+  reference:
+    title: "3-Node Hsw"
+    data:
+      csit-dpdk-perf-verify-1807-3n-hsw:
+      - 11
+      - 12  # full
+      - 14
+      - 15
+      - 16
+      - 17
+      - 20
+      - 21
+      - 22
+      - 23
+  compare:
+    title: "3-Node Skx"
+    data:
+      csit-dpdk-perf-verify-1807-3n-skx:
+      - 1
+      - 2
+      - 3
+      - 6
+      - 7
+      - 8
+      - 9
+      - 10
+      - 11
+      - 12
+  data: "dpdk-compare-testbeds"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'NIC_Intel-X710' and MRR" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+
 -
   type: "table"
   title: "Detailed Test Results - DPDK Performance Results 3n-hsw"