Code Review / csit.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | review | tree
raw | inline | side by side
Report 2001: Add data
[csit.git] / resources / tools / presentation / specification.yaml
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index da97b74..05a4bb3 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -276,6 +276,8 @@
         - 72  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 76  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 77  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 78  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 80  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
 
     plot-vpp-hdrh-lat-3n-skx:
       csit-vpp-perf-verify-2001-3n-skx:
@@ -593,6 +595,8 @@
         - 72  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 76  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 77  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 78  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 80  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
 
     vpp-nic-comparison-2n-skx:
       csit-vpp-perf-verify-2001-2n-skx:
@@ -659,9 +663,44 @@
         - 85  # NDRPDR full
 
     vpp-perf-results-3n-skx:
-      # TODO: Replace
+      # TODO: Add data
       csit-vpp-perf-verify-2001-3n-skx:
-        - 69  # NDRPDR sel - new2001.0 - vpp build: 19.08.1-release, new BIOS and ucode, irqbalance set
+        - 82  # NDRPDR full - crypto-1
+        - 85  # NDRPDR full - l2-2
+        - 87  # NDRPDR full - ip6-1
+        - 88  # NDRPDR full - ip6-2
+        - 90  # NDRPDR full - memif
+        - 95  # NDRPDR full - l2-1
+        - 96  # NDRPDR full - ip4-2
+        - 97  # NDRPDR full - vhost-1
+        - 102 # NDRPDR full - crypto-2
+        - 104 # NDRPDR full - ip4tunnels-1
+        - 105 # NDRPDR full - ip4tunnels-2
+        - 106 # NDRPDR full - ip4tunnels-3
+        - 107 # NDRPDR full - ip6tunnels-1
+        - 108 # NDRPDR full - ip4-1
+        - 109 # NDRPDR full - crypto-3
+        - 110 # NDRPDR full - crypto-4
+        - 111 # NDRPDR full - ip4-2
+        - 112 # NDRPDR full - ip4-4
+        - 113 # NDRPDR full - ip4-5
+        - 114 # NDRPDR full - ip4-6
+        - 115 # NDRPDR full - ip4-7
+        - 116 # NDRPDR full - ip4-8
+        - 117 # NDRPDR full - ip4-9
+        - 118 # NDRPDR full - srv6-1
+        - 119 # NDRPDR full - srv6-2
+        - 120 # NDRPDR full - l2-3
+        - 122 # NDRPDR full - l2-5
+        - 123 # NDRPDR full - l2-4
+        - 124 # NDRPDR full - l2-7
+        - 125 # NDRPDR full - vhost-2
+        - 126 # NDRPDR full - vhost-3
+        - 128 # NDRPDR full - l2-8
+        - 129 # NDRPDR full - l2-9
+        - 130 # NDRPDR full - l2-10
+        - 131 # NDRPDR full - l2-11
+        - 132 # NDRPDR full - vhost-4
 
     vpp-perf-results-2n-skx:
       csit-vpp-perf-verify-2001-2n-skx:
@@ -904,6 +943,23 @@
         - 72  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 76  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 77  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 78  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 80  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+
+    vpp-perf-changes-2n-clx-new1908_1:
+      csit-vpp-perf-verify-1908_1-2n-clx:
+        - 5  # NDRPDR sel
+        - 6  # NDRPDR sel
+        - 7  # NDRPDR sel
+        - 8  # NDRPDR sel
+        - 9  # NDRPDR sel
+    vpp-perf-changes-2n-clx-new1908_2:
+      csit-vpp-perf-verify-1908_2-2n-clx:
+        - 4  # NDRPDR sel
+        - 5  # NDRPDR sel
+        - 6  # NDRPDR sel
+        - 7  # NDRPDR sel
+        - 8  # NDRPDR sel
 
     vpp-performance-changes-3n-hsw-h1:
       csit-vpp-perf-verify-1904-3n-hsw:
@@ -974,6 +1030,8 @@
         - 72  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 76  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 77  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 78  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 80  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
 
     vpp-performance-changes-2n-skx-h1:
       csit-vpp-perf-verify-1904-2n-skx:
@@ -1385,6 +1443,8 @@
         - 72  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 76  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 77  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 78  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 80  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
 
     vpp-compare-testbeds-skx-clx-ref:
       csit-vpp-perf-verify-2001-2n-skx:
@@ -1559,6 +1619,8 @@
         - 72  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 76  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
         - 77  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 78  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+        - 80  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
     vpp-compare-topologies-cmp:
       csit-vpp-perf-verify-2001-2n-skx:
         - 143  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
@@ -2435,6 +2497,44 @@
       - 72  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
       - 76  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
       - 77  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+      - 78  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+      - 80  # NDRPDR sel - new2001.0 - vpp build: 20.01.0-release, new BIOS and ucode, irqbalance set
+      - 82  # NDRPDR full - crypto-1
+      - 85  # NDRPDR full - l2-2
+      - 87  # NDRPDR full - ip6-1
+      - 88  # NDRPDR full - ip6-2
+      - 90  # NDRPDR full - memif
+      - 95  # NDRPDR full - l2-1
+      - 96  # NDRPDR full - ip4-2
+      - 97  # NDRPDR full - vhost-1
+      - 102 # NDRPDR full - crypto-2
+      - 104 # NDRPDR full - ip4tunnels-1
+      - 105 # NDRPDR full - ip4tunnels-2
+      - 106 # NDRPDR full - ip4tunnels-3
+      - 107 # NDRPDR full - ip6tunnels-1
+      - 108 # NDRPDR full - ip4-1
+      - 109 # NDRPDR full - crypto-3
+      - 110 # NDRPDR full - crypto-4
+      - 111 # NDRPDR full - ip4-2
+      - 112 # NDRPDR full - ip4-4
+      - 113 # NDRPDR full - ip4-5
+      - 114 # NDRPDR full - ip4-6
+      - 115 # NDRPDR full - ip4-7
+      - 116 # NDRPDR full - ip4-8
+      - 117 # NDRPDR full - ip4-9
+      - 118 # NDRPDR full - srv6-1
+      - 119 # NDRPDR full - srv6-2
+      - 120 # NDRPDR full - l2-3
+      - 122 # NDRPDR full - l2-5
+      - 123 # NDRPDR full - l2-4
+      - 124 # NDRPDR full - l2-7
+      - 125 # NDRPDR full - vhost-2
+      - 126 # NDRPDR full - vhost-3
+      - 128 # NDRPDR full - l2-8
+      - 129 # NDRPDR full - l2-9
+      - 130 # NDRPDR full - l2-10
+      - 131 # NDRPDR full - l2-11
+      - 132 # NDRPDR full - vhost-4
       - 50  # MRR sel
       - 53  # MRR sel addition
 
@@ -2544,6 +2644,18 @@
       - 181  # Soak
       - 182  # Soak
 
+    csit-vpp-perf-verify-1908_1-2n-clx:
+      - 5  # NDRPDR sel
+      - 6  # NDRPDR sel
+      - 7  # NDRPDR sel
+      - 8  # NDRPDR sel
+      - 9  # NDRPDR sel
+    csit-vpp-perf-verify-1908_2-2n-clx:
+      - 4  # NDRPDR sel
+      - 5  # NDRPDR sel
+      - 6  # NDRPDR sel
+      - 7  # NDRPDR sel
+      - 8  # NDRPDR sel
     csit-vpp-perf-verify-2001-2n-clx:
       - 28  # NDRPDR sel
       - 29  # NDRPDR sel
@@ -3342,6 +3454,48 @@
     - "1908.1, 1908.2 - VPP versions tested."
     - "Diff - difference between the previous and the current VPP version."
 
+# VPP Performance changes 2n-clx-xxv710 2t1c ndr
+- type: "table"
+  title: "VPP Performance changes 2n-clx-xxv710 2t1c ndr"
+  algorithm: "table_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-1908_2-2n-clx-2t1c-ndr"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+  columns:
+    - title: "1908.1"
+      data-set: "vpp-perf-changes-2n-clx-new1908_1"
+    - title: "1908.2"
+      data-set: "vpp-perf-changes-2n-clx-new1908_2"
+  comparisons:
+    - title: "Diff"
+      reference: 1
+      compare: 2
+  legend:
+    - "Legend to Table:"
+    - "1908.1, 1908.2 - VPP versions tested."
+    - "Diff - difference between the previous and the current VPP version."
+
+# VPP Performance changes 2n-clx-xxv710 2t1c pdr
+- type: "table"
+  title: "VPP Performance changes 2n-clx-xxv710 2t1c pdr"
+  algorithm: "table_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-1908_2-2n-clx-2t1c-pdr"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+  columns:
+    - title: "1908.1"
+      data-set: "vpp-perf-changes-2n-clx-new1908_1"
+    - title: "1908.2"
+      data-set: "vpp-perf-changes-2n-clx-new1908_2"
+  comparisons:
+    - title: "Diff"
+      reference: 1
+      compare: 2
+  legend:
+    - "Legend to Table:"
+    - "1908.1, 1908.2 - VPP versions tested."
+    - "Diff - difference between the previous and the current VPP version."
+
 # VPP Performance Changes 3n-hsw 1t1c pdr
 - type: "table"
   title: "VPP Performance Changes 3n-hsw 1t1c pdr"
@@ -3447,14 +3601,14 @@
     - title: "RCA2"
       data: "rca/rca-3n-skx-2t1c-pdr-rca2.yaml"
   columns:
-    - title: "1908.0.pe"
+    - title: "1908.0-1"
       data-set: "vpp-perf-changes-3n-skx-old1908_0"
       data-replacement: "replacement-1908-vpp-throughput-lat-tsa-3n-skx-vhost"
-    - title: "1908.1.pe"
+    - title: "1908.1-1"
       data-set: "vpp-perf-changes-3n-skx-old1908_1"
-    - title: "1908.1.ce"
+    - title: "1908.1-2"
       data-set: "vpp-perf-changes-3n-skx-new1908_1"
-    - title: "2001.0.ce"
+    - title: "2001.0-2"
       data-set: "vpp-perf-changes-3n-skx-new2001_0"
   comparisons:
     - title: "Diff1"
@@ -3465,14 +3619,12 @@
       reference: 3
       compare: 4
   legend:
-    - "Legend to Table"
-    - "1908.0, 1908.1, 2001.0 - VPP versions tested."
-    - "pe - previous environment used for testing, per CSIT-1908 release, <pointer to BIOS, ucode, Linux, TRex versions and settings>."
-    - "ce - current environment used for testing, per CSIT-2001 release, <pointer to BIOS, ucode, Linux, TRex versions and settings>."
-    - "Diff1 - difference between the previous and the current environment for the same VPP version, 1908.1.ce vs. 1908.1.pe (if latter results not available, 1908.0.pe is used)."
-    - "Diff2 - difference between the previous and the current VPP version in the current environment, 2001.0.ce vs. 1908.1.ce."
-    - "RCA1 - root cause analysis reference for Diff1."
-    - "RCA2 - root cause analysis reference for Diff2."
+    - 'Legend to Table:'
+    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).'
+    - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).'
+    - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.'
+    - 'RCA1 - root cause analysis reference for Diff1.'
+    - 'RCA2 - root cause analysis reference for Diff2.'
 
 # VPP Root Cause Analysis 3n-skx-xxv710 2t1c ndr
 - type: "table"
@@ -3487,14 +3639,14 @@
     - title: "RCA2"
       data: "rca/rca-3n-skx-2t1c-ndr-rca2.yaml"
   columns:
-    - title: "1908.0.pe"
+    - title: "1908.0-1"
       data-set: "vpp-perf-changes-3n-skx-old1908_0"
       data-replacement: "replacement-1908-vpp-throughput-lat-tsa-3n-skx-vhost"
-    - title: "1908.1.pe"
+    - title: "1908.1-1"
       data-set: "vpp-perf-changes-3n-skx-old1908_1"
-    - title: "1908.1.ce"
+    - title: "1908.1-2"
       data-set: "vpp-perf-changes-3n-skx-new1908_1"
-    - title: "2001.0.ce"
+    - title: "2001.0-2"
       data-set: "vpp-perf-changes-3n-skx-new2001_0"
   comparisons:
     - title: "Diff1"
@@ -3505,14 +3657,12 @@
       reference: 3
       compare: 4
   legend:
-    - "Legend to Table"
-    - "1908.0, 1908.1, 2001.0 - VPP versions tested."
-    - "pe - previous environment used for testing, per CSIT-1908 release, <pointer to BIOS, ucode, Linux, TRex versions and settings>."
-    - "ce - current environment used for testing, per CSIT-2001 release, <pointer to BIOS, ucode, Linux, TRex versions and settings>."
-    - "Diff1 - difference between the previous and the current environment for the same VPP version, 1908.1.ce vs. 1908.1.pe (if latter results not available, 1908.0.pe is used)."
-    - "Diff2 - difference between the previous and the current VPP version in the current environment, 2001.0.ce vs. 1908.1.ce."
-    - "RCA1 - root cause analysis reference for Diff1."
-    - "RCA2 - root cause analysis reference for Diff2."
+    - 'Legend to Table:'
+    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).'
+    - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).'
+    - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.'
+    - 'RCA1 - root cause analysis reference for Diff1.'
+    - 'RCA2 - root cause analysis reference for Diff2.'
 
 # VPP Performance Changes 3n-skx 2t1c pdr
 - type: "table"
@@ -3635,14 +3785,14 @@
     - title: "RCA2"
       data: "rca/rca-2n-skx-2t1c-pdr-rca2.yaml"
   columns:
-    - title: "1908.0.pe"
+    - title: "1908.0-1"
       data-set: "vpp-perf-changes-2n-skx-old1908_0"
       data-replacement: "replacement-1908-vpp-throughput-lat-tsa-2n-skx-vhost"
-    - title: "1908.1.pe"
+    - title: "1908.1-1"
       data-set: "vpp-perf-changes-2n-skx-old1908_1"
-    - title: "1908.1.ce"
+    - title: "1908.1-2"
       data-set: "vpp-perf-changes-2n-skx-new1908_1"
-    - title: "2001.0.ce"
+    - title: "2001.0-2"
       data-set: "vpp-perf-changes-2n-skx-new2001_0"
   comparisons:
     - title: "Diff1"
@@ -3653,14 +3803,12 @@
       reference: 3
       compare: 4
   legend:
-    - "Legend to Table"
-    - "1908.0, 1908.1, 2001.0 - VPP versions tested."
-    - "pe - previous environment used for testing, per CSIT-1908 release, <pointer to BIOS, ucode, Linux, TRex versions and settings>."
-    - "ce - current environment used for testing, per CSIT-2001 release, <pointer to BIOS, ucode, Linux, TRex versions and settings>."
-    - "Diff1 - difference between the previous and the current environment for the same VPP version, 1908.1.ce vs. 1908.1.pe (if latter results not available, 1908.0.pe is used)."
-    - "Diff2 - difference between the previous and the current VPP version in the current environment, 2001.0.ce vs. 1908.1.ce."
-    - "RCA1 - root cause analysis reference for Diff1."
-    - "RCA2 - root cause analysis reference for Diff2."
+    - 'Legend to Table:'
+    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).'
+    - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).'
+    - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.'
+    - 'RCA1 - root cause analysis reference for Diff1.'
+    - 'RCA2 - root cause analysis reference for Diff2.'
 
 # VPP Root Cause Analysis 2n-skx-xxv710 2t1c ndr
 - type: "table"
@@ -3675,14 +3823,14 @@
     - title: "RCA2"
       data: "rca/rca-2n-skx-2t1c-ndr-rca2.yaml"
   columns:
-    - title: "1908.0.pe"
+    - title: "1908.0-1"
       data-set: "vpp-perf-changes-2n-skx-old1908_0"
       data-replacement: "replacement-1908-vpp-throughput-lat-tsa-2n-skx-vhost"
-    - title: "1908.1.pe"
+    - title: "1908.1-1"
       data-set: "vpp-perf-changes-2n-skx-old1908_1"
-    - title: "1908.1.ce"
+    - title: "1908.1-2"
       data-set: "vpp-perf-changes-2n-skx-new1908_1"
-    - title: "2001.0.ce"
+    - title: "2001.0-2"
       data-set: "vpp-perf-changes-2n-skx-new2001_0"
   comparisons:
     - title: "Diff1"
@@ -3693,14 +3841,12 @@
       reference: 3
       compare: 4
   legend:
-    - "Legend to Table"
-    - "1908.0, 1908.1, 2001.0 - VPP versions tested."
-    - "pe - previous environment used for testing, per CSIT-1908 release, <pointer to BIOS, ucode, Linux, TRex versions and settings>."
-    - "ce - current environment used for testing, per CSIT-2001 release, <pointer to BIOS, ucode, Linux, TRex versions and settings>."
-    - "Diff1 - difference between the previous and the current environment for the same VPP version, 1908.1.ce vs. 1908.1.pe (if latter results not available, 1908.0.pe is used)."
-    - "Diff2 - difference between the previous and the current VPP version in the current environment, 2001.0.ce vs. 1908.1.ce."
-    - "RCA1 - root cause analysis reference for Diff1."
-    - "RCA2 - root cause analysis reference for Diff2."
+    - 'Legend to Table:'
+    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).'
+    - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).'
+    - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.'
+    - 'RCA1 - root cause analysis reference for Diff1.'
+    - 'RCA2 - root cause analysis reference for Diff2.'
 
 # VPP Performance Changes 2n-skx 2t1c pdr
 - type: "table"
@@ -10542,7 +10688,7 @@
            'NF_DENSITY' and
            'VHOST' and
            'VXLAN' and
-           'CHAIN' a
+           'CHAIN' and
            'IMIX' and
            '2T1C' and
            'NDRPDR'"
Integration tests