X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fspecification.yaml;h=2103534754c0626ef5cc0661ae5034b1abafe39e;hb=refs%2Fchanges%2F93%2F27793%2F1;hp=045b4d85777e3944d3710b22a151d4b95c862123;hpb=3ac9ae837b482f0220fea4f44a4eb2e490e8fa9e;p=csit.git diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 045b4d8577..2103534754 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -152,16 +152,26 @@ plot-vpp-throughput-lat-tsa-3n-hsw: csit-vpp-perf-report-iterative-2005-3n-hsw: - - 3 # NDRPDR sel - - 4 # NDRPDR sel - - 5 # NDRPDR sel - - 10 # NDRPDR sel - - 11 # NDRPDR sel - - 12 # NDRPDR sel - - 15 # NDRPDR sel - - 16 # NDRPDR sel - - 18 # NDRPDR sel - - 19 # NDRPDR sel +# - 3 # NDRPDR sel +# - 4 # NDRPDR sel +# - 5 # NDRPDR sel +# - 10 # NDRPDR sel +# - 11 # NDRPDR sel +# - 12 # NDRPDR sel +# - 15 # NDRPDR sel +# - 16 # NDRPDR sel +# - 18 # NDRPDR sel +# - 19 # NDRPDR sel + - 20 # NDRPDR sel env 4 + - 21 # NDRPDR sel env 4 + - 22 # NDRPDR sel env 4 + - 23 # NDRPDR sel env 4 + - 24 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 26 # NDRPDR sel env 4 + - 27 # NDRPDR sel env 4 + - 28 # NDRPDR sel env 4 + - 29 # NDRPDR sel env 4 plot-vpp-hdrh-lat-3n-hsw: csit-vpp-perf-report-iterative-2005-3n-hsw: @@ -218,16 +228,21 @@ plot-vpp-throughput-lat-tsa-3n-skx: csit-vpp-perf-report-iterative-2005-3n-skx: - - 4 # NDRPDR sel - - 5 # NDRPDR sel ??? all x710 failed - - 33 # NDRPDR sel - - 28 # NDRPDR sel - - 34 # NDRPDR sel - - 36 # NDRPDR sel - - 37 # NDRPDR sel - - 38 # NDRPDR sel - - 39 # NDRPDR sel - - 41 # NDRPDR sel +# - 4 # NDRPDR sel +# - 5 # NDRPDR sel ??? all x710 failed +# - 33 # NDRPDR sel +# - 28 # NDRPDR sel +# - 34 # NDRPDR sel +# - 36 # NDRPDR sel +# - 37 # NDRPDR sel +# - 38 # NDRPDR sel +# - 39 # NDRPDR sel +# - 41 # NDRPDR sel + - 42 # NDRPDR sel env 4 + - 43 # NDRPDR sel env 4 + - 44 # NDRPDR sel env 4 + - 45 # NDRPDR sel env 4 + - 47 # NDRPDR sel env 4 plot-vpp-hdrh-lat-3n-skx: csit-vpp-perf-report-iterative-2005-3n-skx: @@ -267,16 +282,23 @@ plot-vpp-throughput-lat-tsa-2n-skx: csit-vpp-perf-report-iterative-2005-2n-skx: - - 43 # NDRPDR sel - - 38 # NDRPDR sel - - 7 # NDRPDR sel - - 65 # NDRPDR sel - - 47 # NDRPDR sel - - 55 # NDRPDR sel - - 67 # NDRPDR sel - - 75 # NDRPDR sel - - 76 # NDRPDR sel - - 73 # NDRPDR sel +# - 43 # NDRPDR sel +# - 38 # NDRPDR sel +# - 7 # NDRPDR sel +# - 65 # NDRPDR sel +# - 47 # NDRPDR sel +# - 55 # NDRPDR sel +# - 67 # NDRPDR sel +# - 75 # NDRPDR sel +# - 76 # NDRPDR sel +# - 73 # NDRPDR sel + - 88 # NDRPDR sel env 4 + - 89 # NDRPDR sel env 4 + - 90 # NDRPDR sel env 4 + - 91 # NDRPDR sel env 4 + - 92 # NDRPDR sel env 4 + - 93 # NDRPDR sel env 4 + - 94 # NDRPDR sel env 4 plot-vpp-hdrh-lat-2n-skx: csit-vpp-perf-report-iterative-2005-2n-skx: @@ -293,6 +315,7 @@ - 43 # NDRPDR sel - 38 # NDRPDR sel - 7 # NDRPDR sel + - 78 # nf-density-sel - 83 # nf-density-sel plot-vpp-nfv-2n-skx-mrr: @@ -333,16 +356,24 @@ plot-vpp-throughput-lat-tsa-2n-clx: csit-vpp-perf-report-iterative-2005-2n-clx: - - 7 # NDRPDR sel - - 38 # NDRPDR sel - - 9 # NDRPDR sel - - 32 # NDRPDR sel ??? - - 33 # NDRPDR sel ??? - - 39 # NDRPDR sel - - 40 # NDRPDR sel - - 43 # NDRPDR sel - - 50 # NDRPDR sel - - 51 # NDRPDR sel +# - 7 # NDRPDR sel +# - 38 # NDRPDR sel +# - 9 # NDRPDR sel +# - 32 # NDRPDR sel ??? +# - 33 # NDRPDR sel ??? +# - 39 # NDRPDR sel +# - 40 # NDRPDR sel +# - 43 # NDRPDR sel +# - 50 # NDRPDR sel +# - 51 # NDRPDR sel + - 62 # NDRPDR sel env 4 + - 63 # NDRPDR sel env 4 + - 64 # NDRPDR sel env 4 + - 65 # NDRPDR sel env 4 + - 66 # NDRPDR sel env 4 + - 68 # NDRPDR sel env 4 + - 69 # NDRPDR sel env 4 + - 70 # NDRPDR sel env 4 plot-vpp-hdrh-lat-2n-clx: csit-vpp-perf-report-iterative-2005-2n-clx: @@ -410,11 +441,14 @@ plot-vpp-throughput-lat-tsa-3n-tsh: # TODO: Add data: csit-vpp-perf-report-iterative-2005-3n-tsh: - - 9 # NDRPDR sel - - 11 # NDRPDR sel - - 12 # NDRPDR sel - - 20 # NDRPDR sel - - 14 # NDRPDR sel +# - 9 # NDRPDR sel +# - 11 # NDRPDR sel +# - 12 # NDRPDR sel +# - 20 # NDRPDR sel +# - 14 # NDRPDR sel +# - 21 # NDRPDR sel + - 23 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 plot-vpp-hdrh-lat-3n-tsh: csit-vpp-perf-report-iterative-2005-3n-tsh: @@ -580,6 +614,7 @@ - 20 # perf-full-3n-skx-ip4_IX - 24 # perf-full-3n-skx-ip6_I - 25 # perf-full-3n-skx-ip6_II + - 21 # perf-full-3n-skx-ip4tunnels-I - 22 # perf-full-3n-skx-ip4tunnels-II - 23 # perf-full-3n-skx-ip4tunnels-III - 26 # perf-full-3n-skx-ip6tunnels-I @@ -588,9 +623,14 @@ - 30 # perf-full-3n-skx-l2_IV - 32 # perf-full-3n-skx-l2_VI - 33 # perf-full-3n-skx-l2_VII + - 34 # perf-full-3n-skx-l2_VIII - 35 # perf-full-3n-skx-l2_IX - 36 # perf-full-3n-skx-l2_X - 37 # perf-full-3n-skx-l2_XI + - 38 # perf-full-3n-skx-memif + - 39 # perf-full-3n-skx-srv6-I + - 40 # perf-full-3n-skx-srv6-II + - 43 # perf-full-3n-skx-vhost-III vpp-perf-results-2n-skx: # TODO: Add data: @@ -757,9 +797,25 @@ - 19 # NDRPDR sel vpp-performance-changes-3n-hsw-cmp-2: csit-vpp-perf-report-iterative-2005-3n-hsw: - - 20 # NDRPDR sel env 2 - - 21 # NDRPDR sel env 2 - - 22 # NDRPDR sel env 2 + - 20 # NDRPDR sel env 4 + - 21 # NDRPDR sel env 4 + - 22 # NDRPDR sel env 4 + - 23 # NDRPDR sel env 4 + - 24 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 26 # NDRPDR sel env 4 + - 27 # NDRPDR sel env 4 + - 28 # NDRPDR sel env 4 + - 29 # NDRPDR sel env 4 + vpp-performance-changes-3n-hsw-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-hsw: + - 31 # NDRPDR sel 2001.0-4 + - 32 # NDRPDR sel 2001.0-4 + - 33 # NDRPDR sel 2001.0-4 + - 34 # NDRPDR sel 2001.0-4 + - 35 # NDRPDR sel 2001.0-4 + - 36 # NDRPDR sel 2001.0-4 + - 38 # NDRPDR sel 2001.0-4 vpp-performance-changes-3n-skx-h1: csit-vpp-perf-verify-1908_1-3n-skx: @@ -796,17 +852,17 @@ - 41 # NDRPDR sel vpp-performance-changes-3n-skx-cmp-2: csit-vpp-perf-report-iterative-2005-3n-skx: - # TODO: Replace - - 4 # NDRPDR sel - - 5 # NDRPDR sel ??? all x710 failed - - 33 # NDRPDR sel - - 28 # NDRPDR sel - - 34 # NDRPDR sel - - 36 # NDRPDR sel - - 37 # NDRPDR sel - - 38 # NDRPDR sel - - 39 # NDRPDR sel - - 41 # NDRPDR sel + - 42 # NDRPDR sel env 4 + - 43 # NDRPDR sel env 4 + - 44 # NDRPDR sel env 4 + - 45 # NDRPDR sel env 4 + - 47 # NDRPDR sel env 4 + vpp-performance-changes-3n-skx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-skx: + - 50 # NDRPDR sel 2001.0-4 + - 51 # NDRPDR sel 2001.0-4 + - 52 # NDRPDR sel 2001.0-4 + - 53 # NDRPDR sel 2001.0-4 vpp-performance-changes-2n-skx-h1: csit-vpp-perf-verify-1908_1-2n-skx: @@ -846,17 +902,21 @@ - 73 # NDRPDR sel vpp-performance-changes-2n-skx-cmp-2: csit-vpp-perf-report-iterative-2005-2n-skx: - # TODO: Replace - - 43 # NDRPDR sel - - 38 # NDRPDR sel - - 7 # NDRPDR sel - - 65 # NDRPDR sel - - 47 # NDRPDR sel - - 55 # NDRPDR sel - - 67 # NDRPDR sel - - 75 # NDRPDR sel - - 76 # NDRPDR sel - - 73 # NDRPDR sel + - 88 # NDRPDR sel env 4 + - 89 # NDRPDR sel env 4 + - 90 # NDRPDR sel env 4 + - 91 # NDRPDR sel env 4 + - 92 # NDRPDR sel env 4 + - 93 # NDRPDR sel env 4 + - 94 # NDRPDR sel env 4 + vpp-performance-changes-2n-skx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-2n-skx: + - 100 # NDRPDR sel 2001.0-4 + - 101 # NDRPDR sel 2001.0-4 + - 102 # NDRPDR sel 2001.0-4 + - 103 # NDRPDR sel 2001.0-4 + - 104 # NDRPDR sel 2001.0-4 + - 105 # NDRPDR sel 2001.0-4 # vpp-performance-changes-nfv-2n-skx-ref: # csit-vpp-perf-verify-2001-2n-skx: @@ -867,6 +927,7 @@ # - 43 # NDRPDR sel # - 38 # NDRPDR sel # - 7 # NDRPDR sel +# - 78 # nf-density-sel # - 83 # nf-density-sel vpp-performance-changes-2n-clx-ref: @@ -895,8 +956,24 @@ - 51 # NDRPDR sel vpp-performance-changes-2n-clx-cmp-2: csit-vpp-perf-report-iterative-2005-2n-clx: - - 62 # NDRPDR sel env 2 - - 63 # NDRPDR sel env 2 + - 62 # NDRPDR sel env 4 + - 63 # NDRPDR sel env 4 + - 64 # NDRPDR sel env 4 + - 65 # NDRPDR sel env 4 + - 66 # NDRPDR sel env 4 + - 68 # NDRPDR sel env 4 + - 69 # NDRPDR sel env 4 + - 70 # NDRPDR sel env 4 + vpp-performance-changes-2n-clx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-2n-clx: + - 73 # NDRPDR sel 2001.0-4 + - 74 # NDRPDR sel 2001.0-4 + - 75 # NDRPDR sel 2001.0-4 + - 76 # NDRPDR sel 2001.0-4 + - 77 # NDRPDR sel 2001.0-4 + - 78 # NDRPDR sel 2001.0-4 + - 79 # NDRPDR sel 2001.0-4 + - 80 # NDRPDR sel 2001.0-4 vpp-performance-changes-2n-dnv-h1: csit-vpp-perf-verify-1908-2n-dnv: @@ -1004,14 +1081,15 @@ - 12 # NDRPDR sel - 20 # NDRPDR sel - 14 # NDRPDR sel + - 21 # NDRPDR sel vpp-performance-changes-3n-tsh-cmp-2: - # TODO: replace: csit-vpp-perf-report-iterative-2005-3n-tsh: - - 9 # NDRPDR sel - - 11 # NDRPDR sel - - 12 # NDRPDR sel - - 20 # NDRPDR sel - - 14 # NDRPDR sel + - 23 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + vpp-performance-changes-3n-tsh-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-tsh: + - 28 # NDRPDR sel 2001.0-4 + - 29 # NDRPDR sel 2001.0-4 vpp-soak-vs-ndr-2n-skx-ref: csit-vpp-perf-report-iterative-2005-2n-skx: @@ -1084,6 +1162,9 @@ vpp-performance-changes-3n-hsw-mrr-cmp: csit-vpp-perf-report-iterative-2005-3n-hsw: - 13 # MRR + vpp-performance-changes-3n-hsw-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-hsw: + - 30 # MRR env 4 # vpp-performance-changes-3n-skx-mrr-h1: # csit-vpp-perf-verify-1908-3n-skx: @@ -1095,6 +1176,9 @@ vpp-performance-changes-3n-skx-mrr-cmp: csit-vpp-perf-report-iterative-2005-3n-skx: - 6 # MRR sel + vpp-performance-changes-3n-skx-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-skx: + - 49 # MRR sel env 4 # vpp-performance-changes-2n-skx-mrr-h1: # csit-vpp-perf-verify-1908-2n-skx: @@ -1106,6 +1190,9 @@ vpp-performance-changes-2n-skx-mrr-cmp: csit-vpp-perf-report-iterative-2005-2n-skx: - 42 # MRR sel + vpp-performance-changes-2n-skx-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-skx: + - 99 # MRR sel env 4 vpp-performance-changes-2n-clx-mrr-ref: csit-vpp-perf-verify-2001-2n-clx: @@ -1113,6 +1200,9 @@ vpp-performance-changes-2n-clx-mrr-cmp: csit-vpp-perf-report-iterative-2005-2n-clx: - 12 # MRR + vpp-performance-changes-2n-clx-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-clx: + - 72 # MRR # vpp-performance-changes-nfv-2n-skx-mrr-ref: # csit-vpp-perf-verify-1908-2n-skx: @@ -1151,6 +1241,9 @@ vpp-performance-changes-3n-tsh-mrr-cmp: csit-vpp-perf-report-iterative-2005-3n-tsh: - 10 # MRR sel + vpp-performance-changes-3n-tsh-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-tsh: + - 27 # MRR sel env 4 dpdk-performance-changes-3n-hsw-h1: csit-dpdk-perf-verify-1908-3n-hsw: @@ -1689,120 +1782,6 @@ hoverlabel: namelength: -1 -# plot-cps: -# titlefont: -# size: 18 -# xaxis: -# title: "Test Cases [Index]" -# titlefont: -# size: 16 -# autorange: True -# fixedrange: False -# gridcolor: "rgb(230, 230, 230)" -# linecolor: "rgb(220, 220, 220)" -# linewidth: 1 -# showgrid: True -# showline: True -# showticklabels: True -# tickcolor: "rgb(220, 220, 220)" -# tickmode: "linear" -# tickfont: -# size: 16 -# zeroline: False -# yaxis: -# title: "Connections Per Second [cps]" -# titlefont: -# size: 16 -# gridcolor: "rgb(230, 230, 230)" -# hoverformat: ".4s" -# linecolor: "rgb(220, 220, 220)" -# linewidth: 1 -# rangemode: "tozero" -# showgrid: True -# showline: True -# showticklabels: True -# tickcolor: "rgb(220, 220, 220)" -# tickformat: ".3s" -# tickfont: -# size: 16 -# zeroline: False -# boxmode: "group" -# boxgroupgap: 0.5 -# autosize: False -# margin: -# t: 50 -# b: 20 -# l: 80 -# r: 20 -# showlegend: True -# legend: -# orientation: "h" -# font: -# size: 16 -# width: 700 -# height: 800 -# paper_bgcolor: "#fff" -# plot_bgcolor: "#fff" -# hoverlabel: -# namelength: -1 -# -# plot-rps: -# titlefont: -# size: 18 -# xaxis: -# title: "Test Cases [Index]" -# titlefont: -# size: 16 -# autorange: True -# fixedrange: False -# gridcolor: "rgb(230, 230, 230)" -# linecolor: "rgb(220, 220, 220)" -# linewidth: 1 -# showgrid: True -# showline: True -# showticklabels: True -# tickcolor: "rgb(220, 220, 220)" -# tickmode: "linear" -# tickfont: -# size: 16 -# zeroline: False -# yaxis: -# title: "Requests Per Second [rps]" -# titlefont: -# size: 16 -# gridcolor: "rgb(230, 230, 230)" -# hoverformat: ".4s" -# linecolor: "rgb(220, 220, 220)" -# linewidth: 1 -# rangemode: "tozero" -# showgrid: True -# showline: True -# showticklabels: True -# tickcolor: "rgb(230, 230, 230)" -# tickformat: ".3s" -# tickfont: -# size: 16 -# zeroline: False -# boxmode: "group" -# boxgroupgap: 0.5 -# autosize: False -# margin: -# t: 50 -# b: 20 -# l: 80 -# r: 20 -# showlegend: True -# legend: -# orientation: "h" -# font: -# size: 16 -# width: 700 -# height: 800 -# paper_bgcolor: "#fff" -# plot_bgcolor: "#fff" -# hoverlabel: -# namelength: -1 - plot-hoststack: titlefont: size: 16 @@ -2288,10 +2267,25 @@ - 16 # NDRPDR sel - 18 # NDRPDR sel - 19 # NDRPDR sel - - 20 # NDRPDR sel env 2 - - 21 # NDRPDR sel env 2 - - 22 # NDRPDR sel env 2 + - 20 # NDRPDR sel env 4 + - 21 # NDRPDR sel env 4 + - 22 # NDRPDR sel env 4 + - 23 # NDRPDR sel env 4 + - 24 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 26 # NDRPDR sel env 4 + - 27 # NDRPDR sel env 4 + - 28 # NDRPDR sel env 4 + - 29 # NDRPDR sel env 4 - 13 # MRR + - 30 # MRR env 4 + - 31 # NDRPDR sel 2001.0-4 + - 32 # NDRPDR sel 2001.0-4 + - 33 # NDRPDR sel 2001.0-4 + - 34 # NDRPDR sel 2001.0-4 + - 35 # NDRPDR sel 2001.0-4 + - 36 # NDRPDR sel 2001.0-4 + - 38 # NDRPDR sel 2001.0-4 csit-vpp-perf-verify-2005-3n-hsw: - 3 # hoststack - 5 # hoststack @@ -2373,7 +2367,17 @@ - 38 # NDRPDR sel - 39 # NDRPDR sel - 41 # NDRPDR sel + - 42 # NDRPDR sel env 4 + - 43 # NDRPDR sel env 4 + - 44 # NDRPDR sel env 4 + - 45 # NDRPDR sel env 4 + - 47 # NDRPDR sel env 4 + - 50 # NDRPDR sel 2001.0-4 + - 51 # NDRPDR sel 2001.0-4 + - 52 # NDRPDR sel 2001.0-4 + - 53 # NDRPDR sel 2001.0-4 - 6 # MRR sel + - 49 # MRR sel env 4 csit-vpp-perf-report-coverage-2005-3n-skx: - 10 # perf-full-3n-skx-crypto_I - 11 # perf-full-3n-skx-crypto_II @@ -2389,6 +2393,7 @@ - 20 # perf-full-3n-skx-ip4_IX - 24 # perf-full-3n-skx-ip6_I - 25 # perf-full-3n-skx-ip6_II + - 21 # perf-full-3n-skx-ip4tunnels-I - 22 # perf-full-3n-skx-ip4tunnels-II - 23 # perf-full-3n-skx-ip4tunnels-III - 26 # perf-full-3n-skx-ip6tunnels-I @@ -2397,9 +2402,14 @@ - 30 # perf-full-3n-skx-l2_IV - 32 # perf-full-3n-skx-l2_VI - 33 # perf-full-3n-skx-l2_VII + - 34 # perf-full-3n-skx-l2_VIII - 35 # perf-full-3n-skx-l2_IX - 36 # perf-full-3n-skx-l2_X - 37 # perf-full-3n-skx-l2_XI + - 38 # perf-full-3n-skx-memif + - 39 # perf-full-3n-skx-srv6-I + - 40 # perf-full-3n-skx-srv6-II + - 43 # perf-full-3n-skx-vhost-III csit-vpp-perf-verify-1908_1-2n-skx: - 39 # NDRPDR sel - new1908.1 @@ -2436,7 +2446,21 @@ - 75 # NDRPDR sel - 76 # NDRPDR sel - 73 # NDRPDR sel + - 88 # NDRPDR sel env 4 + - 89 # NDRPDR sel env 4 + - 90 # NDRPDR sel env 4 + - 91 # NDRPDR sel env 4 + - 92 # NDRPDR sel env 4 + - 93 # NDRPDR sel env 4 + - 94 # NDRPDR sel env 4 + - 100 # NDRPDR sel 2001.0-4 + - 101 # NDRPDR sel 2001.0-4 + - 102 # NDRPDR sel 2001.0-4 + - 103 # NDRPDR sel 2001.0-4 + - 104 # NDRPDR sel 2001.0-4 + - 105 # NDRPDR sel 2001.0-4 - 42 # MRR sel + - 99 # MRR sel env 4 - 48 # Reconf - 50 # Reconf - 51 # Reconf @@ -2449,6 +2473,7 @@ - 66 # Reconf - 49 # Soak - 54 # Soak + - 78 # nf-density-sel - 83 # nf-density-sel csit-vpp-perf-report-coverage-2005-2n-skx: - 11 # perf-full-2n-skx-ip4_I @@ -2496,9 +2521,24 @@ - 43 # NDRPDR sel - 50 # NDRPDR sel - 51 # NDRPDR sel - - 62 # NDRPDR sel env 2 - - 63 # NDRPDR sel env 2 + - 62 # NDRPDR sel env 4 + - 63 # NDRPDR sel env 4 + - 64 # NDRPDR sel env 4 + - 65 # NDRPDR sel env 4 + - 66 # NDRPDR sel env 4 + - 68 # NDRPDR sel env 4 + - 69 # NDRPDR sel env 4 + - 70 # NDRPDR sel env 4 + - 73 # NDRPDR sel 2001.0-4 + - 74 # NDRPDR sel 2001.0-4 + - 75 # NDRPDR sel 2001.0-4 + - 76 # NDRPDR sel 2001.0-4 + - 77 # NDRPDR sel 2001.0-4 + - 78 # NDRPDR sel 2001.0-4 + - 79 # NDRPDR sel 2001.0-4 + - 80 # NDRPDR sel 2001.0-4 - 12 # MRR + - 72 # MRR env 4 - 42 # Reconf - 45 # Reconf - 46 # Reconf @@ -2564,7 +2604,13 @@ - 12 # NDRPDR sel - 20 # NDRPDR sel - 14 # NDRPDR sel + - 21 # NDRPDR sel + - 23 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 28 # NDRPDR sel 2001.0-4 + - 29 # NDRPDR sel 2001.0-4 - 10 # MRR sel + - 27 # MRR sel env 4 csit-vpp-perf-verify-1908-3n-dnv: - 6 # NDRPDR sel @@ -2956,7 +3002,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -2979,7 +3025,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3002,7 +3048,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3025,7 +3071,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3048,7 +3094,7 @@ # reference: 1 # compare: 2 # legend: -# - "Legend to Table:" +# - "" # - "x710, xxv710 - NICs tested." # - "Diff - difference between x710 and xxv710 NIC." # @@ -3071,7 +3117,7 @@ # reference: 1 # compare: 2 # legend: -# - "Legend to Table:" +# - "" # - "x710, xxv710 - NICs tested." # - "Diff - difference between x710 and xxv710 NIC." # @@ -3094,7 +3140,7 @@ # reference: 1 # compare: 2 # legend: -# - "Legend to Table:" +# - "" # - "x710, xxv710 - NICs tested." # - "Diff - difference between x710 and xxv710 NIC." @@ -3117,7 +3163,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3140,7 +3186,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3163,7 +3209,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3186,7 +3232,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3209,7 +3255,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3232,7 +3278,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3255,7 +3301,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3274,29 +3320,31 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-hsw-cmp" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-hsw 2t2c pdr - type: "table" @@ -3310,29 +3358,31 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-hsw-cmp" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: -# - title: "RCA1" -# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml" # - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml" +# - title: "RCA3" # data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-hsw 1t1c ndr - type: "table" @@ -3346,29 +3396,31 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-hsw-cmp" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: # - title: "RCA1" -# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca1.yaml" -# - title: "RCA2" # data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca2.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca3.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-hsw 2t2c ndr - type: "table" @@ -3382,29 +3434,219 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 1t1c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-hsw 1t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "Diff1" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" reference: 3 - compare: 4 - - title: "Diff2" + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 2t2c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-hsw 2t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 1t1c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-hsw 1t1c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 2t2c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-hsw 2t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca3.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-skx 2t1c pdr - type: "table" @@ -3418,29 +3660,31 @@ data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-skx-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-skx-cmp" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-skx 4t2c pdr - type: "table" @@ -3454,29 +3698,31 @@ data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-skx-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-skx-cmp" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-skx 2t1c ndr - type: "table" @@ -3490,29 +3736,31 @@ data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-skx-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-skx-cmp" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-skx 4t2c ndr - type: "table" @@ -3526,65 +3774,257 @@ data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-skx-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-skx-cmp" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' - -# VPP Performance Changes 2n-skx 2t1c pdr + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c pdr-full - type: "table" - title: "VPP Performance Changes 2n-skx 2t1c pdr" + title: "VPP Performance Changes 3n-skx 2t1c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - title: "1908.1-2" - data-set: "vpp-performance-changes-2n-skx-h1" + data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" - data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005.0-2" - data-set: "vpp-performance-changes-2n-skx-cmp-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-skx-cmp" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" -# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca1.yaml" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c pdr +- type: "table" + title: "VPP Performance Changes 2n-skx 2t1c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-skx-2t1c-pdr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 4t2c pdr - type: "table" @@ -3598,29 +4038,31 @@ data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-skx-cmp" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 2t1c ndr - type: "table" @@ -3634,29 +4076,31 @@ data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-skx-cmp" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 4t2c ndr - type: "table" @@ -3670,29 +4114,219 @@ data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 2t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "Diff1" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" reference: 3 - compare: 4 - - title: "Diff2" + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 4t2c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 4t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c ndr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 2t1c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 4t2c ndr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 4t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca3.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-xxv710 2t1c pdr - type: "table" @@ -3704,29 +4338,31 @@ columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - - title: "Diff1" - reference: 2 - compare: 3 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 1 compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 3 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-xxv710 4t2c pdr - type: "table" @@ -3738,29 +4374,31 @@ columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - - title: "Diff1" - reference: 2 - compare: 3 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 1 compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 3 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-xxv710 2t1c ndr - type: "table" @@ -3772,29 +4410,31 @@ columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - - title: "Diff1" - reference: 2 - compare: 3 - - title: "Diff2" + - title: "2001.0-4 vs 2001.0-2" reference: 1 compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 3 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-xxv710 4t2c ndr - type: "table" @@ -3806,29 +4446,211 @@ columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 1 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - - title: "Diff1" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2001.0-4 vs 2001.0-2" + reference: 1 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" reference: 2 - compare: 3 - - title: "Diff2" + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2001.0-4 vs 2001.0-2" reference: 1 compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2001.0-4 vs 2001.0-2" + reference: 1 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c ndr-full +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2001.0-4 vs 2001.0-2" + reference: 1 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca3.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-x556 2t1c pdr - type: "table" @@ -3840,16 +4662,19 @@ columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005.0-3" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" reference: 1 compare: 2 legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with x556.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-x556 4t2c pdr - type: "table" @@ -3861,16 +4686,19 @@ columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005.0-3" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" reference: 1 compare: 2 legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with x556.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-x556 2t1c ndr - type: "table" @@ -3882,16 +4710,19 @@ columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005.0-3" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" reference: 1 compare: 2 legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with x556.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-x556 4t2c ndr - type: "table" @@ -3903,16 +4734,19 @@ columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005.0-3" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" reference: 1 compare: 2 legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with x556.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-dnv 1t1c pdr - type: "table" @@ -3933,7 +4767,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -3956,7 +4790,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -3979,7 +4813,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4002,7 +4836,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4025,7 +4859,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4048,7 +4882,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4071,7 +4905,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4094,7 +4928,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4110,35 +4944,198 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005.0-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c pdr +- type: "table" + title: "VPP Performance Changes 3n-tsh 2t2c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 1t1c ndr +- type: "table" + title: "VPP Performance Changes 3n-tsh 1t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c ndr +- type: "table" + title: "VPP Performance Changes 3n-tsh 2t2c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 1t1c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-tsh 1t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca3.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' - -# VPP Performance Changes 3n-tsh 2t2c pdr + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c pdr-full - type: "table" - title: "VPP Performance Changes 3n-tsh 2t2c pdr" + title: "VPP Performance Changes 3n-tsh 2t2c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: @@ -4146,35 +5143,46 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005.0-2" - data-set: "vpp-performance-changes-3n-tsh-cmp-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca3.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' - -# VPP Performance Changes 3n-tsh 1t1c ndr + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 1t1c ndr-full - type: "table" - title: "VPP Performance Changes 3n-tsh 1t1c ndr" + title: "VPP Performance Changes 3n-tsh 1t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: @@ -4182,35 +5190,46 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005.0-2" - data-set: "vpp-performance-changes-3n-tsh-cmp-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca3.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' - -# VPP Performance Changes 3n-tsh 2t2c ndr + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c ndr-full - type: "table" - title: "VPP Performance Changes 3n-tsh 2t2c ndr" + title: "VPP Performance Changes 3n-tsh 2t2c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: @@ -4218,29 +5237,40 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005.0-2" - data-set: "vpp-performance-changes-3n-tsh-cmp-2" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff1" - reference: 3 - compare: 4 - - title: "Diff2" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca3.yaml" legend: - - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr - type: "table" @@ -4261,7 +5291,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -4284,7 +5314,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -4307,7 +5337,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -4330,7 +5360,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -4351,7 +5381,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -4372,7 +5402,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -4393,7 +5423,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -4414,7 +5444,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -4437,7 +5467,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -4460,7 +5490,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -4483,7 +5513,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -4506,7 +5536,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -4527,7 +5557,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -4548,7 +5578,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -4569,7 +5599,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -4590,7 +5620,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -4602,20 +5632,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'1T1C' and 'MRR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-hsw 2t2c mrr - type: "table" @@ -4625,20 +5669,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T2C' and 'MRR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-2t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-hsw 4t4c mrr - type: "table" @@ -4648,20 +5706,35 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T4C' and 'MRR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-4t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-4t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + # VPP Performance Changes 3n-skx 2t1c mrr - type: "table" @@ -4673,18 +5746,32 @@ columns: # - title: "1908" # data-set: "vpp-performance-changes-3n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-skx 4t2c mrr - type: "table" @@ -4696,18 +5783,32 @@ columns: # - title: "1908" # data-set: "vpp-performance-changes-3n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-skx 8t4c mrr - type: "table" @@ -4719,18 +5820,32 @@ columns: # - title: "1908" # data-set: "vpp-performance-changes-3n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-8t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-8t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-skx 2t1c mrr - type: "table" @@ -4740,20 +5855,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: -# - title: "1908" -# data-set: "vpp-performance-changes-2n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-skx 4t2c mrr - type: "table" @@ -4763,20 +5890,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: -# - title: "1908" -# data-set: "vpp-performance-changes-2n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-skx 8t4c mrr - type: "table" @@ -4786,20 +5925,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'8T4C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: -# - title: "1908" -# data-set: "vpp-performance-changes-2n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-8t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-8t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-xxv710 2t1c mrr - type: "table" @@ -4809,18 +5960,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-xxv710 4t2c mrr - type: "table" @@ -4830,18 +5995,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-xxv710 8t4c mrr - type: "table" @@ -4851,18 +6030,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'8T4C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-8t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-8t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-clx-x556 2t1c mrr - type: "table" @@ -4881,7 +6074,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4902,7 +6095,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4923,7 +6116,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4946,7 +6139,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4969,7 +6162,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4992,7 +6185,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -5015,7 +6208,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -5038,7 +6231,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -5061,7 +6254,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -5073,20 +6266,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-tsh 2t2c mrr - type: "table" @@ -5096,20 +6303,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-tsh 4t4c mrr - type: "table" @@ -5119,20 +6340,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T4C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-4t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-4t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c mrr - type: "table" @@ -5153,7 +6388,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -5176,7 +6411,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -5199,7 +6434,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -5220,7 +6455,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -5241,7 +6476,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -5262,7 +6497,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -5283,7 +6518,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -5304,7 +6539,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -5325,7 +6560,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -5348,7 +6583,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -5371,7 +6606,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -5394,7 +6629,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -5988,7 +7223,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6011,7 +7246,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6034,7 +7269,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6057,7 +7292,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6080,7 +7315,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6103,7 +7338,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6126,7 +7361,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6149,7 +7384,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6172,7 +7407,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6195,7 +7430,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6218,7 +7453,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6241,7 +7476,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6262,7 +7497,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6283,7 +7518,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6304,7 +7539,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6325,7 +7560,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6346,7 +7581,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6367,7 +7602,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6388,7 +7623,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6409,7 +7644,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6430,7 +7665,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6451,7 +7686,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6472,7 +7707,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6493,7 +7728,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6514,7 +7749,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6535,7 +7770,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6556,7 +7791,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6577,7 +7812,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6598,7 +7833,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw x710, 3n-skx x710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw x710 and 3n-skx x710 testbeds." @@ -6619,7 +7854,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw x710, 3n-skx x710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw x710 and 3n-skx x710 testbeds." @@ -6640,7 +7875,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -6661,7 +7896,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -6682,7 +7917,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -6703,7 +7938,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies."