X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fspecification.yaml;h=ff8c10544a226c3cb33e51c596810c9a48d603d8;hb=9e06ee18ddcd7287162afd6848121e9a094fee57;hp=492a595127e20975be59cb11ceb25a8df07af665;hpb=1a0f5d37b11bafc784985875540f0e99abec88c0;p=csit.git diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 492a595127..ff8c10544a 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -152,16 +152,26 @@ plot-vpp-throughput-lat-tsa-3n-hsw: csit-vpp-perf-report-iterative-2005-3n-hsw: - - 3 # NDRPDR sel - - 4 # NDRPDR sel - - 5 # NDRPDR sel - - 10 # NDRPDR sel - - 11 # NDRPDR sel - - 12 # NDRPDR sel - - 15 # NDRPDR sel - - 16 # NDRPDR sel - - 18 # NDRPDR sel - - 19 # NDRPDR sel +# - 3 # NDRPDR sel +# - 4 # NDRPDR sel +# - 5 # NDRPDR sel +# - 10 # NDRPDR sel +# - 11 # NDRPDR sel +# - 12 # NDRPDR sel +# - 15 # NDRPDR sel +# - 16 # NDRPDR sel +# - 18 # NDRPDR sel +# - 19 # NDRPDR sel + - 20 # NDRPDR sel env 4 + - 21 # NDRPDR sel env 4 + - 22 # NDRPDR sel env 4 + - 23 # NDRPDR sel env 4 + - 24 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 26 # NDRPDR sel env 4 + - 27 # NDRPDR sel env 4 + - 28 # NDRPDR sel env 4 + - 29 # NDRPDR sel env 4 plot-vpp-hdrh-lat-3n-hsw: csit-vpp-perf-report-iterative-2005-3n-hsw: @@ -218,16 +228,21 @@ plot-vpp-throughput-lat-tsa-3n-skx: csit-vpp-perf-report-iterative-2005-3n-skx: - - 4 # NDRPDR sel - - 5 # NDRPDR sel ??? all x710 failed - - 33 # NDRPDR sel - - 28 # NDRPDR sel - - 34 # NDRPDR sel - - 36 # NDRPDR sel - - 37 # NDRPDR sel - - 38 # NDRPDR sel - - 39 # NDRPDR sel - - 41 # NDRPDR sel +# - 4 # NDRPDR sel +# - 5 # NDRPDR sel ??? all x710 failed +# - 33 # NDRPDR sel +# - 28 # NDRPDR sel +# - 34 # NDRPDR sel +# - 36 # NDRPDR sel +# - 37 # NDRPDR sel +# - 38 # NDRPDR sel +# - 39 # NDRPDR sel +# - 41 # NDRPDR sel + - 42 # NDRPDR sel env 4 + - 43 # NDRPDR sel env 4 + - 44 # NDRPDR sel env 4 + - 45 # NDRPDR sel env 4 + - 47 # NDRPDR sel env 4 plot-vpp-hdrh-lat-3n-skx: csit-vpp-perf-report-iterative-2005-3n-skx: @@ -267,16 +282,23 @@ plot-vpp-throughput-lat-tsa-2n-skx: csit-vpp-perf-report-iterative-2005-2n-skx: - - 43 # NDRPDR sel - - 38 # NDRPDR sel - - 7 # NDRPDR sel - - 65 # NDRPDR sel - - 47 # NDRPDR sel - - 55 # NDRPDR sel - - 67 # NDRPDR sel - - 75 # NDRPDR sel - - 76 # NDRPDR sel - - 73 # NDRPDR sel +# - 43 # NDRPDR sel +# - 38 # NDRPDR sel +# - 7 # NDRPDR sel +# - 65 # NDRPDR sel +# - 47 # NDRPDR sel +# - 55 # NDRPDR sel +# - 67 # NDRPDR sel +# - 75 # NDRPDR sel +# - 76 # NDRPDR sel +# - 73 # NDRPDR sel + - 88 # NDRPDR sel env 4 + - 89 # NDRPDR sel env 4 + - 90 # NDRPDR sel env 4 + - 91 # NDRPDR sel env 4 + - 92 # NDRPDR sel env 4 + - 93 # NDRPDR sel env 4 + - 94 # NDRPDR sel env 4 plot-vpp-hdrh-lat-2n-skx: csit-vpp-perf-report-iterative-2005-2n-skx: @@ -293,6 +315,8 @@ - 43 # NDRPDR sel - 38 # NDRPDR sel - 7 # NDRPDR sel + - 78 # nf-density-sel + - 83 # nf-density-sel plot-vpp-nfv-2n-skx-mrr: csit-vpp-perf-report-iterative-2005-2n-skx: @@ -332,16 +356,24 @@ plot-vpp-throughput-lat-tsa-2n-clx: csit-vpp-perf-report-iterative-2005-2n-clx: - - 7 # NDRPDR sel - - 38 # NDRPDR sel - - 9 # NDRPDR sel - - 32 # NDRPDR sel ??? - - 33 # NDRPDR sel ??? - - 39 # NDRPDR sel - - 40 # NDRPDR sel - - 43 # NDRPDR sel - - 50 # NDRPDR sel - - 51 # NDRPDR sel +# - 7 # NDRPDR sel +# - 38 # NDRPDR sel +# - 9 # NDRPDR sel +# - 32 # NDRPDR sel ??? +# - 33 # NDRPDR sel ??? +# - 39 # NDRPDR sel +# - 40 # NDRPDR sel +# - 43 # NDRPDR sel +# - 50 # NDRPDR sel +# - 51 # NDRPDR sel + - 62 # NDRPDR sel env 4 + - 63 # NDRPDR sel env 4 + - 64 # NDRPDR sel env 4 + - 65 # NDRPDR sel env 4 + - 66 # NDRPDR sel env 4 + - 68 # NDRPDR sel env 4 + - 69 # NDRPDR sel env 4 + - 70 # NDRPDR sel env 4 plot-vpp-hdrh-lat-2n-clx: csit-vpp-perf-report-iterative-2005-2n-clx: @@ -409,11 +441,14 @@ plot-vpp-throughput-lat-tsa-3n-tsh: # TODO: Add data: csit-vpp-perf-report-iterative-2005-3n-tsh: - - 9 # NDRPDR sel - - 11 # NDRPDR sel - - 12 # NDRPDR sel - - 20 # NDRPDR sel - - 14 # NDRPDR sel +# - 9 # NDRPDR sel +# - 11 # NDRPDR sel +# - 12 # NDRPDR sel +# - 20 # NDRPDR sel +# - 14 # NDRPDR sel +# - 21 # NDRPDR sel + - 23 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 plot-vpp-hdrh-lat-3n-tsh: csit-vpp-perf-report-iterative-2005-3n-tsh: @@ -579,16 +614,23 @@ - 20 # perf-full-3n-skx-ip4_IX - 24 # perf-full-3n-skx-ip6_I - 25 # perf-full-3n-skx-ip6_II + - 21 # perf-full-3n-skx-ip4tunnels-I - 22 # perf-full-3n-skx-ip4tunnels-II - 23 # perf-full-3n-skx-ip4tunnels-III - 26 # perf-full-3n-skx-ip6tunnels-I - 28 # perf-full-3n-skx-l2_II + - 29 # perf-full-3n-skx-l2_III - 30 # perf-full-3n-skx-l2_IV - 32 # perf-full-3n-skx-l2_VI - 33 # perf-full-3n-skx-l2_VII + - 34 # perf-full-3n-skx-l2_VIII - 35 # perf-full-3n-skx-l2_IX - 36 # perf-full-3n-skx-l2_X - 37 # perf-full-3n-skx-l2_XI + - 38 # perf-full-3n-skx-memif + - 39 # perf-full-3n-skx-srv6-I + - 40 # perf-full-3n-skx-srv6-II + - 43 # perf-full-3n-skx-vhost-III vpp-perf-results-2n-skx: # TODO: Add data: @@ -753,6 +795,21 @@ - 16 # NDRPDR sel - 18 # NDRPDR sel - 19 # NDRPDR sel + vpp-performance-changes-3n-hsw-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-hsw: + - 20 # NDRPDR sel env 4 + - 21 # NDRPDR sel env 4 + - 22 # NDRPDR sel env 4 + - 23 # NDRPDR sel env 4 + - 24 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 26 # NDRPDR sel env 4 + - 27 # NDRPDR sel env 4 + - 28 # NDRPDR sel env 4 + - 29 # NDRPDR sel env 4 + vpp-performance-changes-3n-hsw-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-hsw: + - 31 # NDRPDR sel 2001.0-4 vpp-performance-changes-3n-skx-h1: csit-vpp-perf-verify-1908_1-3n-skx: @@ -787,6 +844,16 @@ - 38 # NDRPDR sel - 39 # NDRPDR sel - 41 # NDRPDR sel + vpp-performance-changes-3n-skx-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-skx: + - 42 # NDRPDR sel env 4 + - 43 # NDRPDR sel env 4 + - 44 # NDRPDR sel env 4 + - 45 # NDRPDR sel env 4 + - 47 # NDRPDR sel env 4 + vpp-performance-changes-3n-skx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-skx: + - 50 # NDRPDR sel 2001.0-4 vpp-performance-changes-2n-skx-h1: csit-vpp-perf-verify-1908_1-2n-skx: @@ -824,6 +891,18 @@ - 75 # NDRPDR sel - 76 # NDRPDR sel - 73 # NDRPDR sel + vpp-performance-changes-2n-skx-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-skx: + - 88 # NDRPDR sel env 4 + - 89 # NDRPDR sel env 4 + - 90 # NDRPDR sel env 4 + - 91 # NDRPDR sel env 4 + - 92 # NDRPDR sel env 4 + - 93 # NDRPDR sel env 4 + - 94 # NDRPDR sel env 4 + vpp-performance-changes-2n-skx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-2n-skx: + - 100 # NDRPDR sel 2001.0-4 # vpp-performance-changes-nfv-2n-skx-ref: # csit-vpp-perf-verify-2001-2n-skx: @@ -834,6 +913,8 @@ # - 43 # NDRPDR sel # - 38 # NDRPDR sel # - 7 # NDRPDR sel +# - 78 # nf-density-sel +# - 83 # nf-density-sel vpp-performance-changes-2n-clx-ref: csit-vpp-perf-verify-2001-2n-clx: @@ -859,6 +940,21 @@ - 43 # NDRPDR sel - 50 # NDRPDR sel - 51 # NDRPDR sel + vpp-performance-changes-2n-clx-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-clx: + - 62 # NDRPDR sel env 4 + - 63 # NDRPDR sel env 4 + - 64 # NDRPDR sel env 4 + - 65 # NDRPDR sel env 4 + - 66 # NDRPDR sel env 4 + - 68 # NDRPDR sel env 4 + - 69 # NDRPDR sel env 4 + - 70 # NDRPDR sel env 4 + vpp-performance-changes-2n-clx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-2n-clx: + - 73 # NDRPDR sel 2001.0-4 + - 74 # NDRPDR sel 2001.0-4 + - 75 # NDRPDR sel 2001.0-4 vpp-performance-changes-2n-dnv-h1: csit-vpp-perf-verify-1908-2n-dnv: @@ -966,6 +1062,14 @@ - 12 # NDRPDR sel - 20 # NDRPDR sel - 14 # NDRPDR sel + - 21 # NDRPDR sel + vpp-performance-changes-3n-tsh-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-tsh: + - 23 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + vpp-performance-changes-3n-tsh-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-tsh: + - 28 # NDRPDR sel 2001.0-4 vpp-soak-vs-ndr-2n-skx-ref: csit-vpp-perf-report-iterative-2005-2n-skx: @@ -1038,6 +1142,9 @@ vpp-performance-changes-3n-hsw-mrr-cmp: csit-vpp-perf-report-iterative-2005-3n-hsw: - 13 # MRR + vpp-performance-changes-3n-hsw-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-hsw: + - 30 # MRR env 4 # vpp-performance-changes-3n-skx-mrr-h1: # csit-vpp-perf-verify-1908-3n-skx: @@ -1049,6 +1156,9 @@ vpp-performance-changes-3n-skx-mrr-cmp: csit-vpp-perf-report-iterative-2005-3n-skx: - 6 # MRR sel + vpp-performance-changes-3n-skx-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-skx: + - 49 # MRR sel env 4 # vpp-performance-changes-2n-skx-mrr-h1: # csit-vpp-perf-verify-1908-2n-skx: @@ -1060,6 +1170,9 @@ vpp-performance-changes-2n-skx-mrr-cmp: csit-vpp-perf-report-iterative-2005-2n-skx: - 42 # MRR sel + vpp-performance-changes-2n-skx-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-skx: + - 99 # MRR sel env 4 vpp-performance-changes-2n-clx-mrr-ref: csit-vpp-perf-verify-2001-2n-clx: @@ -1067,6 +1180,9 @@ vpp-performance-changes-2n-clx-mrr-cmp: csit-vpp-perf-report-iterative-2005-2n-clx: - 12 # MRR + vpp-performance-changes-2n-clx-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-clx: + - 72 # MRR # vpp-performance-changes-nfv-2n-skx-mrr-ref: # csit-vpp-perf-verify-1908-2n-skx: @@ -1105,6 +1221,9 @@ vpp-performance-changes-3n-tsh-mrr-cmp: csit-vpp-perf-report-iterative-2005-3n-tsh: - 10 # MRR sel + vpp-performance-changes-3n-tsh-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-tsh: + - 27 # MRR sel env 4 dpdk-performance-changes-3n-hsw-h1: csit-dpdk-perf-verify-1908-3n-hsw: @@ -1643,120 +1762,6 @@ hoverlabel: namelength: -1 -# plot-cps: -# titlefont: -# size: 18 -# xaxis: -# title: "Test Cases [Index]" -# titlefont: -# size: 16 -# autorange: True -# fixedrange: False -# gridcolor: "rgb(230, 230, 230)" -# linecolor: "rgb(220, 220, 220)" -# linewidth: 1 -# showgrid: True -# showline: True -# showticklabels: True -# tickcolor: "rgb(220, 220, 220)" -# tickmode: "linear" -# tickfont: -# size: 16 -# zeroline: False -# yaxis: -# title: "Connections Per Second [cps]" -# titlefont: -# size: 16 -# gridcolor: "rgb(230, 230, 230)" -# hoverformat: ".4s" -# linecolor: "rgb(220, 220, 220)" -# linewidth: 1 -# rangemode: "tozero" -# showgrid: True -# showline: True -# showticklabels: True -# tickcolor: "rgb(220, 220, 220)" -# tickformat: ".3s" -# tickfont: -# size: 16 -# zeroline: False -# boxmode: "group" -# boxgroupgap: 0.5 -# autosize: False -# margin: -# t: 50 -# b: 20 -# l: 80 -# r: 20 -# showlegend: True -# legend: -# orientation: "h" -# font: -# size: 16 -# width: 700 -# height: 800 -# paper_bgcolor: "#fff" -# plot_bgcolor: "#fff" -# hoverlabel: -# namelength: -1 -# -# plot-rps: -# titlefont: -# size: 18 -# xaxis: -# title: "Test Cases [Index]" -# titlefont: -# size: 16 -# autorange: True -# fixedrange: False -# gridcolor: "rgb(230, 230, 230)" -# linecolor: "rgb(220, 220, 220)" -# linewidth: 1 -# showgrid: True -# showline: True -# showticklabels: True -# tickcolor: "rgb(220, 220, 220)" -# tickmode: "linear" -# tickfont: -# size: 16 -# zeroline: False -# yaxis: -# title: "Requests Per Second [rps]" -# titlefont: -# size: 16 -# gridcolor: "rgb(230, 230, 230)" -# hoverformat: ".4s" -# linecolor: "rgb(220, 220, 220)" -# linewidth: 1 -# rangemode: "tozero" -# showgrid: True -# showline: True -# showticklabels: True -# tickcolor: "rgb(230, 230, 230)" -# tickformat: ".3s" -# tickfont: -# size: 16 -# zeroline: False -# boxmode: "group" -# boxgroupgap: 0.5 -# autosize: False -# margin: -# t: 50 -# b: 20 -# l: 80 -# r: 20 -# showlegend: True -# legend: -# orientation: "h" -# font: -# size: 16 -# width: 700 -# height: 800 -# paper_bgcolor: "#fff" -# plot_bgcolor: "#fff" -# hoverlabel: -# namelength: -1 - plot-hoststack: titlefont: size: 16 @@ -2242,7 +2247,19 @@ - 16 # NDRPDR sel - 18 # NDRPDR sel - 19 # NDRPDR sel + - 20 # NDRPDR sel env 4 + - 21 # NDRPDR sel env 4 + - 22 # NDRPDR sel env 4 + - 23 # NDRPDR sel env 4 + - 24 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 26 # NDRPDR sel env 4 + - 27 # NDRPDR sel env 4 + - 28 # NDRPDR sel env 4 + - 29 # NDRPDR sel env 4 - 13 # MRR + - 30 # MRR env 4 + - 31 # NDRPDR sel 2001.0-4 csit-vpp-perf-verify-2005-3n-hsw: - 3 # hoststack - 5 # hoststack @@ -2324,7 +2341,14 @@ - 38 # NDRPDR sel - 39 # NDRPDR sel - 41 # NDRPDR sel + - 42 # NDRPDR sel env 4 + - 43 # NDRPDR sel env 4 + - 44 # NDRPDR sel env 4 + - 45 # NDRPDR sel env 4 + - 47 # NDRPDR sel env 4 + - 50 # NDRPDR sel 2001.0-4 - 6 # MRR sel + - 49 # MRR sel env 4 csit-vpp-perf-report-coverage-2005-3n-skx: - 10 # perf-full-3n-skx-crypto_I - 11 # perf-full-3n-skx-crypto_II @@ -2340,16 +2364,23 @@ - 20 # perf-full-3n-skx-ip4_IX - 24 # perf-full-3n-skx-ip6_I - 25 # perf-full-3n-skx-ip6_II + - 21 # perf-full-3n-skx-ip4tunnels-I - 22 # perf-full-3n-skx-ip4tunnels-II - 23 # perf-full-3n-skx-ip4tunnels-III - 26 # perf-full-3n-skx-ip6tunnels-I - 28 # perf-full-3n-skx-l2_II + - 29 # perf-full-3n-skx-l2_III - 30 # perf-full-3n-skx-l2_IV - 32 # perf-full-3n-skx-l2_VI - 33 # perf-full-3n-skx-l2_VII + - 34 # perf-full-3n-skx-l2_VIII - 35 # perf-full-3n-skx-l2_IX - 36 # perf-full-3n-skx-l2_X - 37 # perf-full-3n-skx-l2_XI + - 38 # perf-full-3n-skx-memif + - 39 # perf-full-3n-skx-srv6-I + - 40 # perf-full-3n-skx-srv6-II + - 43 # perf-full-3n-skx-vhost-III csit-vpp-perf-verify-1908_1-2n-skx: - 39 # NDRPDR sel - new1908.1 @@ -2386,7 +2417,16 @@ - 75 # NDRPDR sel - 76 # NDRPDR sel - 73 # NDRPDR sel + - 88 # NDRPDR sel env 4 + - 89 # NDRPDR sel env 4 + - 90 # NDRPDR sel env 4 + - 91 # NDRPDR sel env 4 + - 92 # NDRPDR sel env 4 + - 93 # NDRPDR sel env 4 + - 94 # NDRPDR sel env 4 + - 100 # NDRPDR sel 2001.0-4 - 42 # MRR sel + - 99 # MRR sel env 4 - 48 # Reconf - 50 # Reconf - 51 # Reconf @@ -2399,6 +2439,8 @@ - 66 # Reconf - 49 # Soak - 54 # Soak + - 78 # nf-density-sel + - 83 # nf-density-sel csit-vpp-perf-report-coverage-2005-2n-skx: - 11 # perf-full-2n-skx-ip4_I - 3 # perf-full-2n-skx-ip4_II @@ -2445,7 +2487,19 @@ - 43 # NDRPDR sel - 50 # NDRPDR sel - 51 # NDRPDR sel + - 62 # NDRPDR sel env 4 + - 63 # NDRPDR sel env 4 + - 64 # NDRPDR sel env 4 + - 65 # NDRPDR sel env 4 + - 66 # NDRPDR sel env 4 + - 68 # NDRPDR sel env 4 + - 69 # NDRPDR sel env 4 + - 70 # NDRPDR sel env 4 + - 73 # NDRPDR sel 2001.0-4 + - 74 # NDRPDR sel 2001.0-4 + - 75 # NDRPDR sel 2001.0-4 - 12 # MRR + - 72 # MRR env 4 - 42 # Reconf - 45 # Reconf - 46 # Reconf @@ -2511,7 +2565,12 @@ - 12 # NDRPDR sel - 20 # NDRPDR sel - 14 # NDRPDR sel + - 21 # NDRPDR sel + - 23 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 28 # NDRPDR sel 2001.0-4 - 10 # MRR sel + - 27 # MRR sel env 4 csit-vpp-perf-verify-1908-3n-dnv: - 6 # NDRPDR sel @@ -3209,132 +3268,6 @@ ################################################################################ # VPP -## VPP Performance changes 2n-skx-xxv710 2t1c ndr -#- type: "table" -# title: "VPP Performance changes 2n-skx-xxv710 2t1c ndr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-changes-1908_2-2n-skx-2t1c-ndr" -# include-tests: "NDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# columns: -# - title: "1908.1" -# data-set: "vpp-perf-changes-2n-skx-new1908_1" -# - title: "1908.2" -# data-set: "vpp-perf-changes-2n-skx-new1908_2" -# comparisons: -# - title: "Diff" -# reference: 1 -# compare: 2 -# legend: -# - "Legend to Table:" -# - "1908.1, 1908.2 - VPP versions tested." -# - "Diff - difference between the previous and the current VPP version." -# -## VPP Performance changes 2n-skx-xxv710 2t1c pdr -#- type: "table" -# title: "VPP Performance changes 2n-skx-xxv710 2t1c pdr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-changes-1908_2-2n-skx-2t1c-pdr" -# include-tests: "PDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# columns: -# - title: "1908.1" -# data-set: "vpp-perf-changes-2n-skx-new1908_1" -# - title: "1908.2" -# data-set: "vpp-perf-changes-2n-skx-new1908_2" -# comparisons: -# - title: "Diff" -# reference: 1 -# compare: 2 -# legend: -# - "Legend to Table:" -# - "1908.1, 1908.2 - VPP versions tested." -# - "Diff - difference between the previous and the current VPP version." -# -## VPP Performance changes 3n-skx-xxv710 2t1c ndr -#- type: "table" -# title: "VPP Performance changes 3n-skx-xxv710 2t1c ndr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-changes-1908_2-3n-skx-2t1c-ndr" -# include-tests: "NDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# columns: -# - title: "1908.1" -# data-set: "vpp-perf-changes-3n-skx-new1908_1" -# - title: "1908.2" -# data-set: "vpp-perf-changes-3n-skx-new1908_2" -# comparisons: -# - title: "Diff" -# reference: 1 -# compare: 2 -# legend: -# - "Legend to Table:" -# - "1908.1, 1908.2 - VPP versions tested." -# - "Diff - difference between the previous and the current VPP version." -# -## VPP Performance changes 3n-skx-xxv710 2t1c pdr -#- type: "table" -# title: "VPP Performance changes 3n-skx-xxv710 2t1c pdr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-changes-1908_2-3n-skx-2t1c-pdr" -# include-tests: "PDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# columns: -# - title: "1908.1" -# data-set: "vpp-perf-changes-3n-skx-new1908_1" -# - title: "1908.2" -# data-set: "vpp-perf-changes-3n-skx-new1908_2" -# comparisons: -# - title: "Diff" -# reference: 1 -# compare: 2 -# legend: -# - "Legend to Table:" -# - "1908.1, 1908.2 - VPP versions tested." -# - "Diff - difference between the previous and the current VPP version." -# -## VPP Performance changes 2n-clx-xxv710 2t1c ndr -#- type: "table" -# title: "VPP Performance changes 2n-clx-xxv710 2t1c ndr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-changes-1908_2-2n-clx-2t1c-ndr" -# include-tests: "NDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# columns: -# - title: "1908.1" -# data-set: "vpp-perf-changes-2n-clx-new1908_1" -# - title: "1908.2" -# data-set: "vpp-perf-changes-2n-clx-new1908_2" -# comparisons: -# - title: "Diff" -# reference: 1 -# compare: 2 -# legend: -# - "Legend to Table:" -# - "1908.1, 1908.2 - VPP versions tested." -# - "Diff - difference between the previous and the current VPP version." -# -## VPP Performance changes 2n-clx-xxv710 2t1c pdr -#- type: "table" -# title: "VPP Performance changes 2n-clx-xxv710 2t1c pdr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-changes-1908_2-2n-clx-2t1c-pdr" -# include-tests: "PDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# columns: -# - title: "1908.1" -# data-set: "vpp-perf-changes-2n-clx-new1908_1" -# - title: "1908.2" -# data-set: "vpp-perf-changes-2n-clx-new1908_2" -# comparisons: -# - title: "Diff" -# reference: 1 -# compare: 2 -# legend: -# - "Legend to Table:" -# - "1908.1, 1908.2 - VPP versions tested." -# - "Diff - difference between the previous and the current VPP version." - # VPP Performance Changes 3n-hsw 1t1c pdr - type: "table" title: "VPP Performance Changes 3n-hsw 1t1c pdr" @@ -3343,20 +3276,35 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2008 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-hsw 2t2c pdr - type: "table" @@ -3366,20 +3314,35 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2008 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-hsw 1t1c ndr - type: "table" @@ -3389,20 +3352,35 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca2.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca3.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2008 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-hsw 2t2c ndr - type: "table" @@ -3412,165 +3390,337 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 - legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2008 VPP version." - -## VPP Root Cause Analysis 3n-skx-xxv710 2t1c pdr -#- type: "table" -# title: "VPP Root Cause Analysis 3n-skx-xxv710 2t1c pdr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-rca-3n-skx-2t1c-pdr" -# include-tests: "PDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# rca: -# - title: "RCA1" -# data: "rca/rca-3n-skx-2t1c-pdr-rca1.yaml" -# - title: "RCA2" -# data: "rca/rca-3n-skx-2t1c-pdr-rca2.yaml" -# columns: -# - title: "1908.0-1" -# data-set: "vpp-perf-changes-3n-skx-old1908_0" -# data-replacement: "replacement-1908-vpp-throughput-lat-tsa-3n-skx-vhost" -# - title: "1908.1-1" -# data-set: "vpp-perf-changes-3n-skx-old1908_1" -# - title: "1908.1-2" -# data-set: "vpp-perf-changes-3n-skx-new1908_1" -# - title: "2001.0-2" -# data-set: "vpp-perf-changes-3n-skx-new2001_0" -# comparisons: -# - title: "Diff1" -# reference: 2 -# reference-alt: 1 -# compare: 3 -# - title: "Diff2" -# reference: 3 -# compare: 4 -# legend: -# - 'Legend to Table:' -# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).' -# - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).' -# - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' -# -## VPP Root Cause Analysis 3n-skx-xxv710 2t1c ndr -#- type: "table" -# title: "VPP Root Cause Analysis 3n-skx-xxv710 2t1c ndr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-rca-3n-skx-2t1c-ndr" -# include-tests: "NDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 # rca: # - title: "RCA1" -# data: "rca/rca-3n-skx-2t1c-ndr-rca1.yaml" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca1.yaml" # - title: "RCA2" -# data: "rca/rca-3n-skx-2t1c-ndr-rca2.yaml" -# columns: -# - title: "1908.0-1" -# data-set: "vpp-perf-changes-3n-skx-old1908_0" -# data-replacement: "replacement-1908-vpp-throughput-lat-tsa-3n-skx-vhost" -# - title: "1908.1-1" -# data-set: "vpp-perf-changes-3n-skx-old1908_1" -# - title: "1908.1-2" -# data-set: "vpp-perf-changes-3n-skx-new1908_1" -# - title: "2001.0-2" -# data-set: "vpp-perf-changes-3n-skx-new2001_0" -# comparisons: -# - title: "Diff1" -# reference: 2 -# reference-alt: 1 -# compare: 3 -# - title: "Diff2" -# reference: 3 -# compare: 4 -# legend: -# - 'Legend to Table:' -# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).' -# - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).' -# - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' - -# VPP Performance Changes 3n-skx 2t1c pdr +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 1t1c pdr-full - type: "table" - title: "VPP Performance Changes 3n-skx 2t1c pdr" + title: "VPP Performance Changes 3n-hsw 1t1c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" - filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-skx-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca3.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 3n-skx 4t2c pdr + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 2t2c pdr-full - type: "table" - title: "VPP Performance Changes 3n-skx 4t2c pdr" + title: "VPP Performance Changes 3n-hsw 2t2c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" - filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-skx-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca3.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 3n-skx 2t1c ndr + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 1t1c ndr-full - type: "table" - title: "VPP Performance Changes 3n-skx 2t1c ndr" + title: "VPP Performance Changes 3n-hsw 1t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" - filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 2t2c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-hsw 2t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c pdr +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c pdr +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c ndr +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-skx 4t2c ndr - type: "table" @@ -3580,96 +3730,223 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca3.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -## VPP Root Cause Analysis 2n-skx-xxv710 2t1c pdr -#- type: "table" -# title: "VPP Root Cause Analysis 2n-skx-xxv710 2t1c pdr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-rca-2n-skx-2t1c-pdr" -# include-tests: "PDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" -# data: "rca/rca-2n-skx-2t1c-pdr-rca1.yaml" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" # - title: "RCA2" -# data: "rca/rca-2n-skx-2t1c-pdr-rca2.yaml" -# columns: -# - title: "1908.0-1" -# data-set: "vpp-perf-changes-2n-skx-old1908_0" -# data-replacement: "replacement-1908-vpp-throughput-lat-tsa-2n-skx-vhost" -# - title: "1908.1-1" -# data-set: "vpp-perf-changes-2n-skx-old1908_1" -# - title: "1908.1-2" -# data-set: "vpp-perf-changes-2n-skx-new1908_1" -# - title: "2001.0-2" -# data-set: "vpp-perf-changes-2n-skx-new2001_0" -# comparisons: -# - title: "Diff1" -# reference: 2 -# reference-alt: 1 -# compare: 3 -# - title: "Diff2" -# reference: 3 -# compare: 4 -# legend: -# - 'Legend to Table:' -# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).' -# - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).' -# - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' -# -## VPP Root Cause Analysis 2n-skx-xxv710 2t1c ndr -#- type: "table" -# title: "VPP Root Cause Analysis 2n-skx-xxv710 2t1c ndr" -# algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-rca-2n-skx-2t1c-ndr" -# include-tests: "NDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" -# data: "rca/rca-2n-skx-2t1c-ndr-rca1.yaml" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" # - title: "RCA2" -# data: "rca/rca-2n-skx-2t1c-ndr-rca2.yaml" -# columns: -# - title: "1908.0-1" -# data-set: "vpp-perf-changes-2n-skx-old1908_0" -# data-replacement: "replacement-1908-vpp-throughput-lat-tsa-2n-skx-vhost" -# - title: "1908.1-1" -# data-set: "vpp-perf-changes-2n-skx-old1908_1" -# - title: "1908.1-2" -# data-set: "vpp-perf-changes-2n-skx-new1908_1" -# - title: "2001.0-2" -# data-set: "vpp-perf-changes-2n-skx-new2001_0" -# comparisons: -# - title: "Diff1" -# reference: 2 -# reference-alt: 1 -# compare: 3 -# - title: "Diff2" -# reference: 3 -# compare: 4 -# legend: -# - 'Legend to Table:' -# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).' -# - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).' -# - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 2t1c pdr - type: "table" @@ -3679,20 +3956,35 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-2n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 4t2c pdr - type: "table" @@ -3702,20 +3994,35 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-2n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 2t1c ndr - type: "table" @@ -3725,20 +4032,35 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-2n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 4t2c ndr - type: "table" @@ -3748,83 +4070,337 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-2n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 2n-clx-xxv710 2t1c pdr + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c pdr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 2t1c pdr" + title: "VPP Performance Changes 2n-skx 2t1c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" - data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" - data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "Diff" - reference: 1 - compare: 2 + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca3.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 2n-clx-xxv710 4t2c pdr + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 4t2c pdr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr" + title: "VPP Performance Changes 2n-skx 4t2c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" - data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" - data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "Diff" - reference: 1 - compare: 2 + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca3.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 2n-clx-xxv710 2t1c ndr + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c ndr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr" + title: "VPP Performance Changes 2n-skx 2t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" - data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" - data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "Diff" - reference: 1 - compare: 2 + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca3.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 4t2c ndr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 4t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c pdr +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c pdr +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c ndr +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-xxv710 4t2c ndr - type: "table" @@ -3834,18 +4410,223 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - - title: "Diff" - reference: 1 - compare: 2 + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca3.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c ndr-full +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-x556 2t1c pdr - type: "table" @@ -3855,9 +4636,9 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Mellanox-CX556A' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" @@ -3866,7 +4647,7 @@ legend: - "Legend to Table" - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." # VPP Performance Changes 2n-clx-x556 4t2c pdr - type: "table" @@ -3876,9 +4657,9 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Mellanox-CX556A' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" @@ -3887,7 +4668,7 @@ legend: - "Legend to Table" - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." # VPP Performance Changes 2n-clx-x556 2t1c ndr - type: "table" @@ -3897,9 +4678,9 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Mellanox-CX556A' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" @@ -3908,7 +4689,7 @@ legend: - "Legend to Table" - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." # VPP Performance Changes 2n-clx-x556 4t2c ndr - type: "table" @@ -3918,9 +4699,9 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Mellanox-CX556A' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" @@ -3929,7 +4710,7 @@ legend: - "Legend to Table" - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." # VPP Performance Changes 2n-dnv 1t1c pdr - type: "table" @@ -4123,20 +4904,35 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-tsh 2t2c pdr - type: "table" @@ -4146,20 +4942,35 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-tsh 1t1c ndr - type: "table" @@ -4169,20 +4980,35 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-tsh 2t2c ndr - type: "table" @@ -4192,65 +5018,268 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 1t1c pdr-full - type: "table" - title: "VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr" + title: "VPP Performance Changes 3n-tsh 1t1c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-1c-ndr" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '1C'" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C'" columns: - - title: "3n-hsw xl710" - data-set: "vpp-compare-testbeds-ref" - tag: "NIC_Intel-XL710" - - title: "3n-skx xxv710" - data-set: "vpp-compare-testbeds-cmp" - tag: "NIC_Intel-XXV710" + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff" - reference: 1 - compare: 2 + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca3.yaml" legend: - - "Legend to Table" - - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." - -# VPP Comparison Across Testbeds 3n-hsw to 3n-skx 2c ndr + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c pdr-full - type: "table" - title: "VPP Comparison Across Testbeds 3n-hsw to 3n-skx 2c ndr" + title: "VPP Performance Changes 3n-tsh 2t2c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-2c-ndr" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '2C'" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C'" columns: - - title: "3n-hsw xl710" - data-set: "vpp-compare-testbeds-ref" - tag: "NIC_Intel-XL710" - - title: "3n-skx xxv710" - data-set: "vpp-compare-testbeds-cmp" - tag: "NIC_Intel-XXV710" + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff" - reference: 1 - compare: 2 + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca3.yaml" legend: - - "Legend to Table" - - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 1t1c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-tsh 1t1c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-tsh 2t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr +- type: "table" + title: "VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '1C'" + columns: + - title: "3n-hsw xl710" + data-set: "vpp-compare-testbeds-ref" + tag: "NIC_Intel-XL710" + - title: "3n-skx xxv710" + data-set: "vpp-compare-testbeds-cmp" + tag: "NIC_Intel-XXV710" + comparisons: + - title: "Diff" + reference: 1 + compare: 2 + legend: + - "Legend to Table" + - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." + - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." + +# VPP Comparison Across Testbeds 3n-hsw to 3n-skx 2c ndr +- type: "table" + title: "VPP Comparison Across Testbeds 3n-hsw to 3n-skx 2c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '2C'" + columns: + - title: "3n-hsw xl710" + data-set: "vpp-compare-testbeds-ref" + tag: "NIC_Intel-XL710" + - title: "3n-skx xxv710" + data-set: "vpp-compare-testbeds-cmp" + tag: "NIC_Intel-XXV710" + comparisons: + - title: "Diff" + reference: 1 + compare: 2 + legend: + - "Legend to Table" + - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c pdr @@ -4567,20 +5596,35 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'1T1C' and 'MRR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-hsw 2t2c mrr - type: "table" @@ -4590,20 +5634,35 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T2C' and 'MRR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-2t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-hsw 4t4c mrr - type: "table" @@ -4613,20 +5672,36 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T4C' and 'MRR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-4t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-4t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + # VPP Performance Changes 3n-skx 2t1c mrr - type: "table" @@ -4638,18 +5713,33 @@ columns: # - title: "1908" # data-set: "vpp-performance-changes-3n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-skx 4t2c mrr - type: "table" @@ -4661,18 +5751,33 @@ columns: # - title: "1908" # data-set: "vpp-performance-changes-3n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-skx 8t4c mrr - type: "table" @@ -4684,18 +5789,33 @@ columns: # - title: "1908" # data-set: "vpp-performance-changes-3n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-8t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-8t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-skx 2t1c mrr - type: "table" @@ -4705,20 +5825,33 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: -# - title: "1908" -# data-set: "vpp-performance-changes-2n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-skx 4t2c mrr - type: "table" @@ -4728,20 +5861,33 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: -# - title: "1908" -# data-set: "vpp-performance-changes-2n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-skx 8t4c mrr - type: "table" @@ -4751,20 +5897,33 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'8T4C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: -# - title: "1908" -# data-set: "vpp-performance-changes-2n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-8t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-8t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-xxv710 2t1c mrr - type: "table" @@ -4774,18 +5933,33 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-xxv710 4t2c mrr - type: "table" @@ -4795,18 +5969,33 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-xxv710 8t4c mrr - type: "table" @@ -4816,18 +6005,33 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'8T4C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-8t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-8t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-clx-x556 2t1c mrr - type: "table" @@ -5038,20 +6242,35 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-tsh 2t2c mrr - type: "table" @@ -5061,20 +6280,35 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-tsh 4t4c mrr - type: "table" @@ -5084,20 +6318,35 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T4C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-4t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-4t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c mrr - type: "table" @@ -7393,622 +8642,52 @@ - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-4Ch-1Ach-8Vh-4Vm1T-Testpmd-Reconf.IMIX-2t1c-avf-dot1qip4vxlan-l2bd-4ch-1ach-8vh-4vm1t-testpmd-reconf" - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-6Ch-1Ach-12Vh-6Vm1T-Testpmd-Reconf.IMIX-2t1c-avf-dot1qip4vxlan-l2bd-6ch-1ach-12vh-6vm1t-testpmd-reconf" layout: - title: "dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-testpmd" - layout: "plot-throughput" - -- type: "plot" - title: "NFV Effective Blocked Time: 2n-clx-xxv710-imix-4t2c-dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-reconf" - algorithm: "plot_nf_reconf_box_name" - output-file: "{DIR[STATIC,VPP]}/2n-clx-xxv710-imix-4t2c-dot1qip4vxlan-l2bd-reconf" - data: "plot-vpp-nfv-reconf-2n-clx" - include: - - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-1Ch-1Ach-2Vh-1Vm1T-Testpmd-Reconf.IMIX-4t2c-avf-dot1qip4vxlan-l2bd-1ch-1ach-2vh-1vm1t-testpmd-reconf" - - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-2Ch-1Ach-4Vh-2Vm1T-Testpmd-Reconf.IMIX-4t2c-avf-dot1qip4vxlan-l2bd-2ch-1ach-4vh-2vm1t-testpmd-reconf" - - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-4Ch-1Ach-8Vh-4Vm1T-Testpmd-Reconf.IMIX-4t2c-avf-dot1qip4vxlan-l2bd-4ch-1ach-8vh-4vm1t-testpmd-reconf" - - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-6Ch-1Ach-12Vh-6Vm1T-Testpmd-Reconf.IMIX-4t2c-avf-dot1qip4vxlan-l2bd-6ch-1ach-12vh-6vm1t-testpmd-reconf" - layout: - title: "dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-testpmd" - layout: "plot-throughput" - -- type: "plot" - title: "NFV Effective Blocked Time: 2n-clx-xxv710-imix-8t4c-dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-reconf" - algorithm: "plot_nf_reconf_box_name" - output-file: "{DIR[STATIC,VPP]}/2n-clx-xxv710-imix-8t4c-dot1qip4vxlan-l2bd-reconf" - data: "plot-vpp-nfv-reconf-2n-clx" - include: - - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-1Ch-1Ach-2Vh-1Vm1T-Testpmd-Reconf.IMIX-8t4c-avf-dot1qip4vxlan-l2bd-1ch-1ach-2vh-1vm1t-testpmd-reconf" - - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-2Ch-1Ach-4Vh-2Vm1T-Testpmd-Reconf.IMIX-8t4c-avf-dot1qip4vxlan-l2bd-2ch-1ach-4vh-2vm1t-testpmd-reconf" - - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-4Ch-1Ach-8Vh-4Vm1T-Testpmd-Reconf.IMIX-8t4c-avf-dot1qip4vxlan-l2bd-4ch-1ach-8vh-4vm1t-testpmd-reconf" - - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-6Ch-1Ach-12Vh-6Vm1T-Testpmd-Reconf.IMIX-8t4c-avf-dot1qip4vxlan-l2bd-6ch-1ach-12vh-6vm1t-testpmd-reconf" - layout: - title: "dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-testpmd" - layout: "plot-throughput" - -############################################################################### -# Plots - NF Density - VSC - MRR 64b -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '2T1C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '4T2C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '8T4C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - layout: "plot-service-density" - -# Plots - NF Density - VSC - MRR IMIX -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '2T1C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '4T2C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '8T4C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" - layout: "plot-service-density" - -# Plots - NF Density - VSC - NDR 64b -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '2T1C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '4T2C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '8T4C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - layout: "plot-service-density" - -# Plots - NF Density - VSC - NDR IMIX -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '2T1C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '4T2C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '8T4C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" - layout: "plot-service-density" - -# Plots - NF Density - VSC - PDR 64b -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '2T1C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '4T2C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '8T4C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - layout: "plot-service-density" - -# Plots - NF Density - VSC - PDR IMIX -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '2T1C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '4T2C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - not 'VXLAN' and - 'CHAIN' and - 'NF_VPPIP4' and - 'IMIX' and - '8T4C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" - layout: "plot-service-density" - -############################################################################### -# Plots - NF Density - VSC - MRR 64b -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-vxlan-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - 'VXLAN' and - 'CHAIN' and - 'IMIX' and - '2T1C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-vxlan-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - 'VXLAN' and - 'CHAIN' and - 'IMIX' and - '4T2C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" - layout: "plot-service-density" + title: "dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-testpmd" + layout: "plot-throughput" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-vxlan-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - 'VXLAN' and - 'CHAIN' and - 'IMIX' and - '8T4C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" + title: "NFV Effective Blocked Time: 2n-clx-xxv710-imix-4t2c-dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-reconf" + algorithm: "plot_nf_reconf_box_name" + output-file: "{DIR[STATIC,VPP]}/2n-clx-xxv710-imix-4t2c-dot1qip4vxlan-l2bd-reconf" + data: "plot-vpp-nfv-reconf-2n-clx" + include: + - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-1Ch-1Ach-2Vh-1Vm1T-Testpmd-Reconf.IMIX-4t2c-avf-dot1qip4vxlan-l2bd-1ch-1ach-2vh-1vm1t-testpmd-reconf" + - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-2Ch-1Ach-4Vh-2Vm1T-Testpmd-Reconf.IMIX-4t2c-avf-dot1qip4vxlan-l2bd-2ch-1ach-4vh-2vm1t-testpmd-reconf" + - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-4Ch-1Ach-8Vh-4Vm1T-Testpmd-Reconf.IMIX-4t2c-avf-dot1qip4vxlan-l2bd-4ch-1ach-8vh-4vm1t-testpmd-reconf" + - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-6Ch-1Ach-12Vh-6Vm1T-Testpmd-Reconf.IMIX-4t2c-avf-dot1qip4vxlan-l2bd-6ch-1ach-12vh-6vm1t-testpmd-reconf" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" - layout: "plot-service-density" + title: "dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-testpmd" + layout: "plot-throughput" + +- type: "plot" + title: "NFV Effective Blocked Time: 2n-clx-xxv710-imix-8t4c-dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-reconf" + algorithm: "plot_nf_reconf_box_name" + output-file: "{DIR[STATIC,VPP]}/2n-clx-xxv710-imix-8t4c-dot1qip4vxlan-l2bd-reconf" + data: "plot-vpp-nfv-reconf-2n-clx" + include: + - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-1Ch-1Ach-2Vh-1Vm1T-Testpmd-Reconf.IMIX-8t4c-avf-dot1qip4vxlan-l2bd-1ch-1ach-2vh-1vm1t-testpmd-reconf" + - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-2Ch-1Ach-4Vh-2Vm1T-Testpmd-Reconf.IMIX-8t4c-avf-dot1qip4vxlan-l2bd-2ch-1ach-4vh-2vm1t-testpmd-reconf" + - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-4Ch-1Ach-8Vh-4Vm1T-Testpmd-Reconf.IMIX-8t4c-avf-dot1qip4vxlan-l2bd-4ch-1ach-8vh-4vm1t-testpmd-reconf" + - "Tests.Vpp.Perf.Nfv Density.Vm Vhost.Chain Dot1Qip4Vxlan.2N-25Ge2P1Xxv710-Avf-Dot1Qip4Vxlan-L2Bd-6Ch-1Ach-12Vh-6Vm1T-Testpmd-Reconf.IMIX-8t4c-avf-dot1qip4vxlan-l2bd-6ch-1ach-12vh-6vm1t-testpmd-reconf" + layout: + title: "dot1qip4vxlan-l2bd-{Y}ch-1ach-{2XY}vh-{XY}vm-testpmd" + layout: "plot-throughput" + +############################################################################### # Plots - NF Density - VSC - MRR IMIX - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-vxlan-mrr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-mrr" data: "plot-vpp-nfv-2n-skx-mrr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'VHOST' and - 'VXLAN' and + not 'VXLAN' and 'CHAIN' and + 'NF_VPPIP4' and 'IMIX' and '2T1C' and 'MRR'" @@ -8021,20 +8700,21 @@ y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-vxlan-mrr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-mrr" data: "plot-vpp-nfv-2n-skx-mrr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'VHOST' and - 'VXLAN' and + not 'VXLAN' and 'CHAIN' and + 'NF_VPPIP4' and 'IMIX' and '4T2C' and 'MRR'" @@ -8047,20 +8727,21 @@ y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-vxlan-mrr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-mrr" data: "plot-vpp-nfv-2n-skx-mrr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'VHOST' and - 'VXLAN' and + not 'VXLAN' and 'CHAIN' and + 'NF_VPPIP4' and 'IMIX' and '8T4C' and 'MRR'" @@ -8073,100 +8754,22 @@ y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" - layout: "plot-service-density" - -# Plots - NF Density - VSC - NDR 64b -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-vxlan-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - 'VXLAN' and - 'CHAIN' and - 'IMIX' and - '2T1C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-vxlan-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - 'VXLAN' and - 'CHAIN' and - 'IMIX' and - '4T2C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-vxlan-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - 'VXLAN' and - 'CHAIN' and - 'IMIX' and - '8T4C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-mrr" layout: "plot-service-density" # Plots - NF Density - VSC - NDR IMIX - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-vxlan-ndr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-ndr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'VHOST' and - 'VXLAN' and + not 'VXLAN' and 'CHAIN' and + 'NF_VPPIP4' and 'IMIX' and '2T1C' and 'NDRPDR'" @@ -8179,20 +8782,21 @@ y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-vxlan-ndr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-ndr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'VHOST' and - 'VXLAN' and + not 'VXLAN' and 'CHAIN' and + 'NF_VPPIP4' and 'IMIX' and '4T2C' and 'NDRPDR'" @@ -8205,99 +8809,21 @@ y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-vxlan-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - 'VXLAN' and - 'CHAIN' and - 'IMIX' and - '8T4C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" - layout: "plot-service-density" - -# Plots - NF Density - VSC - PDR 64b -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-vxlan-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - 'VXLAN' and - 'CHAIN' and - 'IMIX' and - '2T1C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-vxlan-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'VHOST' and - 'VXLAN' and - 'CHAIN' and - 'IMIX' and - '4T2C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = VNFs per Service Chain" - y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-vxlan-pdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-ndr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'VHOST' and - 'VXLAN' and + not 'VXLAN' and 'CHAIN' and + 'NF_VPPIP4' and 'IMIX' and '8T4C' and 'NDRPDR'" @@ -8305,26 +8831,27 @@ - "throughput" - "name" - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-ndr" layout: "plot-service-density" # Plots - NF Density - VSC - PDR IMIX - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-vxlan-pdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-pdr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'VHOST' and - 'VXLAN' and + not 'VXLAN' and 'CHAIN' and + 'NF_VPPIP4' and 'IMIX' and '2T1C' and 'NDRPDR'" @@ -8337,20 +8864,21 @@ y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-vxlan-pdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-pdr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'VHOST' and - 'VXLAN' and + not 'VXLAN' and 'CHAIN' and + 'NF_VPPIP4' and 'IMIX' and '4T2C' and 'NDRPDR'" @@ -8363,20 +8891,21 @@ y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-vxlan-pdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-pdr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'VHOST' and - 'VXLAN' and + not 'VXLAN' and 'CHAIN' and + 'NF_VPPIP4' and 'IMIX' and '8T4C' and 'NDRPDR'" @@ -8389,22 +8918,23 @@ y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vppip4-pdr" layout: "plot-service-density" ############################################################################### -# Plots - NF Density - CSC - MRR 64b + +# Plots - NF Density - VSC - MRR IMIX - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csc-mrr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-vxlan-mrr" data: "plot-vpp-nfv-2n-skx-mrr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and - 'DOCKER' and + 'VHOST' and + 'VXLAN' and 'CHAIN' and - 'NF_VPPIP4' and 'IMIX' and '2T1C' and 'MRR'" @@ -8413,24 +8943,24 @@ - "name" - "tags" include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Chain" + x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csc-mrr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-vxlan-mrr" data: "plot-vpp-nfv-2n-skx-mrr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and - 'DOCKER' and + 'VHOST' and + 'VXLAN' and 'CHAIN' and - 'NF_VPPIP4' and 'IMIX' and '4T2C' and 'MRR'" @@ -8439,24 +8969,24 @@ - "name" - "tags" include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Chain" + x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csc-mrr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-vxlan-mrr" data: "plot-vpp-nfv-2n-skx-mrr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and - 'DOCKER' and + 'VHOST' and + 'VXLAN' and 'CHAIN' and - 'NF_VPPIP4' and 'IMIX' and '8T4C' and 'MRR'" @@ -8465,104 +8995,104 @@ - "name" - "tags" include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Chain" + x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-mrr" layout: "plot-service-density" -# Plots - NF Density - CSC - MRR IMIX +# Plots - NF Density - VSC - NDR IMIX - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csc-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-vxlan-ndr" + data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and - 'DOCKER' and + 'VHOST' and + 'VXLAN' and 'CHAIN' and - 'NF_VPPIP4' and 'IMIX' and '2T1C' and - 'MRR'" + 'NDRPDR'" parameters: - - "result" + - "throughput" - "name" - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Chain" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csc-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-vxlan-ndr" + data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and - 'DOCKER' and + 'VHOST' and + 'VXLAN' and 'CHAIN' and - 'NF_VPPIP4' and 'IMIX' and '4T2C' and - 'MRR'" + 'NDRPDR'" parameters: - - "result" + - "throughput" - "name" - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Chain" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csc-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-vxlan-ndr" + data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and - 'DOCKER' and + 'VHOST' and + 'VXLAN' and 'CHAIN' and - 'NF_VPPIP4' and 'IMIX' and '8T4C' and - 'MRR'" + 'NDRPDR'" parameters: - - "result" + - "throughput" - "name" - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Chain" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-ndr" layout: "plot-service-density" -# Plots - NF Density - CSC - NDR 64b +# Plots - NF Density - VSC - PDR IMIX - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csc-ndr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-vsc-vxlan-pdr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and - 'DOCKER' and + 'VHOST' and + 'VXLAN' and 'CHAIN' and - 'NF_VPPIP4' and 'IMIX' and '2T1C' and 'NDRPDR'" @@ -8570,25 +9100,25 @@ - "throughput" - "name" - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Chain" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csc-ndr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-vsc-vxlan-pdr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and - 'DOCKER' and + 'VHOST' and + 'VXLAN' and 'CHAIN' and - 'NF_VPPIP4' and 'IMIX' and '4T2C' and 'NDRPDR'" @@ -8596,25 +9126,25 @@ - "throughput" - "name" - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Chain" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csc-ndr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-vsc-vxlan-pdr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and - 'DOCKER' and + 'VHOST' and + 'VXLAN' and 'CHAIN' and - 'NF_VPPIP4' and 'IMIX' and '8T4C' and 'NDRPDR'" @@ -8622,21 +9152,23 @@ - "throughput" - "name" - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Chain" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + x-axis: "X = VNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}vh-{XY}vm-vxlan-vppip4-pdr" layout: "plot-service-density" -# Plots - NF Density - CSC - NDR IMIX +############################################################################### + +# Plots - NF Density - CSC - MRR IMIX - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csc-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csc-mrr" + data: "plot-vpp-nfv-2n-skx-mrr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'DOCKER' and @@ -8644,25 +9176,25 @@ 'NF_VPPIP4' and 'IMIX' and '2T1C' and - 'NDRPDR'" + 'MRR'" parameters: - - "throughput" + - "result" - "name" - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" x-axis: "X = CNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csc-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csc-mrr" + data: "plot-vpp-nfv-2n-skx-mrr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'DOCKER' and @@ -8670,25 +9202,25 @@ 'NF_VPPIP4' and 'IMIX' and '4T2C' and - 'NDRPDR'" + 'MRR'" parameters: - - "throughput" + - "result" - "name" - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" x-axis: "X = CNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csc-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csc-mrr" + data: "plot-vpp-nfv-2n-skx-mrr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and 'DOCKER' and @@ -8696,25 +9228,25 @@ 'NF_VPPIP4' and 'IMIX' and '8T4C' and - 'NDRPDR'" + 'MRR'" parameters: - - "throughput" + - "result" - "name" - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" x-axis: "X = CNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-mrr" layout: "plot-service-density" -# Plots - NF Density - CSC - PDR 64b +# Plots - NF Density - CSC - NDR IMIX - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-pdr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csc-pdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csc-ndr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and @@ -8728,19 +9260,19 @@ - "throughput" - "name" - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" x-axis: "X = CNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-pdr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-pdr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csc-pdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csc-ndr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and @@ -8754,19 +9286,19 @@ - "throughput" - "name" - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" x-axis: "X = CNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-pdr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" layout: "plot-service-density" - type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-pdr" + title: "Packet Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" algorithm: "plot_nf_heatmap" output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csc-pdr" + output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csc-ndr" data: "plot-vpp-nfv-2n-skx-ndrpdr" filter: "'NIC_Intel-XXV710' and 'NF_DENSITY' and @@ -8780,12 +9312,12 @@ - "throughput" - "name" - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" x-axis: "X = CNFs per Service Chain" y-axis: "Y = Number of Service Chains" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" + z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" layout: - title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-pdr" + title: "Throughput: eth-l2bd-{Y}ch-{2XY}mif-{XY}dcr-vppip4-ndr" layout: "plot-service-density" # Plots - NF Density - CSC - PDR IMIX @@ -10354,86 +10886,6 @@ ############################################################################### -# Plots - NF Density - CSP - MRR 64b - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csp-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'DOCKER' and - 'PIPELINE' and - 'NF_VPPIP4' and - 'IMIX' and - '2T1C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Pipeline" - y-axis: "Y = Number of Service Pipelines" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-mrr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csp-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'DOCKER' and - 'PIPELINE' and - 'NF_VPPIP4' and - 'IMIX' and - '4T2C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Pipeline" - y-axis: "Y = Number of Service Pipelines" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-mrr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-mrr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csp-mrr" - data: "plot-vpp-nfv-2n-skx-mrr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'DOCKER' and - 'PIPELINE' and - 'NF_VPPIP4' and - 'IMIX' and - '8T4C' and - 'MRR'" - parameters: - - "result" - - "name" - - "tags" - include-tests: "MRR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Pipeline" - y-axis: "Y = Number of Service Pipelines" - z-axis: "IMIX Packet Throughput (mean MRR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-mrr" - layout: "plot-service-density" - # Plots - NF Density - CSP - MRR IMIX - type: "plot" title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-mrr" @@ -10513,85 +10965,6 @@ title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-mrr" layout: "plot-service-density" -# Plots - NF Density - CSP - NDR 64b -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csp-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'DOCKER' and - 'PIPELINE' and - 'NF_VPPIP4' and - 'IMIX' and - '2T1C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Pipeline" - y-axis: "Y = Number of Service Pipelines" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-ndr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csp-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'DOCKER' and - 'PIPELINE' and - 'NF_VPPIP4' and - 'IMIX' and - '4T2C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Pipeline" - y-axis: "Y = Number of Service Pipelines" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-ndr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-ndr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csp-ndr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'DOCKER' and - 'PIPELINE' and - 'NF_VPPIP4' and - 'IMIX' and - '8T4C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Pipeline" - y-axis: "Y = Number of Service Pipelines" - z-axis: "IMIX Packet Throughput (mean NDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-ndr" - layout: "plot-service-density" - # Plots - NF Density - CSP - NDR IMIX - type: "plot" title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-ndr" @@ -10671,85 +11044,6 @@ title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-ndr" layout: "plot-service-density" -# Plots - NF Density - CSP - PDR 64b -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-2t1c-base-csp-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'DOCKER' and - 'PIPELINE' and - 'NF_VPPIP4' and - 'IMIX' and - '2T1C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Pipeline" - y-axis: "Y = Number of Service Pipelines" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-pdr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-4t2c-base-csp-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'DOCKER' and - 'PIPELINE' and - 'NF_VPPIP4' and - 'IMIX' and - '4T2C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Pipeline" - y-axis: "Y = Number of Service Pipelines" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-pdr" - layout: "plot-service-density" - -- type: "plot" - title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-pdr" - algorithm: "plot_nf_heatmap" - output-file-type: ".html" - output-file: "{DIR[STATIC,VPP]}/l2bd-2n-skx-xxv710-imix-8t4c-base-csp-pdr" - data: "plot-vpp-nfv-2n-skx-ndrpdr" - filter: "'NIC_Intel-XXV710' and - 'NF_DENSITY' and - 'DOCKER' and - 'PIPELINE' and - 'NF_VPPIP4' and - 'IMIX' and - '8T4C' and - 'NDRPDR'" - parameters: - - "throughput" - - "name" - - "tags" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - x-axis: "X = CNFs per Service Pipeline" - y-axis: "Y = Number of Service Pipelines" - z-axis: "IMIX Packet Throughput (mean PDR) [Mpps]" - layout: - title: "Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-pdr" - layout: "plot-service-density" - # Plots - NF Density - CSP - PDR IMIX - type: "plot" title: "Packet Throughput: eth-l2bd-{Y}pl-{2Y}mif-{XY}dcr-vppip4-pdr"