X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fspecification.yaml;h=2103534754c0626ef5cc0661ae5034b1abafe39e;hb=921a1c6556b0d8a20a8e7ba2a48b887b41a30e60;hp=1772b7cf3d93d257440da7581799c657df4ebed4;hpb=c9f903e4f7894cb61cf3cc67fb37c7602c917567;p=csit.git diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 1772b7cf3d..2103534754 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -152,16 +152,26 @@ plot-vpp-throughput-lat-tsa-3n-hsw: csit-vpp-perf-report-iterative-2005-3n-hsw: - - 3 # NDRPDR sel - - 4 # NDRPDR sel - - 5 # NDRPDR sel - - 10 # NDRPDR sel - - 11 # NDRPDR sel - - 12 # NDRPDR sel - - 15 # NDRPDR sel - - 16 # NDRPDR sel - - 18 # NDRPDR sel - - 19 # NDRPDR sel +# - 3 # NDRPDR sel +# - 4 # NDRPDR sel +# - 5 # NDRPDR sel +# - 10 # NDRPDR sel +# - 11 # NDRPDR sel +# - 12 # NDRPDR sel +# - 15 # NDRPDR sel +# - 16 # NDRPDR sel +# - 18 # NDRPDR sel +# - 19 # NDRPDR sel + - 20 # NDRPDR sel env 4 + - 21 # NDRPDR sel env 4 + - 22 # NDRPDR sel env 4 + - 23 # NDRPDR sel env 4 + - 24 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 26 # NDRPDR sel env 4 + - 27 # NDRPDR sel env 4 + - 28 # NDRPDR sel env 4 + - 29 # NDRPDR sel env 4 plot-vpp-hdrh-lat-3n-hsw: csit-vpp-perf-report-iterative-2005-3n-hsw: @@ -218,16 +228,21 @@ plot-vpp-throughput-lat-tsa-3n-skx: csit-vpp-perf-report-iterative-2005-3n-skx: - - 4 # NDRPDR sel - - 5 # NDRPDR sel ??? all x710 failed - - 33 # NDRPDR sel - - 28 # NDRPDR sel - - 34 # NDRPDR sel - - 36 # NDRPDR sel - - 37 # NDRPDR sel - - 38 # NDRPDR sel - - 39 # NDRPDR sel - - 41 # NDRPDR sel +# - 4 # NDRPDR sel +# - 5 # NDRPDR sel ??? all x710 failed +# - 33 # NDRPDR sel +# - 28 # NDRPDR sel +# - 34 # NDRPDR sel +# - 36 # NDRPDR sel +# - 37 # NDRPDR sel +# - 38 # NDRPDR sel +# - 39 # NDRPDR sel +# - 41 # NDRPDR sel + - 42 # NDRPDR sel env 4 + - 43 # NDRPDR sel env 4 + - 44 # NDRPDR sel env 4 + - 45 # NDRPDR sel env 4 + - 47 # NDRPDR sel env 4 plot-vpp-hdrh-lat-3n-skx: csit-vpp-perf-report-iterative-2005-3n-skx: @@ -267,16 +282,23 @@ plot-vpp-throughput-lat-tsa-2n-skx: csit-vpp-perf-report-iterative-2005-2n-skx: - - 43 # NDRPDR sel - - 38 # NDRPDR sel - - 7 # NDRPDR sel - - 65 # NDRPDR sel - - 47 # NDRPDR sel - - 55 # NDRPDR sel - - 67 # NDRPDR sel - - 75 # NDRPDR sel - - 76 # NDRPDR sel - - 73 # NDRPDR sel +# - 43 # NDRPDR sel +# - 38 # NDRPDR sel +# - 7 # NDRPDR sel +# - 65 # NDRPDR sel +# - 47 # NDRPDR sel +# - 55 # NDRPDR sel +# - 67 # NDRPDR sel +# - 75 # NDRPDR sel +# - 76 # NDRPDR sel +# - 73 # NDRPDR sel + - 88 # NDRPDR sel env 4 + - 89 # NDRPDR sel env 4 + - 90 # NDRPDR sel env 4 + - 91 # NDRPDR sel env 4 + - 92 # NDRPDR sel env 4 + - 93 # NDRPDR sel env 4 + - 94 # NDRPDR sel env 4 plot-vpp-hdrh-lat-2n-skx: csit-vpp-perf-report-iterative-2005-2n-skx: @@ -334,16 +356,24 @@ plot-vpp-throughput-lat-tsa-2n-clx: csit-vpp-perf-report-iterative-2005-2n-clx: - - 7 # NDRPDR sel - - 38 # NDRPDR sel - - 9 # NDRPDR sel - - 32 # NDRPDR sel ??? - - 33 # NDRPDR sel ??? - - 39 # NDRPDR sel - - 40 # NDRPDR sel - - 43 # NDRPDR sel - - 50 # NDRPDR sel - - 51 # NDRPDR sel +# - 7 # NDRPDR sel +# - 38 # NDRPDR sel +# - 9 # NDRPDR sel +# - 32 # NDRPDR sel ??? +# - 33 # NDRPDR sel ??? +# - 39 # NDRPDR sel +# - 40 # NDRPDR sel +# - 43 # NDRPDR sel +# - 50 # NDRPDR sel +# - 51 # NDRPDR sel + - 62 # NDRPDR sel env 4 + - 63 # NDRPDR sel env 4 + - 64 # NDRPDR sel env 4 + - 65 # NDRPDR sel env 4 + - 66 # NDRPDR sel env 4 + - 68 # NDRPDR sel env 4 + - 69 # NDRPDR sel env 4 + - 70 # NDRPDR sel env 4 plot-vpp-hdrh-lat-2n-clx: csit-vpp-perf-report-iterative-2005-2n-clx: @@ -411,12 +441,14 @@ plot-vpp-throughput-lat-tsa-3n-tsh: # TODO: Add data: csit-vpp-perf-report-iterative-2005-3n-tsh: - - 9 # NDRPDR sel - - 11 # NDRPDR sel - - 12 # NDRPDR sel - - 20 # NDRPDR sel - - 14 # NDRPDR sel - - 21 # NDRPDR sel +# - 9 # NDRPDR sel +# - 11 # NDRPDR sel +# - 12 # NDRPDR sel +# - 20 # NDRPDR sel +# - 14 # NDRPDR sel +# - 21 # NDRPDR sel + - 23 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 plot-vpp-hdrh-lat-3n-tsh: csit-vpp-perf-report-iterative-2005-3n-tsh: @@ -775,6 +807,15 @@ - 27 # NDRPDR sel env 4 - 28 # NDRPDR sel env 4 - 29 # NDRPDR sel env 4 + vpp-performance-changes-3n-hsw-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-hsw: + - 31 # NDRPDR sel 2001.0-4 + - 32 # NDRPDR sel 2001.0-4 + - 33 # NDRPDR sel 2001.0-4 + - 34 # NDRPDR sel 2001.0-4 + - 35 # NDRPDR sel 2001.0-4 + - 36 # NDRPDR sel 2001.0-4 + - 38 # NDRPDR sel 2001.0-4 vpp-performance-changes-3n-skx-h1: csit-vpp-perf-verify-1908_1-3n-skx: @@ -813,7 +854,15 @@ csit-vpp-perf-report-iterative-2005-3n-skx: - 42 # NDRPDR sel env 4 - 43 # NDRPDR sel env 4 + - 44 # NDRPDR sel env 4 - 45 # NDRPDR sel env 4 + - 47 # NDRPDR sel env 4 + vpp-performance-changes-3n-skx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-skx: + - 50 # NDRPDR sel 2001.0-4 + - 51 # NDRPDR sel 2001.0-4 + - 52 # NDRPDR sel 2001.0-4 + - 53 # NDRPDR sel 2001.0-4 vpp-performance-changes-2n-skx-h1: csit-vpp-perf-verify-1908_1-2n-skx: @@ -856,8 +905,18 @@ - 88 # NDRPDR sel env 4 - 89 # NDRPDR sel env 4 - 90 # NDRPDR sel env 4 + - 91 # NDRPDR sel env 4 - 92 # NDRPDR sel env 4 - 93 # NDRPDR sel env 4 + - 94 # NDRPDR sel env 4 + vpp-performance-changes-2n-skx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-2n-skx: + - 100 # NDRPDR sel 2001.0-4 + - 101 # NDRPDR sel 2001.0-4 + - 102 # NDRPDR sel 2001.0-4 + - 103 # NDRPDR sel 2001.0-4 + - 104 # NDRPDR sel 2001.0-4 + - 105 # NDRPDR sel 2001.0-4 # vpp-performance-changes-nfv-2n-skx-ref: # csit-vpp-perf-verify-2001-2n-skx: @@ -902,7 +961,19 @@ - 64 # NDRPDR sel env 4 - 65 # NDRPDR sel env 4 - 66 # NDRPDR sel env 4 + - 68 # NDRPDR sel env 4 + - 69 # NDRPDR sel env 4 - 70 # NDRPDR sel env 4 + vpp-performance-changes-2n-clx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-2n-clx: + - 73 # NDRPDR sel 2001.0-4 + - 74 # NDRPDR sel 2001.0-4 + - 75 # NDRPDR sel 2001.0-4 + - 76 # NDRPDR sel 2001.0-4 + - 77 # NDRPDR sel 2001.0-4 + - 78 # NDRPDR sel 2001.0-4 + - 79 # NDRPDR sel 2001.0-4 + - 80 # NDRPDR sel 2001.0-4 vpp-performance-changes-2n-dnv-h1: csit-vpp-perf-verify-1908-2n-dnv: @@ -1014,6 +1085,11 @@ vpp-performance-changes-3n-tsh-cmp-2: csit-vpp-perf-report-iterative-2005-3n-tsh: - 23 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + vpp-performance-changes-3n-tsh-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-tsh: + - 28 # NDRPDR sel 2001.0-4 + - 29 # NDRPDR sel 2001.0-4 vpp-soak-vs-ndr-2n-skx-ref: csit-vpp-perf-report-iterative-2005-2n-skx: @@ -1086,6 +1162,9 @@ vpp-performance-changes-3n-hsw-mrr-cmp: csit-vpp-perf-report-iterative-2005-3n-hsw: - 13 # MRR + vpp-performance-changes-3n-hsw-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-hsw: + - 30 # MRR env 4 # vpp-performance-changes-3n-skx-mrr-h1: # csit-vpp-perf-verify-1908-3n-skx: @@ -1097,6 +1176,9 @@ vpp-performance-changes-3n-skx-mrr-cmp: csit-vpp-perf-report-iterative-2005-3n-skx: - 6 # MRR sel + vpp-performance-changes-3n-skx-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-skx: + - 49 # MRR sel env 4 # vpp-performance-changes-2n-skx-mrr-h1: # csit-vpp-perf-verify-1908-2n-skx: @@ -1108,6 +1190,9 @@ vpp-performance-changes-2n-skx-mrr-cmp: csit-vpp-perf-report-iterative-2005-2n-skx: - 42 # MRR sel + vpp-performance-changes-2n-skx-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-skx: + - 99 # MRR sel env 4 vpp-performance-changes-2n-clx-mrr-ref: csit-vpp-perf-verify-2001-2n-clx: @@ -1115,6 +1200,9 @@ vpp-performance-changes-2n-clx-mrr-cmp: csit-vpp-perf-report-iterative-2005-2n-clx: - 12 # MRR + vpp-performance-changes-2n-clx-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-clx: + - 72 # MRR # vpp-performance-changes-nfv-2n-skx-mrr-ref: # csit-vpp-perf-verify-1908-2n-skx: @@ -1153,6 +1241,9 @@ vpp-performance-changes-3n-tsh-mrr-cmp: csit-vpp-perf-report-iterative-2005-3n-tsh: - 10 # MRR sel + vpp-performance-changes-3n-tsh-mrr-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-tsh: + - 27 # MRR sel env 4 dpdk-performance-changes-3n-hsw-h1: csit-dpdk-perf-verify-1908-3n-hsw: @@ -2187,6 +2278,14 @@ - 28 # NDRPDR sel env 4 - 29 # NDRPDR sel env 4 - 13 # MRR + - 30 # MRR env 4 + - 31 # NDRPDR sel 2001.0-4 + - 32 # NDRPDR sel 2001.0-4 + - 33 # NDRPDR sel 2001.0-4 + - 34 # NDRPDR sel 2001.0-4 + - 35 # NDRPDR sel 2001.0-4 + - 36 # NDRPDR sel 2001.0-4 + - 38 # NDRPDR sel 2001.0-4 csit-vpp-perf-verify-2005-3n-hsw: - 3 # hoststack - 5 # hoststack @@ -2270,8 +2369,15 @@ - 41 # NDRPDR sel - 42 # NDRPDR sel env 4 - 43 # NDRPDR sel env 4 + - 44 # NDRPDR sel env 4 - 45 # NDRPDR sel env 4 + - 47 # NDRPDR sel env 4 + - 50 # NDRPDR sel 2001.0-4 + - 51 # NDRPDR sel 2001.0-4 + - 52 # NDRPDR sel 2001.0-4 + - 53 # NDRPDR sel 2001.0-4 - 6 # MRR sel + - 49 # MRR sel env 4 csit-vpp-perf-report-coverage-2005-3n-skx: - 10 # perf-full-3n-skx-crypto_I - 11 # perf-full-3n-skx-crypto_II @@ -2343,9 +2449,18 @@ - 88 # NDRPDR sel env 4 - 89 # NDRPDR sel env 4 - 90 # NDRPDR sel env 4 + - 91 # NDRPDR sel env 4 - 92 # NDRPDR sel env 4 - 93 # NDRPDR sel env 4 + - 94 # NDRPDR sel env 4 + - 100 # NDRPDR sel 2001.0-4 + - 101 # NDRPDR sel 2001.0-4 + - 102 # NDRPDR sel 2001.0-4 + - 103 # NDRPDR sel 2001.0-4 + - 104 # NDRPDR sel 2001.0-4 + - 105 # NDRPDR sel 2001.0-4 - 42 # MRR sel + - 99 # MRR sel env 4 - 48 # Reconf - 50 # Reconf - 51 # Reconf @@ -2411,8 +2526,19 @@ - 64 # NDRPDR sel env 4 - 65 # NDRPDR sel env 4 - 66 # NDRPDR sel env 4 + - 68 # NDRPDR sel env 4 + - 69 # NDRPDR sel env 4 - 70 # NDRPDR sel env 4 + - 73 # NDRPDR sel 2001.0-4 + - 74 # NDRPDR sel 2001.0-4 + - 75 # NDRPDR sel 2001.0-4 + - 76 # NDRPDR sel 2001.0-4 + - 77 # NDRPDR sel 2001.0-4 + - 78 # NDRPDR sel 2001.0-4 + - 79 # NDRPDR sel 2001.0-4 + - 80 # NDRPDR sel 2001.0-4 - 12 # MRR + - 72 # MRR env 4 - 42 # Reconf - 45 # Reconf - 46 # Reconf @@ -2480,7 +2606,11 @@ - 14 # NDRPDR sel - 21 # NDRPDR sel - 23 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 + - 28 # NDRPDR sel 2001.0-4 + - 29 # NDRPDR sel 2001.0-4 - 10 # MRR sel + - 27 # MRR sel env 4 csit-vpp-perf-verify-1908-3n-dnv: - 6 # NDRPDR sel @@ -2872,7 +3002,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -2895,7 +3025,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -2918,7 +3048,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -2941,7 +3071,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -2964,7 +3094,7 @@ # reference: 1 # compare: 2 # legend: -# - "Legend to Table:" +# - "" # - "x710, xxv710 - NICs tested." # - "Diff - difference between x710 and xxv710 NIC." # @@ -2987,7 +3117,7 @@ # reference: 1 # compare: 2 # legend: -# - "Legend to Table:" +# - "" # - "x710, xxv710 - NICs tested." # - "Diff - difference between x710 and xxv710 NIC." # @@ -3010,7 +3140,7 @@ # reference: 1 # compare: 2 # legend: -# - "Legend to Table:" +# - "" # - "x710, xxv710 - NICs tested." # - "Diff - difference between x710 and xxv710 NIC." @@ -3033,7 +3163,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3056,7 +3186,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3079,7 +3209,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3102,7 +3232,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3125,7 +3255,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3148,7 +3278,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3171,7 +3301,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table:" + - "" - "x710, xxv710 - NICs tested." - "Diff - difference between x710 and xxv710 NIC." @@ -3190,16 +3320,16 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 4 # rca: # - title: "RCA1" @@ -3207,14 +3337,14 @@ # - title: "RCA2" # data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca2.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - 'Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-hsw 2t2c pdr - type: "table" @@ -3228,31 +3358,31 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 4 # rca: -# - title: "RCA1" -# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml" # - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml" +# - title: "RCA3" # data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca2.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - 'Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-hsw 1t1c ndr - type: "table" @@ -3266,31 +3396,31 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 4 # rca: # - title: "RCA1" -# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca1.yaml" -# - title: "RCA2" # data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca2.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - 'Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-hsw 2t2c ndr - type: "table" @@ -3304,16 +3434,16 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 4 # rca: # - title: "RCA1" @@ -3321,132 +3451,320 @@ # - title: "RCA2" # data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca2.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 3n-skx 2t1c pdr + - 'Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 1t1c pdr-full - type: "table" - title: "VPP Performance Changes 3n-skx 2t1c pdr" + title: "VPP Performance Changes 3n-hsw 1t1c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" - filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - title: "1908.1-2" - data-set: "vpp-performance-changes-3n-skx-h1" + data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" - data-set: "vpp-performance-changes-3n-skx-ref" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-skx-cmp" + data-set: "vpp-performance-changes-3n-hsw-cmp" - title: "2005.0-4" - data-set: "vpp-performance-changes-3n-skx-cmp-2" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" -# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca1.yaml" # - title: "RCA2" -# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 3n-skx 4t2c pdr + - 'Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 2t2c pdr-full - type: "table" - title: "VPP Performance Changes 3n-skx 4t2c pdr" + title: "VPP Performance Changes 3n-hsw 2t2c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" - filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - title: "1908.1-2" - data-set: "vpp-performance-changes-3n-skx-h1" + data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" - data-set: "vpp-performance-changes-3n-skx-ref" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-skx-cmp" + data-set: "vpp-performance-changes-3n-hsw-cmp" - title: "2005.0-4" - data-set: "vpp-performance-changes-3n-skx-cmp-2" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" -# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml" # - title: "RCA2" -# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 3n-skx 2t1c ndr + - 'Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 1t1c ndr-full - type: "table" - title: "VPP Performance Changes 3n-skx 2t1c ndr" + title: "VPP Performance Changes 3n-hsw 1t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" - filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - title: "1908.1-2" - data-set: "vpp-performance-changes-3n-skx-h1" + data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" - data-set: "vpp-performance-changes-3n-skx-ref" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-skx-cmp" + data-set: "vpp-performance-changes-3n-hsw-cmp" - title: "2005.0-4" - data-set: "vpp-performance-changes-3n-skx-cmp-2" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" -# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca1.yaml" # - title: "RCA2" -# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 3n-skx 4t2c ndr + - 'Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 2t2c ndr-full - type: "table" - title: "VPP Performance Changes 3n-skx 4t2c ndr" + title: "VPP Performance Changes 3n-hsw 2t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c pdr +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c pdr +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c ndr +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c ndr +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c ndr" algorithm: "table_comparison" output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr" include-tests: "NDR" # "PDR" | "NDR" | "MRR" @@ -3456,31 +3774,219 @@ data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-skx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c ndr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - 'Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 2t1c pdr - type: "table" @@ -3494,16 +4000,16 @@ data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 4 # rca: # - title: "RCA1" @@ -3511,14 +4017,14 @@ # - title: "RCA2" # data: "rca/rca-2005-2n-skx-2t1c-pdr-rca2.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - 'Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 4t2c pdr - type: "table" @@ -3531,252 +4037,620 @@ - title: "1908.1-2" data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" - data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-skx-cmp" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c ndr +- type: "table" + title: "VPP Performance Changes 2n-skx 2t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 4t2c ndr +- type: "table" + title: "VPP Performance Changes 2n-skx 4t2c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 2t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 4t2c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 4t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c ndr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 2t1c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 4t2c ndr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 4t2c ndr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr-full" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca3.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c pdr +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 1 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c pdr +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-4" - data-set: "vpp-performance-changes-2n-skx-cmp-2" + data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" + reference: 1 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" reference: 2 - compare: 4 + compare: 3 # rca: # - title: "RCA1" -# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml" # - title: "RCA2" -# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 2n-skx 2t1c ndr +# VPP Performance Changes 2n-clx-xxv710 2t1c ndr - type: "table" - title: "VPP Performance Changes 2n-skx 2t1c ndr" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908.1-2" - data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" - data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-skx-cmp" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-4" - data-set: "vpp-performance-changes-2n-skx-cmp-2" + data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" + reference: 1 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" reference: 2 - compare: 4 + compare: 3 # rca: # - title: "RCA1" -# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml" # - title: "RCA2" -# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 2n-skx 4t2c ndr +# VPP Performance Changes 2n-clx-xxv710 4t2c ndr - type: "table" - title: "VPP Performance Changes 2n-skx 4t2c ndr" + title: "VPP Performance Changes 2n-clx-xxv710 ndr" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908.1-2" - data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" - data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-skx-cmp" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-4" - data-set: "vpp-performance-changes-2n-skx-cmp-2" + data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" + reference: 1 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" reference: 2 - compare: 4 + compare: 3 # rca: # - title: "RCA1" -# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml" # - title: "RCA2" -# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 2n-clx-xxv710 2t1c pdr + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 2t1c pdr" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 2 - compare: 3 - - title: "2005.0-4 vs 2001.0-2" + reference: 3 + compare: 4 + - title: "2001.0-4 vs 2001.0-2" reference: 1 - compare: 3 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 2n-clx-xxv710 4t2c pdr + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr" + title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 2 - compare: 3 - - title: "2005.0-4 vs 2001.0-2" + reference: 3 + compare: 4 + - title: "2001.0-4 vs 2001.0-2" reference: 1 - compare: 3 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 2n-clx-xxv710 2t1c ndr + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 2 - compare: 3 - - title: "2005.0-4 vs 2001.0-2" + reference: 3 + compare: 4 + - title: "2001.0-4 vs 2001.0-2" reference: 1 - compare: 3 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 2n-clx-xxv710 4t2c ndr + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c ndr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 ndr" + title: "VPP Performance Changes 2n-clx-xxv710 ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 2 - compare: 3 - - title: "2005.0-4 vs 2001.0-2" + reference: 3 + compare: 4 + - title: "2001.0-4 vs 2001.0-2" reference: 1 - compare: 3 + compare: 2 + - title: "2005.0-4 vs 2001.0-4" + reference: 2 + compare: 4 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-x556 2t1c pdr - type: "table" @@ -3795,9 +4669,12 @@ reference: 1 compare: 2 legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with x556.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-x556 4t2c pdr - type: "table" @@ -3816,9 +4693,12 @@ reference: 1 compare: 2 legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." + - '' + - 'Test Case: Test cases executed for: PDR throughput on 2n-clx with x556.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-x556 2t1c ndr - type: "table" @@ -3837,9 +4717,12 @@ reference: 1 compare: 2 legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with x556.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-x556 4t2c ndr - type: "table" @@ -3858,9 +4741,12 @@ reference: 1 compare: 2 legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." + - '' + - 'Test Case: Test cases executed for: NDR throughput on 2n-clx with x556.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-dnv 1t1c pdr - type: "table" @@ -3881,7 +4767,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -3904,7 +4790,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -3927,7 +4813,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -3950,7 +4836,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -3973,7 +4859,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -3992,65 +4878,217 @@ - title: "2005" data-set: "vpp-performance-changes-3n-dnv-cmp" comparisons: - - title: "Diff" + - title: "Diff" + reference: 2 + compare: 3 + legend: + - "" + - "1908, 2001, 2005 - VPP versions tested." + - "Diff - difference between 2001 and 2005 VPP version." + +# VPP Performance Changes 3n-dnv 1t1c ndr +- type: "table" + title: "VPP Performance Changes 3n-dnv 1t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-dnv-1t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C'" + columns: + - title: "1908" + data-set: "vpp-performance-changes-3n-dnv-h1" + - title: "2001" + data-set: "vpp-performance-changes-3n-dnv-ref" + - title: "2005" + data-set: "vpp-performance-changes-3n-dnv-cmp" + comparisons: + - title: "Diff" + reference: 2 + compare: 3 + legend: + - "" + - "1908, 2001, 2005 - VPP versions tested." + - "Diff - difference between 2001 and 2005 VPP version." + +# VPP Performance Changes 3n-dnv 2t2c ndr +- type: "table" + title: "VPP Performance Changes 3n-dnv 2t2c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-dnv-2t2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C'" + columns: + - title: "1908" + data-set: "vpp-performance-changes-3n-dnv-h1" + - title: "2001" + data-set: "vpp-performance-changes-3n-dnv-ref" + - title: "2005" + data-set: "vpp-performance-changes-3n-dnv-cmp" + comparisons: + - title: "Diff" + reference: 2 + compare: 3 + legend: + - "" + - "1908, 2001, 2005 - VPP versions tested." + - "Diff - difference between 2001 and 2005 VPP version." + +# VPP Performance Changes 3n-tsh 1t1c pdr +- type: "table" + title: "VPP Performance Changes 3n-tsh 1t1c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml" + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c pdr +- type: "table" + title: "VPP Performance Changes 3n-tsh 2t2c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 3n-dnv 1t1c ndr +# VPP Performance Changes 3n-tsh 1t1c ndr - type: "table" - title: "VPP Performance Changes 3n-dnv 1t1c ndr" + title: "VPP Performance Changes 3n-tsh 1t1c ndr" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-dnv-1t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-dnv-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-dnv-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-dnv-cmp" + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 3n-dnv 2t2c ndr +# VPP Performance Changes 3n-tsh 2t2c ndr - type: "table" - title: "VPP Performance Changes 3n-dnv 2t2c ndr" + title: "VPP Performance Changes 3n-tsh 2t2c ndr" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-dnv-2t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-dnv-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-dnv-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-dnv-cmp" + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "Diff" + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 3n-tsh 1t1c pdr + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA2: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 1t1c pdr-full - type: "table" - title: "VPP Performance Changes 3n-tsh 1t1c pdr" + title: "VPP Performance Changes 3n-tsh 1t1c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: @@ -4058,37 +5096,46 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 3n-tsh 2t2c pdr + - 'Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c pdr-full - type: "table" - title: "VPP Performance Changes 3n-tsh 2t2c pdr" + title: "VPP Performance Changes 3n-tsh 2t2c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: @@ -4096,37 +5143,46 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 3n-tsh 1t1c ndr + - 'Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 1t1c ndr-full - type: "table" - title: "VPP Performance Changes 3n-tsh 1t1c ndr" + title: "VPP Performance Changes 3n-tsh 1t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: @@ -4134,37 +5190,46 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' - -# VPP Performance Changes 3n-tsh 2t2c ndr + - 'Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c ndr-full - type: "table" - title: "VPP Performance Changes 3n-tsh 2t2c ndr" + title: "VPP Performance Changes 3n-tsh 2t2c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: @@ -4172,31 +5237,40 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca3.yaml" legend: - - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' - - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - 'Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - 'YYMM.V-E: [Mpps +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2005.0-3: Compares CSIT environments "4" to "3". "3" was discarded as it substantially changed TRex behaviour due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: Compares CSIT environment current vs previous with previous VPP code.' + - '2005.0-4 vs 2001.0-4: Compares VPP code current vs previous in the current CSIT environment.' +# - 'RCA1: Root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - 'RCA2: Root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - 'RCA3: Root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr - type: "table" @@ -4217,7 +5291,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -4240,7 +5314,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -4263,7 +5337,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -4286,7 +5360,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -4307,7 +5381,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -4328,7 +5402,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -4349,7 +5423,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -4370,7 +5444,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -4393,7 +5467,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -4416,7 +5490,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -4439,7 +5513,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -4462,7 +5536,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -4483,7 +5557,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -4504,7 +5578,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -4525,7 +5599,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -4546,7 +5620,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -4558,20 +5632,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'1T1C' and 'MRR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-hsw 2t2c mrr - type: "table" @@ -4581,20 +5669,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T2C' and 'MRR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-2t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-hsw 4t4c mrr - type: "table" @@ -4604,20 +5706,35 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T4C' and 'MRR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-hsw-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-hsw-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-4t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-4t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + # VPP Performance Changes 3n-skx 2t1c mrr - type: "table" @@ -4629,18 +5746,32 @@ columns: # - title: "1908" # data-set: "vpp-performance-changes-3n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-skx 4t2c mrr - type: "table" @@ -4652,18 +5783,32 @@ columns: # - title: "1908" # data-set: "vpp-performance-changes-3n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-skx 8t4c mrr - type: "table" @@ -4675,18 +5820,32 @@ columns: # - title: "1908" # data-set: "vpp-performance-changes-3n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-8t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-8t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-skx 2t1c mrr - type: "table" @@ -4696,20 +5855,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: -# - title: "1908" -# data-set: "vpp-performance-changes-2n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-skx 4t2c mrr - type: "table" @@ -4719,20 +5890,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: -# - title: "1908" -# data-set: "vpp-performance-changes-2n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-skx 8t4c mrr - type: "table" @@ -4742,20 +5925,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'8T4C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: -# - title: "1908" -# data-set: "vpp-performance-changes-2n-skx-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-8t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-8t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-xxv710 2t1c mrr - type: "table" @@ -4765,18 +5960,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-xxv710 4t2c mrr - type: "table" @@ -4786,18 +5995,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-xxv710 8t4c mrr - type: "table" @@ -4807,18 +6030,32 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'8T4C' and 'MRR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-2" reference: 1 - compare: 2 + compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-8t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-8t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 2n-clx-clx-x556 2t1c mrr - type: "table" @@ -4837,7 +6074,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4858,7 +6095,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4879,7 +6116,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4902,7 +6139,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4925,7 +6162,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4948,7 +6185,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4971,7 +6208,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -4994,7 +6231,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -5017,7 +6254,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." @@ -5029,20 +6266,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-tsh 2t2c mrr - type: "table" @@ -5052,20 +6303,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Performance Changes 3n-tsh 4t4c mrr - type: "table" @@ -5075,20 +6340,34 @@ include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'4T4C'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-3n-tsh-mrr-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-mrr-ref" - - title: "2005" + - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-mrr-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-mrr-cmp-2" comparisons: - - title: "Diff" + - title: "2005.0-4 vs 2005.0-3" + reference: 3 + compare: 4 + - title: "2005.0-4 vs 2001.0-2" reference: 2 - compare: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-4t4c-mrr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-4t4c-mrr-rca2.yaml" legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c mrr - type: "table" @@ -5109,7 +6388,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -5132,7 +6411,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -5155,7 +6434,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 3n-skx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 3n-skx xxv710 testbeds." @@ -5176,7 +6455,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -5197,7 +6476,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -5218,7 +6497,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -5239,7 +6518,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -5260,7 +6539,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -5281,7 +6560,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -5304,7 +6583,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -5327,7 +6606,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -5350,7 +6629,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw xl710, 2n-clx xxv710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw xl710 and 2n-clx xxv710 testbeds." @@ -5944,7 +7223,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -5967,7 +7246,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -5990,7 +7269,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6013,7 +7292,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6036,7 +7315,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6059,7 +7338,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6082,7 +7361,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6105,7 +7384,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6128,7 +7407,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6151,7 +7430,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6174,7 +7453,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6197,7 +7476,7 @@ reference: 2 compare: 3 legend: - - "Legend to Table" + - "" - "1908, 2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6218,7 +7497,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6239,7 +7518,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6260,7 +7539,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6281,7 +7560,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6302,7 +7581,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6323,7 +7602,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6344,7 +7623,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6365,7 +7644,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6386,7 +7665,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6407,7 +7686,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6428,7 +7707,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6449,7 +7728,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6470,7 +7749,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6491,7 +7770,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6512,7 +7791,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6533,7 +7812,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2001, 2005 - DPDK versions tested." - "Diff - difference between 2001 and 2005 DPDK version." @@ -6554,7 +7833,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw x710, 3n-skx x710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw x710 and 3n-skx x710 testbeds." @@ -6575,7 +7854,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-hsw x710, 3n-skx x710 - testbeds and NICs tested." - "Diff - difference between 3n-hsw x710 and 3n-skx x710 testbeds." @@ -6596,7 +7875,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -6617,7 +7896,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "2n-skx, 2n-clx - testbeds and NICs tested." - "Diff - difference between 2n-skx and 2n-clx testbeds." @@ -6638,7 +7917,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies." @@ -6659,7 +7938,7 @@ reference: 1 compare: 2 legend: - - "Legend to Table" + - "" - "3n-skx, 2n-skx - topologies tested." - "Diff - difference between 3n-skx and 2n-skx topologies."