X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fspecification.yaml;h=d670800ae5c0173e3aa0f985af0635659f8f5e43;hb=0074e91ecc792db04f82f51dc760ddf29375dc13;hp=cf677648e5f03cb266922a0133a955e9e9564c47;hpb=6d45a74ca33951a02b9a1bcc8b0e2cf2f9db8745;p=csit.git diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index cf677648e5..d670800ae5 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -757,9 +757,12 @@ - 19 # NDRPDR sel vpp-performance-changes-3n-hsw-cmp-2: csit-vpp-perf-report-iterative-2005-3n-hsw: - - 20 # NDRPDR sel env 2 - - 21 # NDRPDR sel env 2 - - 22 # NDRPDR sel env 2 + - 20 # NDRPDR sel env 4 + - 21 # NDRPDR sel env 4 + - 22 # NDRPDR sel env 4 + - 23 # NDRPDR sel env 4 + - 24 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 vpp-performance-changes-3n-skx-h1: csit-vpp-perf-verify-1908_1-3n-skx: @@ -794,6 +797,10 @@ - 38 # NDRPDR sel - 39 # NDRPDR sel - 41 # NDRPDR sel + vpp-performance-changes-3n-skx-cmp-2: + csit-vpp-perf-report-iterative-2005-3n-skx: + # TODO: Add data + - 43 # NDRPDR sel env 4 vpp-performance-changes-2n-skx-h1: csit-vpp-perf-verify-1908_1-2n-skx: @@ -831,6 +838,11 @@ - 75 # NDRPDR sel - 76 # NDRPDR sel - 73 # NDRPDR sel + vpp-performance-changes-2n-skx-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-skx: + # TODO: Add data + - 88 # NDRPDR sel env 4 + - 89 # NDRPDR sel env 4 # vpp-performance-changes-nfv-2n-skx-ref: # csit-vpp-perf-verify-2001-2n-skx: @@ -867,6 +879,10 @@ - 43 # NDRPDR sel - 50 # NDRPDR sel - 51 # NDRPDR sel + vpp-performance-changes-2n-clx-cmp-2: + csit-vpp-perf-report-iterative-2005-2n-clx: + - 62 # NDRPDR sel env 4 + - 63 # NDRPDR sel env 4 vpp-performance-changes-2n-dnv-h1: csit-vpp-perf-verify-1908-2n-dnv: @@ -974,6 +990,10 @@ - 12 # NDRPDR sel - 20 # NDRPDR sel - 14 # NDRPDR sel + vpp-performance-changes-3n-tsh-cmp-2: + # TODO: Add data: + csit-vpp-perf-report-iterative-2005-3n-tsh: + - 23 # NDRPDR sel env 4 vpp-soak-vs-ndr-2n-skx-ref: csit-vpp-perf-report-iterative-2005-2n-skx: @@ -2250,9 +2270,12 @@ - 16 # NDRPDR sel - 18 # NDRPDR sel - 19 # NDRPDR sel - - 20 # NDRPDR sel env 2 - - 21 # NDRPDR sel env 2 - - 22 # NDRPDR sel env 2 + - 20 # NDRPDR sel env 4 + - 21 # NDRPDR sel env 4 + - 22 # NDRPDR sel env 4 + - 23 # NDRPDR sel env 4 + - 24 # NDRPDR sel env 4 + - 25 # NDRPDR sel env 4 - 13 # MRR csit-vpp-perf-verify-2005-3n-hsw: - 3 # hoststack @@ -2335,6 +2358,7 @@ - 38 # NDRPDR sel - 39 # NDRPDR sel - 41 # NDRPDR sel + - 43 # NDRPDR sel env 4 - 6 # MRR sel csit-vpp-perf-report-coverage-2005-3n-skx: - 10 # perf-full-3n-skx-crypto_I @@ -2398,6 +2422,8 @@ - 75 # NDRPDR sel - 76 # NDRPDR sel - 73 # NDRPDR sel + - 88 # NDRPDR sel env 4 + - 89 # NDRPDR sel env 4 - 42 # MRR sel - 48 # Reconf - 50 # Reconf @@ -2458,6 +2484,8 @@ - 43 # NDRPDR sel - 50 # NDRPDR sel - 51 # NDRPDR sel + - 62 # NDRPDR sel env 4 + - 63 # NDRPDR sel env 4 - 12 # MRR - 42 # Reconf - 45 # Reconf @@ -2524,6 +2552,7 @@ - 12 # NDRPDR sel - 20 # NDRPDR sel - 14 # NDRPDR sel + - 23 # NDRPDR sel env 4 - 10 # MRR sel csit-vpp-perf-verify-1908-3n-dnv: @@ -3234,9 +3263,9 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-2" - data-set: "vpp-performance-changes-3n-hsw-cmp-2" - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp" comparisons: - title: "Diff1" @@ -3252,9 +3281,9 @@ # data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca2.yaml" legend: - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' # - 'RCA1 - root cause analysis reference for Diff1.' # - 'RCA2 - root cause analysis reference for Diff2.' @@ -3270,9 +3299,9 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-2" - data-set: "vpp-performance-changes-3n-hsw-cmp-2" - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp" comparisons: - title: "Diff1" @@ -3288,9 +3317,9 @@ # data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca2.yaml" legend: - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' # - 'RCA1 - root cause analysis reference for Diff1.' # - 'RCA2 - root cause analysis reference for Diff2.' @@ -3306,9 +3335,9 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-2" - data-set: "vpp-performance-changes-3n-hsw-cmp-2" - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp" comparisons: - title: "Diff1" @@ -3324,9 +3353,9 @@ # data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca2.yaml" legend: - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' # - 'RCA1 - root cause analysis reference for Diff1.' # - 'RCA2 - root cause analysis reference for Diff2.' @@ -3342,9 +3371,9 @@ data-set: "vpp-performance-changes-3n-hsw-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-hsw-ref" - - title: "2005.0-2" - data-set: "vpp-performance-changes-3n-hsw-cmp-2" - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp" comparisons: - title: "Diff1" @@ -3360,255 +3389,155 @@ # data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca2.yaml" legend: - "Legend to Table" - - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-XX-XX).' - - 'Diff1 - difference between environment versions "2" and "3" for the same VPP version, 2005.0-2 vs. 2005.0-3.' - - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2005.0-2 vs. 2001.0-2.' + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' # - 'RCA1 - root cause analysis reference for Diff1.' # - 'RCA2 - root cause analysis reference for Diff2.' -## VPP Root Cause Analysis 3n-skx-xxv710 2t1c pdr +## VPP Performance Changes 3n-skx 2t1c pdr #- type: "table" -# title: "VPP Root Cause Analysis 3n-skx-xxv710 2t1c pdr" +# title: "VPP Performance Changes 3n-skx 2t1c pdr" # algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-rca-3n-skx-2t1c-pdr" +# output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr" # include-tests: "PDR" # "PDR" | "NDR" | "MRR" # filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# rca: -# - title: "RCA1" -# data: "rca/rca-3n-skx-2t1c-pdr-rca1.yaml" -# - title: "RCA2" -# data: "rca/rca-3n-skx-2t1c-pdr-rca2.yaml" # columns: -# - title: "1908.0-1" -# data-set: "vpp-perf-changes-3n-skx-old1908_0" -# data-replacement: "replacement-1908-vpp-throughput-lat-tsa-3n-skx-vhost" -# - title: "1908.1-1" -# data-set: "vpp-perf-changes-3n-skx-old1908_1" # - title: "1908.1-2" -# data-set: "vpp-perf-changes-3n-skx-new1908_1" +# data-set: "vpp-performance-changes-3n-skx-h1" # - title: "2001.0-2" -# data-set: "vpp-perf-changes-3n-skx-new2001_0" +# data-set: "vpp-performance-changes-3n-skx-ref" +# - title: "2005.0-3" +# data-set: "vpp-performance-changes-3n-skx-cmp-2" +# - title: "2005.0-4" +# data-set: "vpp-performance-changes-3n-skx-cmp" # comparisons: # - title: "Diff1" -# reference: 2 -# reference-alt: 1 -# compare: 3 -# - title: "Diff2" # reference: 3 # compare: 4 +# - title: "Diff2" +# reference: 2 +# compare: 3 +## rca: +## - title: "RCA1" +## data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml" +## - title: "RCA2" +## data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml" # legend: -# - 'Legend to Table:' -# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).' -# - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).' -# - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' +# - "Legend to Table" +# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' +# - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' +# - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +## - 'RCA1 - root cause analysis reference for Diff1.' +## - 'RCA2 - root cause analysis reference for Diff2.' # -## VPP Root Cause Analysis 3n-skx-xxv710 2t1c ndr +## VPP Performance Changes 3n-skx 4t2c pdr #- type: "table" -# title: "VPP Root Cause Analysis 3n-skx-xxv710 2t1c ndr" +# title: "VPP Performance Changes 3n-skx 4t2c pdr" # algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-rca-3n-skx-2t1c-ndr" -# include-tests: "NDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# rca: -# - title: "RCA1" -# data: "rca/rca-3n-skx-2t1c-ndr-rca1.yaml" -# - title: "RCA2" -# data: "rca/rca-3n-skx-2t1c-ndr-rca2.yaml" +# output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr" +# include-tests: "PDR" # "PDR" | "NDR" | "MRR" +# filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" # columns: -# - title: "1908.0-1" -# data-set: "vpp-perf-changes-3n-skx-old1908_0" -# data-replacement: "replacement-1908-vpp-throughput-lat-tsa-3n-skx-vhost" -# - title: "1908.1-1" -# data-set: "vpp-perf-changes-3n-skx-old1908_1" # - title: "1908.1-2" -# data-set: "vpp-perf-changes-3n-skx-new1908_1" +# data-set: "vpp-performance-changes-3n-skx-h1" # - title: "2001.0-2" -# data-set: "vpp-perf-changes-3n-skx-new2001_0" +# data-set: "vpp-performance-changes-3n-skx-ref" +# - title: "2005.0-3" +# data-set: "vpp-performance-changes-3n-skx-cmp-2" +# - title: "2005.0-4" +# data-set: "vpp-performance-changes-3n-skx-cmp" # comparisons: # - title: "Diff1" -# reference: 2 -# reference-alt: 1 -# compare: 3 -# - title: "Diff2" # reference: 3 # compare: 4 +# - title: "Diff2" +# reference: 2 +# compare: 3 +## rca: +## - title: "RCA1" +## data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" +## - title: "RCA2" +## data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" # legend: -# - 'Legend to Table:' -# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).' -# - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).' -# - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' - -# VPP Performance Changes 3n-skx 2t1c pdr -- type: "table" - title: "VPP Performance Changes 3n-skx 2t1c pdr" - algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" - columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-skx-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-skx-cmp" - comparisons: - - title: "Diff" - reference: 2 - compare: 3 - legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 3n-skx 4t2c pdr -- type: "table" - title: "VPP Performance Changes 3n-skx 4t2c pdr" - algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" - columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-skx-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-skx-cmp" - comparisons: - - title: "Diff" - reference: 2 - compare: 3 - legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 3n-skx 2t1c ndr -- type: "table" - title: "VPP Performance Changes 3n-skx 2t1c ndr" - algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" - columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-skx-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-skx-cmp" - comparisons: - - title: "Diff" - reference: 2 - compare: 3 - legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 3n-skx 4t2c ndr -- type: "table" - title: "VPP Performance Changes 3n-skx 4t2c ndr" - algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" - columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-skx-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-skx-cmp" - comparisons: - - title: "Diff" - reference: 2 - compare: 3 - legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -## VPP Root Cause Analysis 2n-skx-xxv710 2t1c pdr +# - "Legend to Table" +# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' +# - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' +# - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +## - 'RCA1 - root cause analysis reference for Diff1.' +## - 'RCA2 - root cause analysis reference for Diff2.' +# +## VPP Performance Changes 3n-skx 2t1c ndr #- type: "table" -# title: "VPP Root Cause Analysis 2n-skx-xxv710 2t1c pdr" +# title: "VPP Performance Changes 3n-skx 2t1c ndr" # algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-rca-2n-skx-2t1c-pdr" -# include-tests: "PDR" # "PDR" | "NDR" | "MRR" +# output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr" +# include-tests: "NDR" # "PDR" | "NDR" | "MRR" # filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# rca: -# - title: "RCA1" -# data: "rca/rca-2n-skx-2t1c-pdr-rca1.yaml" -# - title: "RCA2" -# data: "rca/rca-2n-skx-2t1c-pdr-rca2.yaml" # columns: -# - title: "1908.0-1" -# data-set: "vpp-perf-changes-2n-skx-old1908_0" -# data-replacement: "replacement-1908-vpp-throughput-lat-tsa-2n-skx-vhost" -# - title: "1908.1-1" -# data-set: "vpp-perf-changes-2n-skx-old1908_1" # - title: "1908.1-2" -# data-set: "vpp-perf-changes-2n-skx-new1908_1" +# data-set: "vpp-performance-changes-3n-skx-h1" # - title: "2001.0-2" -# data-set: "vpp-perf-changes-2n-skx-new2001_0" +# data-set: "vpp-performance-changes-3n-skx-ref" +# - title: "2005.0-3" +# data-set: "vpp-performance-changes-3n-skx-cmp-2" +# - title: "2005.0-4" +# data-set: "vpp-performance-changes-3n-skx-cmp" # comparisons: # - title: "Diff1" -# reference: 2 -# reference-alt: 1 -# compare: 3 -# - title: "Diff2" # reference: 3 # compare: 4 +# - title: "Diff2" +# reference: 2 +# compare: 3 +## rca: +## - title: "RCA1" +## data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" +## - title: "RCA2" +## data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" # legend: -# - 'Legend to Table:' -# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).' -# - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).' -# - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' +# - "Legend to Table" +# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' +# - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' +# - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +## - 'RCA1 - root cause analysis reference for Diff1.' +## - 'RCA2 - root cause analysis reference for Diff2.' # -## VPP Root Cause Analysis 2n-skx-xxv710 2t1c ndr +## VPP Performance Changes 3n-skx 4t2c ndr #- type: "table" -# title: "VPP Root Cause Analysis 2n-skx-xxv710 2t1c ndr" +# title: "VPP Performance Changes 3n-skx 4t2c ndr" # algorithm: "table_comparison" -# output-file: "{DIR[STATIC,VPP]}/performance-rca-2n-skx-2t1c-ndr" +# output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr" # include-tests: "NDR" # "PDR" | "NDR" | "MRR" -# filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" -# rca: -# - title: "RCA1" -# data: "rca/rca-2n-skx-2t1c-ndr-rca1.yaml" -# - title: "RCA2" -# data: "rca/rca-2n-skx-2t1c-ndr-rca2.yaml" +# filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" # columns: -# - title: "1908.0-1" -# data-set: "vpp-perf-changes-2n-skx-old1908_0" -# data-replacement: "replacement-1908-vpp-throughput-lat-tsa-2n-skx-vhost" -# - title: "1908.1-1" -# data-set: "vpp-perf-changes-2n-skx-old1908_1" # - title: "1908.1-2" -# data-set: "vpp-perf-changes-2n-skx-new1908_1" +# data-set: "vpp-performance-changes-3n-skx-h1" # - title: "2001.0-2" -# data-set: "vpp-perf-changes-2n-skx-new2001_0" +# data-set: "vpp-performance-changes-3n-skx-ref" +# - title: "2005.0-3" +# data-set: "vpp-performance-changes-3n-skx-cmp-2" +# - title: "2005.0-4" +# data-set: "vpp-performance-changes-3n-skx-cmp" # comparisons: # - title: "Diff1" -# reference: 2 -# reference-alt: 1 -# compare: 3 -# - title: "Diff2" # reference: 3 # compare: 4 +# - title: "Diff2" +# reference: 2 +# compare: 3 +## rca: +## - title: "RCA1" +## data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml" +## - title: "RCA2" +## data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml" # legend: -# - 'Legend to Table:' -# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "1" for 2019-08-21, "2" for 2020-03-27).' -# - 'Diff1 - difference between environment versions "2" and "1" for the same VPP version, 1908.1-2 vs. 1908.1-1 (if data for the latter not present, 1908.0-1 is used).' -# - 'Diff2 - difference between the current and the previous VPP version in the current environment version "2", 2001.0-2 vs. 1908.1-2.' -# - 'RCA1 - root cause analysis reference for Diff1.' -# - 'RCA2 - root cause analysis reference for Diff2.' +# - "Legend to Table" +# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' +# - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' +# - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +## - 'RCA1 - root cause analysis reference for Diff1.' +## - 'RCA2 - root cause analysis reference for Diff2.' # VPP Performance Changes 2n-skx 2t1c pdr - type: "table" @@ -3618,20 +3547,33 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-2n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp" comparisons: - - title: "Diff" + - title: "Diff1" + reference: 3 + compare: 4 + - title: "Diff2" reference: 2 compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca2.yaml" legend: - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +# - 'RCA1 - root cause analysis reference for Diff1.' +# - 'RCA2 - root cause analysis reference for Diff2.' # VPP Performance Changes 2n-skx 4t2c pdr - type: "table" @@ -3641,20 +3583,33 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-2n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp" comparisons: - - title: "Diff" + - title: "Diff1" + reference: 3 + compare: 4 + - title: "Diff2" reference: 2 compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" legend: - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +# - 'RCA1 - root cause analysis reference for Diff1.' +# - 'RCA2 - root cause analysis reference for Diff2.' # VPP Performance Changes 2n-skx 2t1c ndr - type: "table" @@ -3664,20 +3619,33 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-2n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp" comparisons: - - title: "Diff" + - title: "Diff1" + reference: 3 + compare: 4 + - title: "Diff2" reference: 2 compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" legend: - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +# - 'RCA1 - root cause analysis reference for Diff1.' +# - 'RCA2 - root cause analysis reference for Diff2.' # VPP Performance Changes 2n-skx 4t2c ndr - type: "table" @@ -3687,20 +3655,33 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "1908" + - title: "1908.1-2" data-set: "vpp-performance-changes-2n-skx-h1" - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp" comparisons: - - title: "Diff" + - title: "Diff1" + reference: 3 + compare: 4 + - title: "Diff2" reference: 2 compare: 3 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" legend: - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +# - 'RCA1 - root cause analysis reference for Diff1.' +# - 'RCA2 - root cause analysis reference for Diff2.' # VPP Performance Changes 2n-clx-xxv710 2t1c pdr - type: "table" @@ -3710,18 +3691,31 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - - title: "Diff" + - title: "Diff1" + reference: 2 + compare: 3 + - title: "Diff2" reference: 1 compare: 2 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml" legend: - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +# - 'RCA1 - root cause analysis reference for Diff1.' +# - 'RCA2 - root cause analysis reference for Diff2.' # VPP Performance Changes 2n-clx-xxv710 4t2c pdr - type: "table" @@ -3731,18 +3725,31 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - - title: "Diff" + - title: "Diff1" + reference: 2 + compare: 3 + - title: "Diff2" reference: 1 compare: 2 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml" legend: - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +# - 'RCA1 - root cause analysis reference for Diff1.' +# - 'RCA2 - root cause analysis reference for Diff2.' # VPP Performance Changes 2n-clx-xxv710 2t1c ndr - type: "table" @@ -3752,18 +3759,31 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - - title: "Diff" + - title: "Diff1" + reference: 2 + compare: 3 + - title: "Diff2" reference: 1 compare: 2 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml" legend: - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +# - 'RCA1 - root cause analysis reference for Diff1.' +# - 'RCA2 - root cause analysis reference for Diff2.' # VPP Performance Changes 2n-clx-xxv710 4t2c ndr - type: "table" @@ -3773,18 +3793,31 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - - title: "Diff" + - title: "Diff1" + reference: 2 + compare: 3 + - title: "Diff2" reference: 1 compare: 2 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml" legend: - "Legend to Table" - - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' + - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +# - 'RCA1 - root cause analysis reference for Diff1.' +# - 'RCA2 - root cause analysis reference for Diff2.' # VPP Performance Changes 2n-clx-x556 2t1c pdr - type: "table" @@ -3794,9 +3827,9 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Mellanox-CX556A' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" @@ -3805,7 +3838,7 @@ legend: - "Legend to Table" - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." # VPP Performance Changes 2n-clx-x556 4t2c pdr - type: "table" @@ -3815,9 +3848,9 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Mellanox-CX556A' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" @@ -3826,7 +3859,7 @@ legend: - "Legend to Table" - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." # VPP Performance Changes 2n-clx-x556 2t1c ndr - type: "table" @@ -3836,9 +3869,9 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Mellanox-CX556A' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" @@ -3847,7 +3880,7 @@ legend: - "Legend to Table" - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." # VPP Performance Changes 2n-clx-x556 4t2c ndr - type: "table" @@ -3857,9 +3890,9 @@ include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Mellanox-CX556A' and not 'NF_DENSITY'" columns: - - title: "2001" + - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" - - title: "2005" + - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp" comparisons: - title: "Diff" @@ -3868,7 +3901,7 @@ legend: - "Legend to Table" - "2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." + - "Diff - difference between 2001.0-2 and 2005.0-3 VPP version." # VPP Performance Changes 2n-dnv 1t1c pdr - type: "table" @@ -4054,97 +4087,149 @@ - "1908, 2001, 2005 - VPP versions tested." - "Diff - difference between 2001 and 2005 VPP version." -# VPP Performance Changes 3n-tsh 1t1c pdr -- type: "table" - title: "VPP Performance Changes 3n-tsh 1t1c pdr" - algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - filter: "'1T1C'" - columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-tsh-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-tsh-cmp" - comparisons: - - title: "Diff" - reference: 2 - compare: 3 - legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 3n-tsh 2t2c pdr -- type: "table" - title: "VPP Performance Changes 3n-tsh 2t2c pdr" - algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr" - include-tests: "PDR" # "PDR" | "NDR" | "MRR" - filter: "'2T2C'" - columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-tsh-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-tsh-cmp" - comparisons: - - title: "Diff" - reference: 2 - compare: 3 - legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 3n-tsh 1t1c ndr -- type: "table" - title: "VPP Performance Changes 3n-tsh 1t1c ndr" - algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - filter: "'1T1C'" - columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-tsh-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-tsh-cmp" - comparisons: - - title: "Diff" - reference: 2 - compare: 3 - legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." - -# VPP Performance Changes 3n-tsh 2t2c ndr -- type: "table" - title: "VPP Performance Changes 3n-tsh 2t2c ndr" - algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr" - include-tests: "NDR" # "PDR" | "NDR" | "MRR" - filter: "'2T2C'" - columns: - - title: "1908" - data-set: "vpp-performance-changes-3n-tsh-h1" - - title: "2001" - data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005" - data-set: "vpp-performance-changes-3n-tsh-cmp" - comparisons: - - title: "Diff" - reference: 2 - compare: 3 - legend: - - "Legend to Table" - - "1908, 2001, 2005 - VPP versions tested." - - "Diff - difference between 2001 and 2005 VPP version." +## VPP Performance Changes 3n-tsh 1t1c pdr +#- type: "table" +# title: "VPP Performance Changes 3n-tsh 1t1c pdr" +# algorithm: "table_comparison" +# output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr" +# include-tests: "PDR" # "PDR" | "NDR" | "MRR" +# filter: "'1T1C'" +# columns: +# - title: "1908.1-2" +# data-set: "vpp-performance-changes-3n-tsh-h1" +# - title: "2001.0-2" +# data-set: "vpp-performance-changes-3n-tsh-ref" +# - title: "2005.0-3" +# data-set: "vpp-performance-changes-3n-tsh-cmp-2" +# - title: "2005.0-4" +# data-set: "vpp-performance-changes-3n-tsh-cmp" +# comparisons: +# - title: "Diff1" +# reference: 3 +# compare: 4 +# - title: "Diff2" +# reference: 2 +# compare: 3 +## rca: +## - title: "RCA1" +## data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml" +## - title: "RCA2" +## data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml" +# legend: +# - "Legend to Table" +# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' +# - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' +# - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +## - 'RCA1 - root cause analysis reference for Diff1.' +## - 'RCA2 - root cause analysis reference for Diff2.' +# +## VPP Performance Changes 3n-tsh 2t2c pdr +#- type: "table" +# title: "VPP Performance Changes 3n-tsh 2t2c pdr" +# algorithm: "table_comparison" +# output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr" +# include-tests: "PDR" # "PDR" | "NDR" | "MRR" +# filter: "'2T2C'" +# columns: +# - title: "1908.1-2" +# data-set: "vpp-performance-changes-3n-tsh-h1" +# - title: "2001.0-2" +# data-set: "vpp-performance-changes-3n-tsh-ref" +# - title: "2005.0-3" +# data-set: "vpp-performance-changes-3n-tsh-cmp-2" +# - title: "2005.0-4" +# data-set: "vpp-performance-changes-3n-tsh-cmp" +# comparisons: +# - title: "Diff1" +# reference: 3 +# compare: 4 +# - title: "Diff2" +# reference: 2 +# compare: 3 +## rca: +## - title: "RCA1" +## data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml" +## - title: "RCA2" +## data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml" +# legend: +# - "Legend to Table" +# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' +# - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' +# - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +## - 'RCA1 - root cause analysis reference for Diff1.' +## - 'RCA2 - root cause analysis reference for Diff2.' +# +## VPP Performance Changes 3n-tsh 1t1c ndr +#- type: "table" +# title: "VPP Performance Changes 3n-tsh 1t1c ndr" +# algorithm: "table_comparison" +# output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr" +# include-tests: "NDR" # "PDR" | "NDR" | "MRR" +# filter: "'1T1C'" +# columns: +# - title: "1908.1-2" +# data-set: "vpp-performance-changes-3n-tsh-h1" +# - title: "2001.0-2" +# data-set: "vpp-performance-changes-3n-tsh-ref" +# - title: "2005.0-3" +# data-set: "vpp-performance-changes-3n-tsh-cmp-2" +# - title: "2005.0-4" +# data-set: "vpp-performance-changes-3n-tsh-cmp" +# comparisons: +# - title: "Diff1" +# reference: 3 +# compare: 4 +# - title: "Diff2" +# reference: 2 +# compare: 3 +## rca: +## - title: "RCA1" +## data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml" +## - title: "RCA2" +## data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml" +# legend: +# - "Legend to Table" +# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' +# - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' +# - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +## - 'RCA1 - root cause analysis reference for Diff1.' +## - 'RCA2 - root cause analysis reference for Diff2.' +# +## VPP Performance Changes 3n-tsh 2t2c ndr +#- type: "table" +# title: "VPP Performance Changes 3n-tsh 2t2c ndr" +# algorithm: "table_comparison" +# output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr" +# include-tests: "NDR" # "PDR" | "NDR" | "MRR" +# filter: "'2T2C'" +# columns: +# - title: "1908.1-2" +# data-set: "vpp-performance-changes-3n-tsh-h1" +# - title: "2001.0-2" +# data-set: "vpp-performance-changes-3n-tsh-ref" +# - title: "2005.0-3" +# data-set: "vpp-performance-changes-3n-tsh-cmp-2" +# - title: "2005.0-4" +# data-set: "vpp-performance-changes-3n-tsh-cmp" +# comparisons: +# - title: "Diff1" +# reference: 3 +# compare: 4 +# - title: "Diff2" +# reference: 2 +# compare: 3 +## rca: +## - title: "RCA1" +## data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml" +## - title: "RCA2" +## data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml" +# legend: +# - "Legend to Table" +# - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' +# - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.' +# - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.' +## - 'RCA1 - root cause analysis reference for Diff1.' +## - 'RCA2 - root cause analysis reference for Diff2.' # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr - type: "table"