X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fspecifications%2Freport%2Felements.yaml;h=e78b142e8760aa80900caf8cdbba27a0bc2682af;hb=0169b3d8ed690fb76f0c6d43ee306bb2b2761976;hp=1ffa605906b3aa4bb4bbc0f5b45da581d49ad5f3;hpb=d4d766e181d645295a16fb035fefc96490429d6d;p=csit.git diff --git a/resources/tools/presentation/specifications/report/elements.yaml b/resources/tools/presentation/specifications/report/elements.yaml index 1ffa605906..e78b142e87 100644 --- a/resources/tools/presentation/specifications/report/elements.yaml +++ b/resources/tools/presentation/specifications/report/elements.yaml @@ -1403,6 +1403,147 @@ # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' +# VPP Performance Changes 3n-icx-e810xxv 2t1c pdr +- type: "table" + title: "VPP Performance Changes 3n-icx-e810xxv 2t1c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-icx-e810xxv-2t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-E810XXV'" + columns: + - title: "2206.0-10" + data-set: "vpp-3n-icx-hist-iter" + - title: "2210.0-11" + data-set: "vpp-3n-icx-prev-iter-old-env" + # - title: "2206.0-11" + # data-set: "vpp-3n-icx-prev-iter-new-env" + - title: "2302.0-11" + data-set: "vpp-3n-icx-curr-iter" + comparisons: + # - title: "2206.0-11 vs 2206.0-10" + # reference: 2 + # compare: 3 + # - title: "2210.0-11 vs 2206.0-11" + # reference: 3 + # compare: 4 + - title: "2302.0-11 vs 2210.0-11" + reference: 2 + compare: 3 + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-icx with e810xxv.' + - 'YYMM.V-E: [uSec +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "10" for CSIT rls2206, "11" for CSIT rls2210 and rls2302).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base. [N] for Root Cause Analysis (RCA) reference, see RCA list below.' + # - '2206.0-11 vs 2206.0-10: Compares CSIT environment current vs previous with previous VPP code.' + # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' + - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' + +# VPP Performance Changes 3n-icx-e810xxv 4t2c pdr +- type: "table" + title: "VPP Performance Changes 3n-icx-e810xxv 4t2c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-icx-e810xxv-4t2c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-E810XXV'" + columns: + - title: "2206.0-10" + data-set: "vpp-3n-icx-hist-iter" + - title: "2210.0-11" + data-set: "vpp-3n-icx-prev-iter-old-env" + # - title: "2206.0-11" + # data-set: "vpp-3n-icx-prev-iter-new-env" + - title: "2302.0-11" + data-set: "vpp-3n-icx-curr-iter" + comparisons: + # - title: "2206.0-11 vs 2206.0-10" + # reference: 2 + # compare: 3 + # - title: "2210.0-11 vs 2206.0-11" + # reference: 3 + # compare: 4 + - title: "2302.0-11 vs 2210.0-11" + reference: 2 + compare: 3 + legend: + - '' + - 'Test Case: Test cases executed for: PDR throughput on 3n-icx with e810xxv.' + - 'YYMM.V-E: [uSec +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "10" for CSIT rls2206, "11" for CSIT rls2210 and rls2302).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base. [N] for Root Cause Analysis (RCA) reference, see RCA list below.' + # - '2206.0-11 vs 2206.0-10: Compares CSIT environment current vs previous with previous VPP code.' + # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' + - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' + +# VPP Performance Changes 3n-icx-e810xxv 2t1c ndr +- type: "table" + title: "VPP Performance Changes 3n-icx-e810xxv 2t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-icx-e810xxv-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-E810XXV'" + columns: + - title: "2206.0-10" + data-set: "vpp-3n-icx-hist-iter" + - title: "2210.0-11" + data-set: "vpp-3n-icx-prev-iter-old-env" + # - title: "2206.0-11" + # data-set: "vpp-3n-icx-prev-iter-new-env" + - title: "2302.0-11" + data-set: "vpp-3n-icx-curr-iter" + comparisons: + # - title: "2206.0-11 vs 2206.0-10" + # reference: 2 + # compare: 3 + # - title: "2210.0-11 vs 2206.0-11" + # reference: 3 + # compare: 4 + - title: "2302.0-11 vs 2210.0-11" + reference: 2 + compare: 3 + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-icx with e810xxv.' + - 'YYMM.V-E: [uSec +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "10" for CSIT rls2206, "11" for CSIT rls2210 and rls2302).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base. [N] for Root Cause Analysis (RCA) reference, see RCA list below.' + # - '2206.0-11 vs 2206.0-10: Compares CSIT environment current vs previous with previous VPP code.' + # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' + - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' + +# VPP Performance Changes 3n-icx-e810xxv 4t2c ndr +- type: "table" + title: "VPP Performance Changes 3n-icx-e810xxv 4t2c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-icx-e810xxv-4t2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-E810XXV'" + columns: + - title: "2206.0-10" + data-set: "vpp-3n-icx-hist-iter" + - title: "2210.0-11" + data-set: "vpp-3n-icx-prev-iter-old-env" + # - title: "2206.0-11" + # data-set: "vpp-3n-icx-prev-iter-new-env" + - title: "2302.0-11" + data-set: "vpp-3n-icx-curr-iter" + comparisons: + # - title: "2206.0-11 vs 2206.0-10" + # reference: 2 + # compare: 3 + # - title: "2210.0-11 vs 2206.0-11" + # reference: 3 + # compare: 4 + - title: "2302.0-11 vs 2210.0-11" + reference: 2 + compare: 3 + legend: + - '' + - 'Test Case: Test cases executed for: NDR throughput on 3n-icx with e810xxv.' + - 'YYMM.V-E: [uSec +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "10" for CSIT rls2206, "11" for CSIT rls2210 and rls2302).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base. [N] for Root Cause Analysis (RCA) reference, see RCA list below.' + # - '2206.0-11 vs 2206.0-10: Compares CSIT environment current vs previous with previous VPP code.' + # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' + - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' + + # VPP Performance Changes 2n-icx 2t1c pdr - type: "table" title: "VPP Performance Changes 2n-icx 2t1c pdr" @@ -2623,7 +2764,6 @@ # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' - # VPP Performance Changes 3n-icx 2t1c mrr - type: "table" title: "VPP Performance Changes 3n-icx 2t1c mrr" @@ -2729,6 +2869,112 @@ # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' + +# VPP Performance Changes 3n-icx-e810xxv 2t1c mrr +- type: "table" + title: "VPP Performance Changes 3n-icx-e810xxv 2t1c mrr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-icx-e810xxv-2t1c-mrr" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'MRR' and 'NIC_Intel-E810XXV'" + columns: + - title: "2206.0-10" + data-set: "vpp-3n-icx-hist-iter-mrr" + - title: "2210.0-11" + data-set: "vpp-3n-icx-prev-iter-mrr-old-env" + # - title: "2206.0-11" + # data-set: "vpp-3n-icx-prev-iter-mrr-new-env" + - title: "2302.0-11" + data-set: "vpp-3n-icx-curr-iter-mrr" + comparisons: + # - title: "2206.0-11 vs 2206.0-10" + # reference: 2 + # compare: 3 + # - title: "2210.0-11 vs 2206.0-11" + # reference: 3 + # compare: 4 + - title: "2302.0-11 vs 2210.0-11" + reference: 2 + compare: 3 + legend: + - '' + - 'Test Case: Test cases executed for: MRR throughput on 3n-icx with e810xxv.' + - 'YYMM.V-E: [uSec +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "10" for CSIT rls2206, "11" for CSIT rls2210 and rls2302).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base. [N] for Root Cause Analysis (RCA) reference, see RCA list below.' + # - '2206.0-11 vs 2206.0-10: Compares CSIT environment current vs previous with previous VPP code.' + # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' + - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' + +# VPP Performance Changes 3n-icx-e810xxv 4t2c mrr +- type: "table" + title: "VPP Performance Changes 3n-icx-e810xxv 4t2c mrr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-icx-e810xxv-4t2c-mrr" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'MRR' and 'NIC_Intel-E810XXV'" + columns: + - title: "2206.0-10" + data-set: "vpp-3n-icx-hist-iter-mrr" + - title: "2210.0-11" + data-set: "vpp-3n-icx-prev-iter-mrr-old-env" + # - title: "2206.0-11" + # data-set: "vpp-3n-icx-prev-iter-mrr-new-env" + - title: "2302.0-11" + data-set: "vpp-3n-icx-curr-iter-mrr" + comparisons: + # - title: "2206.0-11 vs 2206.0-10" + # reference: 2 + # compare: 3 + # - title: "2210.0-11 vs 2206.0-11" + # reference: 3 + # compare: 4 + - title: "2302.0-11 vs 2210.0-11" + reference: 2 + compare: 3 + legend: + - '' + - 'Test Case: Test cases executed for: MRR throughput on 3n-icx with e810xxv.' + - 'YYMM.V-E: [uSec +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "10" for CSIT rls2206, "11" for CSIT rls2210 and rls2302).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base. [N] for Root Cause Analysis (RCA) reference, see RCA list below.' + # - '2206.0-11 vs 2206.0-10: Compares CSIT environment current vs previous with previous VPP code.' + # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' + - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' + +# VPP Performance Changes 3n-icx-e810xxv 8t4c mrr +- type: "table" + title: "VPP Performance Changes 3n-icx-e810xxv 8t4c mrr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-icx-e810xxv-8t4c-mrr" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" + filter: "'8T4C' and 'MRR' and 'NIC_Intel-E810XXV'" + columns: + - title: "2206.0-10" + data-set: "vpp-3n-icx-hist-iter-mrr" + - title: "2210.0-11" + data-set: "vpp-3n-icx-prev-iter-mrr-old-env" + # - title: "2206.0-11" + # data-set: "vpp-3n-icx-prev-iter-mrr-new-env" + - title: "2302.0-11" + data-set: "vpp-3n-icx-curr-iter-mrr" + comparisons: + # - title: "2206.0-11 vs 2206.0-10" + # reference: 2 + # compare: 3 + # - title: "2210.0-11 vs 2206.0-11" + # reference: 3 + # compare: 4 + - title: "2302.0-11 vs 2210.0-11" + reference: 2 + compare: 3 + legend: + - '' + - 'Test Case: Test cases executed for: MRR throughput on 3n-icx with e810xxv.' + - 'YYMM.V-E: [uSec +- StDev] results for VPP (YY year, MM month, V minor version) tested in CSIT (E environment version, "10" for CSIT rls2206, "11" for CSIT rls2210 and rls2302).' + - 'YYMM.V-E vs YYMM.V-E: [% +- StDev] Relative difference, latter is the base. [N] for Root Cause Analysis (RCA) reference, see RCA list below.' + # - '2206.0-11 vs 2206.0-10: Compares CSIT environment current vs previous with previous VPP code.' + # - '2210.0-11 vs 2206.0-11: Compares VPP code current vs previous in the current CSIT environment.' + - '2302.0-11 vs 2210.0-11: Compares current VPP code in the current CSIT environment vs previous VPP code in the previous CSIT environment.' + # VPP Performance Changes 2n-icx 2t1c mrr - type: "table" title: "VPP Performance Changes 2n-icx 2t1c mrr" @@ -4308,17 +4554,35 @@ - "name" - "conf-history" -# Detailed Test Results - VPP Device Results - Ubuntu +# Detailed Test Results - VPP Device Results - 1n-skx +- type: "table" + title: "Detailed Test Results - VPP Device Results - 1n-skx" + algorithm: "table_merged_details" + output-file: "{DIR[DTR,FUNC,VPP,DEVICE,1N,SKX]}/vpp_device_results_1n_skx" + columns: + - title: " |prein| Test Name |preout| " + data: "data name" + - title: " |prein| Status |preout| " + data: "data status" + data: "vpp-device-1n-skx" + filter: "all" + parameters: + - "name" + - "parent" + - "doc" + - "status" + +# Detailed Test Results - VPP Device Results - 1n-tx2 - type: "table" - title: "Detailed Test Results - VPP Device Results - Ubuntu" + title: "Detailed Test Results - VPP Device Results - 1n-tx2" algorithm: "table_merged_details" - output-file: "{DIR[DTR,FUNC,VPP,DEVICE,UBUNTU]}/vpp_device_results_ubuntu" + output-file: "{DIR[DTR,FUNC,VPP,DEVICE,1N,TX2]}/vpp_device_results_1n_tx2" columns: - title: " |prein| Test Name |preout| " data: "data name" - title: " |prein| Status |preout| " data: "data status" - data: "vpp-device-results-ubuntu" + data: "vpp-device-1n-tx2" filter: "all" parameters: - "name" @@ -4326,17 +4590,34 @@ - "doc" - "status" -# Test configuration - VPP Device Test Configs - Ubuntu +# Test configuration - VPP Device Test Configs - 1n-skx - type: "table" - title: "Test configuration - VPP Device Test Configs - Ubuntu" + title: "Test configuration - VPP Device Test Configs - 1n-skx" algorithm: "table_merged_details" - output-file: "{DIR[DTC,FUNC,VPP,DEVICE,UBUNTU]}/vpp_device_configuration_ubuntu" + output-file: "{DIR[DTC,FUNC,VPP,DEVICE,1N,SKX]}/vpp_device_configuration_1n_skx" columns: - title: " |prein| Test Name |preout| " data: "data name" - title: " |prein| VPP API Test Commands History - Commands Used Per Test Case |preout| " data: "data conf-history" - data: "vpp-device-results-ubuntu" + data: "vpp-device-1n-skx" + filter: "all" + parameters: + - "parent" + - "name" + - "conf-history" + +# Test configuration - VPP Device Test Configs - 1n-tx2 +- type: "table" + title: "Test configuration - VPP Device Test Configs - 1n-tx2" + algorithm: "table_merged_details" + output-file: "{DIR[DTC,FUNC,VPP,DEVICE,1N,TX2]}/vpp_device_configuration_1n_tx2" + columns: + - title: " |prein| Test Name |preout| " + data: "data name" + - title: " |prein| VPP API Test Commands History - Commands Used Per Test Case |preout| " + data: "data conf-history" + data: "vpp-device-1n-tx2" filter: "all" parameters: - "parent" @@ -5139,7 +5420,7 @@ - "vts" - "lb" - "nfv_density" - - "hoststack" + # - "hoststack" level-2: - "2n-clx" - "2n-icx" @@ -5156,6 +5437,7 @@ - "xxv710" - "x710" - "e810cq" + - "e822cq" - "e810xxv" - "x553" - "cx556a" @@ -5204,7 +5486,7 @@ - "vts" - "lb" - "nfv_density" - - "hoststack" + # - "hoststack" level-2: - "2n-clx" - "2n-icx" @@ -5221,6 +5503,7 @@ - "xxv710" - "x710" - "e810cq" + - "e822cq" - "e810xxv" - "x553" - "cx556a" @@ -5269,7 +5552,7 @@ - "vts" - "lb" - "nfv_density" - - "hoststack" + # - "hoststack" level-2: - "2n-clx" - "2n-icx" @@ -5286,6 +5569,7 @@ - "xxv710" - "x710" - "e810cq" + - "e822cq" - "e810xxv" - "x553" - "cx556a" @@ -5334,7 +5618,7 @@ - "vts" - "lb" - "nfv_density" - - "hoststack" + # - "hoststack" level-2: - "2n-clx" - "2n-icx" @@ -5351,6 +5635,7 @@ - "xxv710" - "x710" - "e810cq" + - "e822cq" - "e810xxv" - "x553" - "cx556a" @@ -5399,7 +5684,7 @@ - "vts" - "lb" - "nfv_density" - - "hoststack" + # - "hoststack" level-2: - "2n-clx" - "2n-icx" @@ -5416,6 +5701,7 @@ - "xxv710" - "x710" - "e810cq" + - "e822cq" - "e810xxv" - "x553" - "cx556a" @@ -5423,23 +5709,43 @@ - "vic1385" - "ena" -# VPP Device Results - Ubuntu +# VPP Device Results - 1n-skx +- type: "file" + title: "VPP Device Results - 1n-skx" + algorithm: "file_test_results" + output-file: "{DIR[DTR,FUNC,VPP,DEVICE,1N,SKX]}" + dir-tables: "{DIR[DTR,FUNC,VPP,DEVICE,1N,SKX]}" + data: "vpp-device-1n-skx" + filter: "all" + data-start-level: 4 + +# VPP Device Results - 1n-tx2 +- type: "file" + title: "VPP Device Results - 1n-tx2" + algorithm: "file_test_results" + output-file: "{DIR[DTR,FUNC,VPP,DEVICE,1N,TX2]}" + dir-tables: "{DIR[DTR,FUNC,VPP,DEVICE,1N,TX2]}" + data: "vpp-device-1n-tx2" + filter: "all" + data-start-level: 4 + +# VPP Device Configuration - 1n-skx - type: "file" - title: "VPP Device Results - Ubuntu" + title: "VPP Device Configuration - 1n-skx" algorithm: "file_test_results" - output-file: "{DIR[DTR,FUNC,VPP,DEVICE,UBUNTU]}" - dir-tables: "{DIR[DTR,FUNC,VPP,DEVICE,UBUNTU]}" - data: "vpp-device-results-ubuntu" + output-file: "{DIR[DTC,FUNC,VPP,DEVICE,1N,SKX]}" + dir-tables: "{DIR[DTC,FUNC,VPP,DEVICE,1N,SKX]}" + data: "vpp-device-1n-skx" filter: "all" data-start-level: 4 -# VPP Device Configuration - Ubuntu +# VPP Device Configuration - 1n-tx2 - type: "file" - title: "VPP Device Configuration - Ubuntu" + title: "VPP Device Configuration - 1n-tx2" algorithm: "file_test_results" - output-file: "{DIR[DTC,FUNC,VPP,DEVICE,UBUNTU]}" - dir-tables: "{DIR[DTC,FUNC,VPP,DEVICE,UBUNTU]}" - data: "vpp-device-results-ubuntu" + output-file: "{DIR[DTC,FUNC,VPP,DEVICE,1N,TX2]}" + dir-tables: "{DIR[DTC,FUNC,VPP,DEVICE,1N,TX2]}" + data: "vpp-device-1n-tx2" filter: "all" data-start-level: 4