X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fspecification.yaml;h=fe81e71d1f1dc34037af8e3e907fcfc280d3d09b;hp=4b61ddb19e399f23110e6e90eb4f1c46d01c1564;hb=f1583b9230d23cae435baed107aed6f94c9095b6;hpb=6addc914c636a0dbc1e12a661019cce1715baf77 diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 4b61ddb19e..fe81e71d1f 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -103,7 +103,7 @@ # - 10 # - 13 # csit-vpp-perf-1710-all: -# - 11 +# - 11l # - 12 # - 13 # TODO: Add the data sources @@ -116,24 +116,7 @@ # - 11 # - 12 # - 13 -# TODO: Add the data sources - plot-throughput-speedup-analysis: - csit-vpp-perf-1801-all: - - 122 - - 126 - - 129 - performance-improvements: - csit-vpp-perf-1707-all: - - 9 - - 10 - - 13 - - 14 - - 15 - - 16 - - 17 - - 18 - - 19 - - 21 + vpp-performance-changes: csit-vpp-perf-1710-all: - 11 - 12 @@ -147,34 +130,79 @@ - 20 # TODO: Add the data sources csit-vpp-perf-1801-all: - - 124 - - 127 - - 128 - csit-ligato-perf-1710-all: - - 5 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 16 - - 17 - csit-ligato-perf-1801-all: - - 1 - - 3 - - 7 -# TODO: + - 124 # sel + - 127 # sel + - 128 # sel + - 141 # sel + - 142 # sel + - 143 # sel + - 145 # sel + - 146 # sel +# TODO: Add the data sources + plot-throughput-speedup-analysis: + csit-vpp-perf-1801-all: + - 124 # sel + - 127 # sel + - 128 # sel + - 141 # sel + - 142 # sel + - 143 # sel + - 145 # sel + - 146 # sel +# performance-improvements: +# csit-vpp-perf-1707-all: +# - 9 +# - 10 +# - 13 +# - 14 +# - 15 +# - 16 +# - 17 +# - 18 +# - 19 +# - 21 +# csit-vpp-perf-1710-all: +# - 11 +# - 12 +# - 13 +# - 14 +# - 15 +# - 16 +# - 17 +# - 18 +# - 19 +# - 20 +## TODO: Add the data sources +# csit-vpp-perf-1801-all: +# - 124 +# - 127 +# - 128 +# csit-ligato-perf-1710-all: +# - 5 +# - 7 +# - 8 +# - 9 +# - 10 +# - 11 +# - 12 +# - 13 +# - 16 +# - 17 +# csit-ligato-perf-1801-all: +# - 1 +# - 3 +# - 7 +# TODO: Add the data sources vpp-perf-results: csit-vpp-perf-1801-all: - 122 - 126 - 129 + - 140 vpp-func-results: csit-vpp-functional-1801-ubuntu1604-virl: - "lastSuccessfulBuild" -# TODO: +# TODO: Add the data sources ligato-perf-results: csit-ligato-perf-1801-all: - 1 @@ -187,12 +215,17 @@ nsh-func-results: csit-nsh_sfc-verify-func-1801-ubuntu1604-virl: - 1 -# TODO: +# TODO: Add the data sources plot-vpp-throughput-latency: csit-vpp-perf-1801-all: - - 124 - - 127 - - 128 + - 124 # sel + - 127 # sel + - 128 # sel + - 141 # sel + - 142 # sel + - 143 # sel + - 145 # sel + - 146 # sel plot-dpdk-throughput-latency: csit-dpdk-perf-1801-all: - 1 @@ -204,7 +237,7 @@ - 8 - 10 - 12 -# TODO: +# TODO: Add the data sources plot-ligato-throughput-latency: csit-ligato-perf-1801-all: - 1 @@ -461,17 +494,17 @@ download-path: "{job}/{build}/robot/report/*zip*/{filename}" extract: "robot-plugin/output.xml" builds: - csit-vpp-perf-1707-all: - - 9 - - 10 - - 13 - - 14 - - 15 - - 16 - - 17 - - 18 - - 19 - - 21 +# csit-vpp-perf-1707-all: +# - 9 +# - 10 +# - 13 +# - 14 +# - 15 +# - 16 +# - 17 +# - 18 +# - 19 +# - 21 csit-vpp-perf-1710-all: - 11 - 12 @@ -488,9 +521,15 @@ - 122 # full - 126 # full - 129 # full + - 140 # full - 124 # sel - 127 # sel - 128 # sel + - 141 # sel + - 142 # sel + - 143 # sel + - 145 # sel + - 146 # sel csit-ligato-perf-1710-all: - 5 - 7 @@ -617,45 +656,91 @@ - type: "table" - title: "Performance improvements" - algorithm: "table_performance_improvements" - template: "{DIR[DTR,PERF,VPP,IMPRV]}/tmpl_performance_improvements.csv" + title: "VPP Performance Changes" + algorithm: "table_performance_comparison" output-file-ext: ".csv" - output-file: "{DIR[DTR,PERF,VPP,IMPRV]}/performance_improvements" - columns: - - - title: "Test Name" - data: "template 1" - - - title: "16.09 mean [Mpps]" - data: "template 2" - - - title: "17.01 mean [Mpps]" - data: "template 3" - - - title: "17.04 mean [Mpps]" - data: "template 4" - - - title: "17.07 mean [Mpps]" - data: "data csit-vpp-perf-1707-all mean" - - - title: "17.10 mean [Mpps]" - data: "data csit-vpp-perf-1710-all csit-ligato-perf-1710-all mean" - - - title: "18.01 mean [Mpps]" - data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all mean" - - - title: "18.01 stdev [Mpps]" - data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all stdev" - - - title: "17.10 to 18.01 change [%]" - data: "operation relative_change 5 6" - rows: "generated" - data: - "performance-improvements" - filter: "template" +# TODO: specify dir + output-file: "{DIR[STATIC,VPP]}/performance-changes" + reference: + title: "Release 1710" +# TODO: specify data sources + data: + csit-vpp-perf-1710-all: + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + compare: + title: "Release 1801" +# TODO: specify data sources + data: + csit-vpp-perf-1801-all: + - 124 # sel + - 127 # sel + - 128 # sel + - 141 # sel + - 142 # sel + - 143 # sel + - 145 # sel + - 146 # sel + data: "vpp-performance-changes" + filter: "all" parameters: + - "name" + - "parent" - "throughput" + # Number of the best and the worst tests presented in the table. Use 0 (zero) + # to present all tests. + nr-of-tests-shown: 20 + outlier-const: 1.5 + +#- +# type: "table" +# title: "Performance improvements" +# algorithm: "table_performance_improvements" +# template: "{DIR[DTR,PERF,VPP,IMPRV]}/tmpl_performance_improvements.csv" +# output-file-ext: ".csv" +# output-file: "{DIR[DTR,PERF,VPP,IMPRV]}/performance_improvements" +# columns: +# - +# title: "Test Name" +# data: "template 1" +# - +# title: "16.09 mean [Mpps]" +# data: "template 2" +# - +# title: "17.01 mean [Mpps]" +# data: "template 3" +# - +# title: "17.04 mean [Mpps]" +# data: "template 4" +# - +# title: "17.07 mean [Mpps]" +# data: "data csit-vpp-perf-1707-all mean" +# - +# title: "17.10 mean [Mpps]" +# data: "data csit-vpp-perf-1710-all csit-ligato-perf-1710-all mean" +# - +# title: "18.01 mean [Mpps]" +# data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all mean" +# - +# title: "18.01 stdev [Mpps]" +# data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all stdev" +# - +# title: "17.10 to 18.01 change [%]" +# data: "operation relative_change 5 6" +# rows: "generated" +# data: +# "performance-improvements" +# filter: "template" +# parameters: +# - "throughput" - type: "table" @@ -1114,7 +1199,7 @@ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-64B-l2-tsa-ndrdisc" data: "plot-throughput-speedup-analysis" - filter: "'NIC_Intel-X520-DA2' and '64B' and 'BASE' and 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + filter: "'NIC_Intel-X520-DA2' and '64B' and 'BASE' and 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'LXC' and not 'DOCKER'" parameters: - "throughput" - "parent" @@ -1133,7 +1218,7 @@ output-file: "{DIR[STATIC,VPP]}/40ge2p1xl710-64B-l2-tsa-ndrdisc" data: "plot-throughput-speedup-analysis" - filter: "'NIC_Intel-XL710' and '64B' and 'BASE' and 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + filter: "'NIC_Intel-XL710' and '64B' and 'BASE' and 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'LXC' and not 'DOCKER'" parameters: - "throughput" - "parent" @@ -1152,7 +1237,7 @@ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-64B-l2-tsa-pdrdisc" data: "plot-throughput-speedup-analysis" - filter: "'NIC_Intel-X520-DA2' and '64B' and 'BASE' and 'PDRDISC' and not 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + filter: "'NIC_Intel-X520-DA2' and '64B' and 'BASE' and 'PDRDISC' and not 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'LXC' and not 'DOCKER'" parameters: - "throughput" - "parent" @@ -1171,7 +1256,7 @@ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-64B-ip4-tsa-ndrdisc" data: "plot-throughput-speedup-analysis" - filter: "'NIC_Intel-X520-DA2' and '64B' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'" + filter: "'NIC_Intel-X520-DA2' and '64B' and 'IP4FWD' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST' and not 'FEATURE' and not 'DOT1Q' and not 'IPSEC'" parameters: - "throughput" - "parent" @@ -1190,7 +1275,7 @@ output-file: "{DIR[STATIC,VPP]}/40ge2p1xl710-64B-ip4-tsa-ndrdisc" data: "plot-throughput-speedup-analysis" - filter: "'NIC_Intel-XL710' and '64B' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'" + filter: "'NIC_Intel-XL710' and '64B' and 'IP4FWD' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST' and not 'FEATURE' and not 'DOT1Q' and not 'IPSEC'" parameters: - "throughput" - "parent" @@ -1209,7 +1294,7 @@ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-64B-ip4-tsa-pdrdisc" data: "plot-throughput-speedup-analysis" - filter: "'NIC_Intel-X520-DA2' and '64B' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and not 'VHOST'" + filter: "'NIC_Intel-X520-DA2' and '64B' and 'IP4FWD' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and not 'VHOST' and not 'FEATURE' and not 'DOT1Q' and not 'IPSEC'" parameters: - "throughput" - "parent" @@ -1228,7 +1313,7 @@ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-78B-ip6-tsa-ndrdisc" data: "plot-throughput-speedup-analysis" - filter: "'NIC_Intel-X520-DA2' and '78B' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'" + filter: "'NIC_Intel-X520-DA2' and '78B' and 'IP6FWD' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'" parameters: - "throughput" - "parent" @@ -1247,7 +1332,7 @@ output-file: "{DIR[STATIC,VPP]}/40ge2p1xl710-78B-ip6-tsa-ndrdisc" data: "plot-throughput-speedup-analysis" - filter: "'NIC_Intel-XL710' and '78B' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'" + filter: "'NIC_Intel-XL710' and '78B' and 'IP6FWD' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'" parameters: - "throughput" - "parent" @@ -1266,7 +1351,7 @@ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-78B-ip6-tsa-pdrdisc" data: "plot-throughput-speedup-analysis" - filter: "'NIC_Intel-X520-DA2' and '78B' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and not 'VHOST'" + filter: "'NIC_Intel-X520-DA2' and '78B' and 'IP6FWD' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and not 'VHOST'" parameters: - "throughput" - "parent" @@ -1724,8 +1809,45 @@ whiskerwidth: 0 layout: title: "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" - layout: - "plot-throughput" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [2000000, 6000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 - type: "plot" @@ -1787,8 +1909,45 @@ whiskerwidth: 0 layout: title: "78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-pdrdisc" - layout: - "plot-throughput" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [4000000, 12000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 # VPP VM VHOST - @@ -1809,8 +1968,45 @@ whiskerwidth: 0 layout: title: "64B-1t1c-.*vhost.*-ndrdisc" - layout: - "plot-throughput" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [0, 3500000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 - type: "plot" @@ -2813,8 +3009,45 @@ whiskerwidth: 0 layout: title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-memif-ndrdisc" - layout: - "plot-throughput" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [0, 4500000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 - type: "plot" @@ -2834,8 +3067,45 @@ whiskerwidth: 0 layout: title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-memif-ndrdisc" - layout: - "plot-throughput" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [0, 8000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 - type: "plot" @@ -2855,8 +3125,45 @@ whiskerwidth: 0 layout: title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-memif-pdrdisc" - layout: - "plot-throughput" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [0, 4500000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 - type: "plot" @@ -2876,8 +3183,45 @@ whiskerwidth: 0 layout: title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-memif-pdrdisc" - layout: - "plot-throughput" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [0, 8000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 # Container orchestrated -