DIR[DTR,FUNC,HC]: "{DIR[DTR]}/honeycomb_functional_results"
DIR[DTR,FUNC,NSHSFC]: "{DIR[DTR]}/nshsfc_functional_results"
DIR[DTR,PERF,VPP,IMPRV]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_improvements"
+ DIR[DTR,PERF,VPP,IMPACT,SPECTRE]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_impact_spectre"
+ DIR[DTR,PERF,VPP,IMPACT,MELTDOWN]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_impact_meltdown"
# Detailed test configurations
DIR[DTC]: "{DIR[WORKING,SRC]}/test_configuration"
-
type: "configuration"
data-sets:
+ vpp-meltdown-impact:
+ csit-vpp-perf-1707-all:
+ - 9
+ - 10
+ - 13
+ csit-vpp-perf-1710-all:
+ - 11
+ - 12
+ - 13
+ vpp-spectre-impact:
+ csit-vpp-perf-1707-all:
+ - 9
+ - 10
+ - 13
+ csit-vpp-perf-1710-all:
+ - 11
+ - 12
+ - 13
+# TODO: Add the data sources
+ plot-throughput-speedup-analysis:
+ csit-vpp-perf-1801-all:
+ - 12
performance-improvements:
csit-vpp-perf-1707-all:
- 9
- 18
- 19
- 20
+# TODO:
+ csit-vpp-perf-1801-all:
+ - 13
+ - 14
csit-ligato-perf-1710-all:
- 5
- 7
- 13
- 16
- 17
+ csit-ligato-perf-1801-all:
+ - 1
+ - 3
+ - 7
+# TODO:
vpp-perf-results:
- csit-vpp-perf-1710-all:
- - 20
- - 23
+ csit-vpp-perf-1801-all:
+ - 12
vpp-func-results:
- csit-vpp-functional-1710-ubuntu1604-virl:
- - 126
+ csit-vpp-functional-1801-ubuntu1604-virl:
+ - "lastSuccessfulBuild"
+# TODO:
ligato-perf-results:
- csit-ligato-perf-1710-all:
- - 17
+ csit-ligato-perf-1801-all:
+ - 1
dpdk-perf-results:
- csit-dpdk-perf-1710-all:
- - 13
+ csit-dpdk-perf-1801-all:
+ - 12
hc-func-results:
- csit-hc2vpp-verify-func-1710-ubuntu1604:
+ csit-hc2vpp-verify-func-1801-ubuntu1604:
- "lastSuccessfulBuild"
+# TODO: Will we use this one?
+# hc2vpp-csit-integration-odl-carbon-1801-ubuntu1604:
+# - lastSuccessfulBuild
nsh-func-results:
- csit-nsh_sfc-verify-func-1710-ubuntu1604-virl:
- - 7
+ csit-nsh_sfc-verify-func-1801-ubuntu1604-virl:
+ - 1
+# TODO:
plot-vpp-throughput-latency:
- csit-vpp-perf-1710-all:
- - 11
- - 12
+ csit-vpp-perf-1801-all:
- 13
- 14
- - 15
- - 16
- - 17
- - 18
- - 19
- - 20
plot-dpdk-throughput-latency:
- csit-dpdk-perf-1710-all:
- - 2
+ csit-dpdk-perf-1801-all:
+ - 1
- 3
+ - 4
- 5
- 6
- 7
- 8
- 9
- 10
- - 11
- - 13
+ - 12
+# TODO:
plot-ligato-throughput-latency:
- csit-ligato-perf-1710-all:
- - 5
+ csit-ligato-perf-1801-all:
+ - 1
+ - 3
- 7
- - 8
- - 9
- - 10
- - 11
- - 12
- - 13
- - 16
- - 17
plot-layouts:
plot-throughput:
xaxis:
orientation: "h"
width: 700
height: 1000
+ plot-throughput-speedup-analysis:
+ xaxis:
+ autorange: True
+ autotick: False
+ fixedrange: False
+ gridcolor: "rgb(238, 238, 238)"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ tickmode: "linear"
+ tickangle: 270
+ zeroline: False
+ yaxis:
+ title: "Throughput speedup factor"
+ gridcolor: "rgb(238, 238, 238)"
+ hoverformat: ".4s"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ range: []
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ zeroline: False
+ legend:
+ orientation: "h"
+ xanchor: "center"
+ yanchor: "top"
+ x: 0.5
+ y: 1
+ bgcolor: "rgba(255, 255, 255, 0)"
+ bordercolor: "rgba(255, 255, 255, 0)"
+ barmode: "group"
+ bargap: 0.15
+ bargroupgap: 0.1
+ autosize: False
+ margin:
+ 't': 50
+ 'b': 200
+ 'l': 50
+ 'r': 20
+ showlegend: True
+ width: 700
+ height: 1000
-
type: "debug"
extract: "robot-plugin/output.xml" # Only for zip
builds:
# The files must be in the directory DIR[WORKING,DATA]
- csit-dpdk-perf-1707-all:
- -
- build: 10
- file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__10.xml"
+ csit-vpp-perf-1801-all:
-
- build: 9
- file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__9.xml"
- csit-nsh_sfc-verify-func-1707-ubuntu1604-virl:
+ build: 1
+ file: "{DIR[WORKING,DATA]}/output.xml"
-
build: 2
- file: "{DIR[WORKING,DATA]}/csit-nsh_sfc-verify-func-1707-ubuntu1604-virl-2.xml"
- csit-vpp-functional-1707-ubuntu1604-virl:
- -
- build: lastSuccessfulBuild
- file: "{DIR[WORKING,DATA]}/csit-vpp-functional-1707-ubuntu1604-virl-lastSuccessfulBuild.xml"
- hc2vpp-csit-integration-1707-ubuntu1604:
- -
- build: lastSuccessfulBuild
- file: "{DIR[WORKING,DATA]}/hc2vpp-csit-integration-1707-ubuntu1604-lastSuccessfulBuild.xml"
- csit-vpp-perf-1707-all:
- -
- build: 16
- file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__16__output.xml"
- -
- build: 17
- file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__17__output.xml"
+ file: "{DIR[WORKING,DATA]}/output.xml"
-
type: "static"
- 19
- 20
- 23
- csit-dpdk-perf-1710-all:
- - 2
- - 3
- - 5
- - 6
- - 7
- - 8
- - 9
- - 10
- - 11
+# TODO:
+ csit-vpp-perf-1801-all:
+ - 12
- 13
+ - 14
csit-ligato-perf-1710-all:
- 5
- 7
- 13
- 16
- 17
- csit-vpp-functional-1710-ubuntu1604-virl:
- - 126
- csit-nsh_sfc-verify-func-1710-ubuntu1604-virl:
+# TODO:
+ csit-ligato-perf-1801-all:
+ - 1
+ - 3
+ - 7
+ csit-dpdk-perf-1801-all:
+ - 1
+ - 3
+ - 4
+ - 5
+ - 6
- 7
- csit-hc2vpp-verify-func-1710-ubuntu1604:
+ - 8
+ - 9
+ - 10
+ - 12
+ csit-vpp-functional-1801-ubuntu1604-virl:
- lastSuccessfulBuild
+ csit-nsh_sfc-verify-func-1801-ubuntu1604-virl:
+ - 1
+ csit-hc2vpp-verify-func-1801-ubuntu1604:
+ - lastSuccessfulBuild
+# TODO: Will we use this one?
+# hc2vpp-csit-integration-odl-carbon-1801-ubuntu1604:
+# - lastSuccessfulBuild
-
type: "output"
### T A B L E S ###
################################################################################
+-
+ type: "table"
+ title: "Performance Impact of Meltdown Patches"
+ algorithm: "table_performance_comparison"
+ output-file-ext: ".csv"
+# TODO: specify dir
+ output-file: "{DIR[DTR,PERF,VPP,IMPACT,MELTDOWN]}/meltdown-impact"
+ reference:
+ title: "No Meltdown"
+# TODO: specify data sources
+ data:
+ csit-vpp-perf-1707-all:
+ - 9
+ - 10
+ - 13
+ compare:
+ title: "Meltdown Patches Applied"
+# TODO: specify data sources
+ data:
+ csit-vpp-perf-1710-all:
+ - 11
+ - 12
+ - 13
+ data:
+ "vpp-meltdown-impact"
+ filter: "all"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ # Number of the best and the worst tests presented in the table. Use 0 (zero)
+ # to present all tests.
+ nr-of-tests-shown: 20
+
+-
+ type: "table"
+ title: "Performance Impact of Spectre Patches"
+ algorithm: "table_performance_comparison"
+ output-file-ext: ".csv"
+# TODO: specify dir
+ output-file: "{DIR[DTR,PERF,VPP,IMPACT,SPECTRE]}/meltdown-spectre-impact"
+ reference:
+ title: "No Spectre"
+# TODO: specify data sources
+ data:
+ csit-vpp-perf-1707-all:
+ - 9
+ - 10
+ - 13
+ compare:
+ title: "Spectre Patches Applied"
+# TODO: specify data sources
+ data:
+ csit-vpp-perf-1710-all:
+ - 11
+ - 12
+ - 13
+ data:
+ "vpp-spectre-impact"
+ filter: "all"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ # Number of the best and the worst tests presented in the table. Use 0 (zero)
+ # to present all tests.
+ nr-of-tests-shown: 20
+
-
type: "table"
title: "Performance improvements"
title: "17.10 mean [Mpps]"
data: "data csit-vpp-perf-1710-all csit-ligato-perf-1710-all mean"
-
- title: "17.10 stdev [Mpps]"
- data: "data csit-vpp-perf-1710-all csit-ligato-perf-1710-all stdev"
+ title: "18.01 mean [Mpps]"
+ data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all mean"
-
- title: "17.07 to 17.10 change [%]"
- data: "operation relative_change 5 6"
+ title: "18.01 stdev [Mpps]"
+ data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all stdev"
+ -
+ title: "17.10 to 18.01 change [%]"
+ data: "operation relative_change 6 7"
rows: "generated"
data:
"performance-improvements"
### P L O T S ###
################################################################################
+# Plot Throughput Speedup Analysis
+
+# L2 - 10ge2p1x520 - NDR
+-
+ type: "plot"
+ title: "TSA: 64B-*-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-64B-l2-tsa-ndrdisc"
+ data:
+ "plot-throughput-speedup-analysis"
+ filter: "'NIC_Intel-X520-DA2' and '64B' and 'BASE' and 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "64B-*-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc"
+ layout:
+ "plot-throughput-speedup-analysis"
+
+# L2 - 40ge2p1xl710 - NDR
+-
+ type: "plot"
+ title: "TSA: 64B-*-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/40ge2p1xl710-64B-l2-tsa-ndrdisc"
+ data:
+ "plot-throughput-speedup-analysis"
+ filter: "'NIC_Intel-XL710' and '64B' and 'BASE' and 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "64B-*-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc"
+ layout:
+ "plot-throughput-speedup-analysis"
+
+# L2 - 10ge2p1x520 - PDR
+-
+ type: "plot"
+ title: "TSA: 64B-*-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-64B-l2-tsa-pdrdisc"
+ data:
+ "plot-throughput-speedup-analysis"
+ filter: "'NIC_Intel-X520-DA2' and '64B' and 'BASE' and 'PDRDISC' and not 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "64B-*-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc"
+ layout:
+ "plot-throughput-speedup-analysis"
+
+# IPv4 - 10ge2p1x520 - NDR
+-
+ type: "plot"
+ title: "TSA: 64B-*-ethip4-ip4(base|scale)*ndrdisc"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-64B-ip4-tsa-ndrdisc"
+ data:
+ "plot-throughput-speedup-analysis"
+ filter: "'NIC_Intel-X520-DA2' and '64B' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "64B-*-ethip4-ip4(base|scale)*ndrdisc"
+ layout:
+ "plot-throughput-speedup-analysis"
+
+# IPv4 - 40ge2p1xl710 - NDR
+-
+ type: "plot"
+ title: "TSA: 64B-*-ethip4-ip4(base|scale)*ndrdisc"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/40ge2p1xl710-64B-ip4-tsa-ndrdisc"
+ data:
+ "plot-throughput-speedup-analysis"
+ filter: "'NIC_Intel-XL710' and '64B' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "64B-*-ethip4-ip4(base|scale)*ndrdisc"
+ layout:
+ "plot-throughput-speedup-analysis"
+
+# IPv4 - 10ge2p1x520 - PDR
+-
+ type: "plot"
+ title: "TSA: 64B-*-ethip4-ip4(base|scale)*pdrdisc"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-64B-ip4-tsa-pdrdisc"
+ data:
+ "plot-throughput-speedup-analysis"
+ filter: "'NIC_Intel-X520-DA2' and '64B' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "64B-*-ethip4-ip4(base|scale)*pdrdisc"
+ layout:
+ "plot-throughput-speedup-analysis"
+
+# IPv6 - 10ge2p1x520 - NDR
+-
+ type: "plot"
+ title: "TSA: 78B-*-ethip6-ip6(base|scale)*ndrdisc"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-78B-ip6-tsa-ndrdisc"
+ data:
+ "plot-throughput-speedup-analysis"
+ filter: "'NIC_Intel-X520-DA2' and '78B' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "78B-*-ethip6-ip6(base|scale)*ndrdisc"
+ layout:
+ "plot-throughput-speedup-analysis"
+
+# IPv6 - 40ge2p1xl710 - NDR
+-
+ type: "plot"
+ title: "TSA: 78B-*-ethip6-ip6(base|scale)*ndrdisc"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/40ge2p1xl710-78B-ip6-tsa-ndrdisc"
+ data:
+ "plot-throughput-speedup-analysis"
+ filter: "'NIC_Intel-XL710' and '78B' and ('BASE' or 'SCALE') and 'NDRDISC' and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "78B-*-ethip6-ip6(base|scale)*ndrdisc"
+ layout:
+ "plot-throughput-speedup-analysis"
+
+# IPv6 - 10ge2p1x520 - PDR
+-
+ type: "plot"
+ title: "TSA: 78B-*-ethip6-ip6(base|scale)*pdrdisc"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/10ge2p1x520-78B-ip6-tsa-pdrdisc"
+ data:
+ "plot-throughput-speedup-analysis"
+ filter: "'NIC_Intel-X520-DA2' and '78B' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "78B-*-ethip6-ip6(base|scale)*pdrdisc"
+ layout:
+ "plot-throughput-speedup-analysis"
+
# Plot packets per second
# VPP L2 sel1
Code Review / csit.git / blobdiff