DIR[DTR,FUNC,HC]: "{DIR[DTR]}/honeycomb_functional_results"
DIR[DTR,FUNC,NSHSFC]: "{DIR[DTR]}/nshsfc_functional_results"
DIR[DTR,PERF,VPP,IMPRV]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_improvements"
- DIR[DTR,PERF,VPP,IMPACT,SPECTRE]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_impact_spectre"
- DIR[DTR,PERF,VPP,IMPACT,MELTDOWN]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_improvements"
# Detailed test configurations
DIR[DTC]: "{DIR[WORKING,SRC]}/test_configuration"
-
type: "configuration"
data-sets:
+# TODO: Add the data sources
+ plot-vpp-http-server-performance:
+ csit-vpp-perf-1801-all:
+ - 1
+ - 2
+ - 3
+ - 4
+# TODO: Add the data sources
vpp-meltdown-impact:
-# TODO: specify data sources
-# csit-vpp-perf-1801-all:
-# - 1
-# - 2
- plot-throughput-speedup-analysis:
+ csit-vpp-perf-1707-all:
+ - 9
+ - 10
+ - 13
+ csit-vpp-perf-1710-all:
+ - 11
+ - 12
+ - 13
# TODO: Add the data sources
-# csit-vpp-perf-1801-all:
-# - 1
-# - 2
vpp-spectre-impact:
-# TODO: specify data sources
-# csit-vpp-perf-1801-all:
-# - 1
-# - 2
- plot-throughput-speedup-analysis:
+ csit-vpp-perf-1707-all:
+ - 9
+ - 10
+ - 13
+ csit-vpp-perf-1710-all:
+ - 11
+ - 12
+ - 13
# TODO: Add the data sources
-# csit-vpp-perf-1801-all:
-# - 1
-# - 2
+ plot-throughput-speedup-analysis:
+ csit-vpp-perf-1801-all:
+ - 13
+ - 14
+ - 21
performance-improvements:
csit-vpp-perf-1707-all:
- 9
- 18
- 19
- 20
+# TODO: Add the data sources
+ csit-vpp-perf-1801-all:
+ - 13
+ - 14
+ - 21
csit-ligato-perf-1710-all:
- 5
- 7
- 13
- 16
- 17
+ csit-ligato-perf-1801-all:
+ - 1
+ - 3
+ - 7
+# TODO:
vpp-perf-results:
- csit-vpp-perf-1710-all:
- - 20
- - 23
+ csit-vpp-perf-1801-all:
+ - 12
vpp-func-results:
- csit-vpp-functional-1710-ubuntu1604-virl:
- - 126
+ csit-vpp-functional-1801-ubuntu1604-virl:
+ - "lastSuccessfulBuild"
+# TODO:
ligato-perf-results:
- csit-ligato-perf-1710-all:
- - 17
+ csit-ligato-perf-1801-all:
+ - 1
dpdk-perf-results:
- csit-dpdk-perf-1710-all:
- - 13
+ csit-dpdk-perf-1801-all:
+ - 12
hc-func-results:
- csit-hc2vpp-verify-func-1710-ubuntu1604:
+ csit-hc2vpp-verify-func-1801-ubuntu1604:
- "lastSuccessfulBuild"
+# TODO: Will we use this one?
+# hc2vpp-csit-integration-odl-carbon-1801-ubuntu1604:
+# - lastSuccessfulBuild
nsh-func-results:
- csit-nsh_sfc-verify-func-1710-ubuntu1604-virl:
- - 7
+ csit-nsh_sfc-verify-func-1801-ubuntu1604-virl:
+ - 1
+# TODO:
plot-vpp-throughput-latency:
- csit-vpp-perf-1710-all:
- - 11
- - 12
+ csit-vpp-perf-1801-all:
- 13
- 14
- - 15
- - 16
- - 17
- - 18
- - 19
- - 20
+ - 21
plot-dpdk-throughput-latency:
- csit-dpdk-perf-1710-all:
- - 2
+ csit-dpdk-perf-1801-all:
+ - 1
- 3
+ - 4
- 5
- 6
- 7
- 8
- 9
- 10
- - 11
- - 13
+ - 12
+# TODO:
plot-ligato-throughput-latency:
- csit-ligato-perf-1710-all:
- - 5
+ csit-ligato-perf-1801-all:
+ - 1
+ - 3
- 7
- - 8
- - 9
- - 10
- - 11
- - 12
- - 13
- - 16
- - 17
+
plot-layouts:
+
+ plot-cps:
+ xaxis:
+ autorange: True
+ autotick: False
+ fixedrange: False
+ gridcolor: "rgb(238, 238, 238)"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ tickmode: "linear"
+ title: "Indexed Test Cases"
+ zeroline: False
+ yaxis:
+ gridcolor: "rgb(238, 238, 238)'"
+ hoverformat: ".4s"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ range: []
+ rangemode: "tozero"
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ title: "Connections Per Second [cps]"
+ zeroline: False
+ boxmode: "group"
+ boxgroupgap: 0.5
+ autosize: False
+ margin:
+ t: 50
+ b: 20
+ l: 50
+ r: 20
+ showlegend: True
+ legend:
+ orientation: "h"
+ width: 700
+ height: 1000
+
+ plot-rps:
+ xaxis:
+ autorange: True
+ autotick: False
+ fixedrange: False
+ gridcolor: "rgb(238, 238, 238)"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ tickmode: "linear"
+ title: "Indexed Test Cases"
+ zeroline: False
+ yaxis:
+ gridcolor: "rgb(238, 238, 238)'"
+ hoverformat: ".4s"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ range: []
+ rangemode: "tozero"
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ title: "Requests Per Second [rps]"
+ zeroline: False
+ boxmode: "group"
+ boxgroupgap: 0.5
+ autosize: False
+ margin:
+ t: 50
+ b: 20
+ l: 50
+ r: 20
+ showlegend: True
+ legend:
+ orientation: "h"
+ width: 700
+ height: 1000
+
plot-throughput:
xaxis:
autorange: True
orientation: "h"
width: 700
height: 1000
+
plot-latency:
xaxis:
autorange: True
orientation: "h"
width: 700
height: 1000
+
plot-throughput-speedup-analysis:
xaxis:
- # title: "Test Cases"
autorange: True
autotick: False
fixedrange: False
-
build: 2
file: "{DIR[WORKING,DATA]}/output.xml"
-# csit-dpdk-perf-1707-all:
-# -
-# build: 10
-# file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__10.xml"
-# -
-# build: 9
-# file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__9.xml"
-# csit-nsh_sfc-verify-func-1707-ubuntu1604-virl:
-# -
-# build: 2
-# file: "{DIR[WORKING,DATA]}/csit-nsh_sfc-verify-func-1707-ubuntu1604-virl-2.xml"
-# csit-vpp-functional-1707-ubuntu1604-virl:
-# -
-# build: lastSuccessfulBuild
-# file: "{DIR[WORKING,DATA]}/csit-vpp-functional-1707-ubuntu1604-virl-lastSuccessfulBuild.xml"
-# hc2vpp-csit-integration-1707-ubuntu1604:
-# -
-# build: lastSuccessfulBuild
-# file: "{DIR[WORKING,DATA]}/hc2vpp-csit-integration-1707-ubuntu1604-lastSuccessfulBuild.xml"
-# csit-vpp-perf-1707-all:
-# -
-# build: 16
-# file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__16__output.xml"
-# -
-# build: 17
-# file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__17__output.xml"
+ -
+ build: 3
+ file: "{DIR[WORKING,DATA]}/output.xml"
+ -
+ build: 4
+ file: "{DIR[WORKING,DATA]}/output.xml"
-
type: "static"
- 19
- 20
- 23
- csit-dpdk-perf-1710-all:
- - 2
- - 3
+# TODO:
+ csit-vpp-perf-1801-all:
+ - 12 # full
+ - 13 # sel
+ - 14 # sel
+ - 21 # sel
+ csit-ligato-perf-1710-all:
- 5
- - 6
- 7
- 8
- 9
- 10
- 11
+ - 12
- 13
- csit-ligato-perf-1710-all:
+ - 16
+ - 17
+# TODO:
+ csit-ligato-perf-1801-all:
+ - 1
+ - 3
+ - 7
+ csit-dpdk-perf-1801-all:
+ - 1
+ - 3
+ - 4
- 5
+ - 6
- 7
- 8
- 9
- 10
- - 11
- 12
- - 13
- - 16
- - 17
- csit-vpp-functional-1710-ubuntu1604-virl:
- - 126
- csit-nsh_sfc-verify-func-1710-ubuntu1604-virl:
- - 7
- csit-hc2vpp-verify-func-1710-ubuntu1604:
+ csit-vpp-functional-1801-ubuntu1604-virl:
+ - lastSuccessfulBuild
+ csit-nsh_sfc-verify-func-1801-ubuntu1604-virl:
+ - 1
+ csit-hc2vpp-verify-func-1801-ubuntu1604:
- lastSuccessfulBuild
+# TODO: Will we use this one?
+# hc2vpp-csit-integration-odl-carbon-1801-ubuntu1604:
+# - lastSuccessfulBuild
-
type: "output"
algorithm: "table_performance_comparison"
output-file-ext: ".csv"
# TODO: specify dir
- output-file: "{DIR[DTR,PERF,VPP,IMPACT,MELTDOWN]}/meltdown-impact"
+ output-file: "{DIR[STATIC,VPP]}/meltdown-impact"
reference:
title: "No Meltdown"
# TODO: specify data sources
-# data:
-# csit-vpp-perf-1801-all:
-# - 1
-# - 2
+ data:
+ csit-vpp-perf-1707-all:
+ - 9
+ - 10
+ - 13
compare:
title: "Meltdown Patches Applied"
# TODO: specify data sources
-# data:
-# csit-vpp-perf-1801-all:
-# - 1
-# - 2
+ data:
+ csit-vpp-perf-1710-all:
+ - 11
+ - 12
+ - 13
data:
"vpp-meltdown-impact"
filter: "all"
algorithm: "table_performance_comparison"
output-file-ext: ".csv"
# TODO: specify dir
- output-file: "{DIR[DTR,PERF,VPP,IMPACT,SPECTRE]}/spectre-impact"
+ output-file: "{DIR[STATIC,VPP]}/meltdown-spectre-impact"
reference:
title: "No Spectre"
# TODO: specify data sources
-# data:
-# csit-vpp-perf-1801-all:
-# - 1
-# - 2
+ data:
+ csit-vpp-perf-1707-all:
+ - 9
+ - 10
+ - 13
compare:
title: "Spectre Patches Applied"
# TODO: specify data sources
-# data:
-# csit-vpp-perf-1801-all:
-# - 1
-# - 2
+ data:
+ csit-vpp-perf-1710-all:
+ - 11
+ - 12
+ - 13
data:
"vpp-spectre-impact"
filter: "all"
output-file-ext: ".csv"
output-file: "{DIR[DTR,PERF,VPP,IMPRV]}/performance_improvements"
columns:
- -
- title: "Functionality"
- data: "template 1"
-
title: "Test Name"
- data: "template 2"
+ data: "template 1"
-
title: "16.09 mean [Mpps]"
- data: "template 3"
+ data: "template 2"
-
title: "17.01 mean [Mpps]"
- data: "template 4"
+ data: "template 3"
-
title: "17.04 mean [Mpps]"
- data: "template 5"
+ data: "template 4"
-
title: "17.07 mean [Mpps]"
data: "data csit-vpp-perf-1707-all mean"
title: "17.10 mean [Mpps]"
data: "data csit-vpp-perf-1710-all csit-ligato-perf-1710-all mean"
-
- title: "17.10 stdev [Mpps]"
- data: "data csit-vpp-perf-1710-all csit-ligato-perf-1710-all stdev"
+ title: "18.01 mean [Mpps]"
+ data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all mean"
+ -
+ title: "18.01 stdev [Mpps]"
+ data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all stdev"
-
- title: "17.07 to 17.10 change [%]"
+ title: "17.10 to 18.01 change [%]"
data: "operation relative_change 5 6"
rows: "generated"
data:
### P L O T S ###
################################################################################
+# Plots VPP HTTP Server Performance
+-
+ type: "plot"
+ title: "VPP HTTP Server Performance"
+ algorithm: "plot_http_server_performance_box"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/http-server-performance-cps"
+ data:
+ "plot-vpp-http-server-performance"
+ # Keep this formatting, the filter is enclosed with " (quotation mark) and
+ # each tag is enclosed with ' (apostrophe).
+ filter: "'HTTP' and 'TCP_CPS'"
+ parameters:
+ - "result"
+ - "name"
+ traces:
+ hoverinfo: "x+y"
+ boxpoints: "outliers"
+ whiskerwidth: 0
+ layout:
+ title: "VPP HTTP Server Performance"
+ layout:
+ "plot-cps"
+
+-
+ type: "plot"
+ title: "VPP HTTP Server Performance"
+ algorithm: "plot_http_server_performance_box"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/http-server-performance-rps"
+ data:
+ "plot-vpp-http-server-performance"
+ filter: "'HTTP' and 'TCP_RPS'"
+ parameters:
+ - "result"
+ - "name"
+ traces:
+ hoverinfo: "x+y"
+ boxpoints: "outliers"
+ whiskerwidth: 0
+ layout:
+ title: "VPP HTTP Server Performance"
+ layout:
+ "plot-rps"
+
# Plot Throughput Speedup Analysis
# L2 - 10ge2p1x520 - NDR
"plot-vpp-throughput-latency"
# Keep this formatting, the filter is enclosed with " (quotation mark) and
# each tag is enclosed with ' (apostrophe).
- filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "throughput"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel1-ndrdisc"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "throughput"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-pdrdisc"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "throughput"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel1-pdrdisc"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and ('BASE' or 'SCALE') and 'PDRDISC' and not 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "throughput"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel2-ndrdisc"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and 'FEATURE' and ('ACL10' or 'ACL50') and '10k_FLOWS' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and 'FEATURE' and ('ACL10' or 'ACL50') and '10k_FLOWS' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "throughput"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel2-ndrdisc"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and 'FEATURE' and ('ACL10' or 'ACL50') and '10k_FLOWS' and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and 'FEATURE' and ('ACL10' or 'ACL50') and '10k_FLOWS' and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "throughput"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel2-pdrdisc"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and 'FEATURE' and ('ACL10' or 'ACL50') and '10k_FLOWS' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and 'FEATURE' and ('ACL10' or 'ACL50') and '10k_FLOWS' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "throughput"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel2-pdrdisc"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and 'FEATURE' and ('ACL10' or 'ACL50') and '10k_FLOWS' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and 'FEATURE' and ('ACL10' or 'ACL50') and '10k_FLOWS' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "throughput"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-ndrdisc-lat50"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "latency"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel1-ndrdisc-lat50"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "latency"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel2-ndrdisc-lat50"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and ('FEATURE' and 'ACL50' and '10k_FLOWS') and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and ('FEATURE' and 'ACL50' and '10k_FLOWS') and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "latency"
- "parent"
output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel2-ndrdisc-lat50"
data:
"plot-vpp-throughput-latency"
- filter: "'64B' and ('FEATURE' and 'ACL50' and '10k_FLOWS') and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ filter: "'64B' and ('FEATURE' and 'ACL50' and '10k_FLOWS') and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
parameters:
- "latency"
- "parent"