X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fspecification.yaml;h=26ce63f10acbbff876d557307880245adb74714d;hp=e8884623a1c147de2e186248780c403f02ce2256;hb=3b4e1e9b779fd3865714681cc224226b5804c546;hpb=1701c0a9db9cc57423e5d1a9009788d0a707ecd6 diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index e8884623a1..26ce63f10a 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -61,6 +61,8 @@ urls: URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job" + URL[NEXUS]: "https://docs.fd.io/csit" + DIR[NEXUS]: "report/_static/archive" make-dirs: # List the directories which are created while preparing the environment. @@ -87,40 +89,34 @@ - type: "configuration" data-sets: - vpp-meltdown-impact: - csit-vpp-perf-1707-all: - - 9 - - 10 - - 13 - csit-vpp-perf-1710-all: - - 11 - - 12 - - 13 - vpp-spectre-impact: - csit-vpp-perf-1707-all: - - 9 - - 10 - - 13 - csit-vpp-perf-1710-all: - - 11 - - 12 - - 13 # TODO: Add the data sources - plot-throughput-speedup-analysis: - csit-vpp-perf-1801-all: - - 12 - performance-improvements: - csit-vpp-perf-1707-all: - - 9 - - 10 - - 13 - - 14 - - 15 - - 16 - - 17 - - 18 - - 19 - - 21 +# plot-vpp-http-server-performance: +# csit-vpp-perf-1801-all: +# - 1 +# - 2 +# - 3 +# - 4 +# TODO: Add the data sources +# vpp-meltdown-impact: +# csit-vpp-perf-1707-all: +# - 9 +# - 10 +# - 13 +# csit-vpp-perf-1710-all: +# - 11 +# - 12 +# - 13 +# TODO: Add the data sources +# vpp-spectre-impact: +# csit-vpp-perf-1707-all: +# - 9 +# - 10 +# - 13 +# csit-vpp-perf-1710-all: +# - 11 +# - 12 +# - 13 + vpp-performance-changes: csit-vpp-perf-1710-all: - 11 - 12 @@ -132,33 +128,70 @@ - 18 - 19 - 20 -# TODO: +# TODO: Add the data sources csit-vpp-perf-1801-all: - - 13 - - 14 - csit-ligato-perf-1710-all: - - 5 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 16 - - 17 - csit-ligato-perf-1801-all: - - 1 - - 3 - - 7 -# TODO: + - 124 + - 127 + - 128 +# TODO: Add the data sources + plot-throughput-speedup-analysis: + csit-vpp-perf-1801-all: + - 122 + - 126 + - 129 +# performance-improvements: +# csit-vpp-perf-1707-all: +# - 9 +# - 10 +# - 13 +# - 14 +# - 15 +# - 16 +# - 17 +# - 18 +# - 19 +# - 21 +# csit-vpp-perf-1710-all: +# - 11 +# - 12 +# - 13 +# - 14 +# - 15 +# - 16 +# - 17 +# - 18 +# - 19 +# - 20 +## TODO: Add the data sources +# csit-vpp-perf-1801-all: +# - 124 +# - 127 +# - 128 +# csit-ligato-perf-1710-all: +# - 5 +# - 7 +# - 8 +# - 9 +# - 10 +# - 11 +# - 12 +# - 13 +# - 16 +# - 17 +# csit-ligato-perf-1801-all: +# - 1 +# - 3 +# - 7 +# TODO: Add the data sources vpp-perf-results: csit-vpp-perf-1801-all: - - 12 + - 122 + - 126 + - 129 vpp-func-results: csit-vpp-functional-1801-ubuntu1604-virl: - "lastSuccessfulBuild" -# TODO: +# TODO: Add the data sources ligato-perf-results: csit-ligato-perf-1801-all: - 1 @@ -168,17 +201,15 @@ hc-func-results: csit-hc2vpp-verify-func-1801-ubuntu1604: - "lastSuccessfulBuild" -# TODO: Will we use this one? -# hc2vpp-csit-integration-odl-carbon-1801-ubuntu1604: -# - lastSuccessfulBuild nsh-func-results: csit-nsh_sfc-verify-func-1801-ubuntu1604-virl: - 1 -# TODO: +# TODO: Add the data sources plot-vpp-throughput-latency: csit-vpp-perf-1801-all: - - 13 - - 14 + - 124 + - 127 + - 128 plot-dpdk-throughput-latency: csit-dpdk-perf-1801-all: - 1 @@ -188,16 +219,101 @@ - 6 - 7 - 8 - - 9 - 10 - 12 -# TODO: +# TODO: Add the data sources plot-ligato-throughput-latency: csit-ligato-perf-1801-all: - 1 - 3 - 7 + plot-layouts: + + plot-cps: + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + rangemode: "tozero" + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Connections Per Second [cps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + + plot-rps: + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + rangemode: "tozero" + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Requests Per Second [rps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + plot-throughput: xaxis: autorange: True @@ -238,6 +354,7 @@ orientation: "h" width: 700 height: 1000 + plot-latency: xaxis: autorange: True @@ -278,6 +395,7 @@ orientation: "h" width: 700 height: 1000 + plot-throughput-speedup-analysis: xaxis: autorange: True @@ -340,6 +458,12 @@ - build: 2 file: "{DIR[WORKING,DATA]}/output.xml" + - + build: 3 + file: "{DIR[WORKING,DATA]}/output.xml" + - + build: 4 + file: "{DIR[WORKING,DATA]}/output.xml" - type: "static" @@ -354,17 +478,17 @@ download-path: "{job}/{build}/robot/report/*zip*/{filename}" extract: "robot-plugin/output.xml" builds: - csit-vpp-perf-1707-all: - - 9 - - 10 - - 13 - - 14 - - 15 - - 16 - - 17 - - 18 - - 19 - - 21 +# csit-vpp-perf-1707-all: +# - 9 +# - 10 +# - 13 +# - 14 +# - 15 +# - 16 +# - 17 +# - 18 +# - 19 +# - 21 csit-vpp-perf-1710-all: - 11 - 12 @@ -376,12 +500,14 @@ - 18 - 19 - 20 - - 23 # TODO: csit-vpp-perf-1801-all: - - 12 - - 13 - - 14 + - 122 # full + - 126 # full + - 129 # full + - 124 # sel + - 127 # sel + - 128 # sel csit-ligato-perf-1710-all: - 5 - 7 @@ -393,6 +519,16 @@ - 13 - 16 - 17 + csit-dpdk-perf-1801-all: + - 1 + - 4 + - 5 + - 7 + - 8 + - 10 + - 12 + - 16 + - 17 # TODO: csit-ligato-perf-1801-all: - 1 @@ -415,9 +551,6 @@ - 1 csit-hc2vpp-verify-func-1801-ubuntu1604: - lastSuccessfulBuild -# TODO: Will we use this one? -# hc2vpp-csit-integration-odl-carbon-1801-ubuntu1604: -# - lastSuccessfulBuild - type: "output" @@ -431,65 +564,105 @@ ### T A B L E S ### ################################################################################ +#- +# type: "table" +# title: "Performance Impact of Meltdown Patches" +# algorithm: "table_performance_comparison" +# output-file-ext: ".csv" +## TODO: specify dir +# output-file: "{DIR[STATIC,VPP]}/meltdown-impact" +# reference: +# title: "No Meltdown" +## TODO: specify data sources +# data: +# csit-vpp-perf-1707-all: +# - 9 +# - 10 +# - 13 +# compare: +# title: "Meltdown Patches Applied" +## TODO: specify data sources +# data: +# csit-vpp-perf-1710-all: +# - 11 +# - 12 +# - 13 +# data: +# "vpp-meltdown-impact" +# filter: "all" +# parameters: +# - "name" +# - "parent" +# - "throughput" +# # Number of the best and the worst tests presented in the table. Use 0 (zero) +# # to present all tests. +# nr-of-tests-shown: 20 +# +#- +# type: "table" +# title: "Performance Impact of Spectre Patches" +# algorithm: "table_performance_comparison" +# output-file-ext: ".csv" +## TODO: specify dir +# output-file: "{DIR[STATIC,VPP]}/meltdown-spectre-impact" +# reference: +# title: "No Spectre" +## TODO: specify data sources +# data: +# csit-vpp-perf-1707-all: +# - 9 +# - 10 +# - 13 +# compare: +# title: "Spectre Patches Applied" +## TODO: specify data sources +# data: +# csit-vpp-perf-1710-all: +# - 11 +# - 12 +# - 13 +# data: +# "vpp-spectre-impact" +# filter: "all" +# parameters: +# - "name" +# - "parent" +# - "throughput" +# # Number of the best and the worst tests presented in the table. Use 0 (zero) +# # to present all tests. +# nr-of-tests-shown: 20 + - type: "table" - title: "Performance Impact of Meltdown Patches" + title: "VPP Performance Changes" algorithm: "table_performance_comparison" output-file-ext: ".csv" # TODO: specify dir - output-file: "{DIR[STATIC,VPP]}/meltdown-impact" + output-file: "{DIR[STATIC,VPP]}/performance-changes" reference: - title: "No Meltdown" -# TODO: specify data sources - data: - csit-vpp-perf-1707-all: - - 9 - - 10 - - 13 - compare: - title: "Meltdown Patches Applied" + title: "Release 1710" # TODO: specify data sources data: csit-vpp-perf-1710-all: - 11 - 12 - 13 - data: - "vpp-meltdown-impact" - filter: "all" - parameters: - - "name" - - "parent" - - "throughput" - # Number of the best and the worst tests presented in the table. Use 0 (zero) - # to present all tests. - nr-of-tests-shown: 20 - -- - type: "table" - title: "Performance Impact of Spectre Patches" - algorithm: "table_performance_comparison" - output-file-ext: ".csv" -# TODO: specify dir - output-file: "{DIR[STATIC,VPP]}/meltdown-spectre-impact" - reference: - title: "No Spectre" -# TODO: specify data sources - data: - csit-vpp-perf-1707-all: - - 9 - - 10 - - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 compare: - title: "Spectre Patches Applied" + title: "Release 1801" # TODO: specify data sources data: - csit-vpp-perf-1710-all: - - 11 - - 12 - - 13 - data: - "vpp-spectre-impact" + csit-vpp-perf-1801-all: + - 124 + - 127 + - 128 + data: "vpp-performance-changes" filter: "all" parameters: - "name" @@ -499,50 +672,47 @@ # to present all tests. nr-of-tests-shown: 20 -- - type: "table" - title: "Performance improvements" - algorithm: "table_performance_improvements" - template: "{DIR[DTR,PERF,VPP,IMPRV]}/tmpl_performance_improvements.csv" - output-file-ext: ".csv" - output-file: "{DIR[DTR,PERF,VPP,IMPRV]}/performance_improvements" - columns: - - - title: "Functionality" - data: "template 1" - - - title: "Test Name" - data: "template 2" - - - title: "16.09 mean [Mpps]" - data: "template 3" - - - title: "17.01 mean [Mpps]" - data: "template 4" - - - title: "17.04 mean [Mpps]" - data: "template 5" - - - title: "17.07 mean [Mpps]" - data: "data csit-vpp-perf-1707-all mean" - - - title: "17.10 mean [Mpps]" - data: "data csit-vpp-perf-1710-all csit-ligato-perf-1710-all mean" - - - title: "18.01 mean [Mpps]" - data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all mean" - - - title: "18.01 stdev [Mpps]" - data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all stdev" - - - title: "17.10 to 18.01 change [%]" - data: "operation relative_change 6 7" - rows: "generated" - data: - "performance-improvements" - filter: "template" - parameters: - - "throughput" +#- +# type: "table" +# title: "Performance improvements" +# algorithm: "table_performance_improvements" +# template: "{DIR[DTR,PERF,VPP,IMPRV]}/tmpl_performance_improvements.csv" +# output-file-ext: ".csv" +# output-file: "{DIR[DTR,PERF,VPP,IMPRV]}/performance_improvements" +# columns: +# - +# title: "Test Name" +# data: "template 1" +# - +# title: "16.09 mean [Mpps]" +# data: "template 2" +# - +# title: "17.01 mean [Mpps]" +# data: "template 3" +# - +# title: "17.04 mean [Mpps]" +# data: "template 4" +# - +# title: "17.07 mean [Mpps]" +# data: "data csit-vpp-perf-1707-all mean" +# - +# title: "17.10 mean [Mpps]" +# data: "data csit-vpp-perf-1710-all csit-ligato-perf-1710-all mean" +# - +# title: "18.01 mean [Mpps]" +# data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all mean" +# - +# title: "18.01 stdev [Mpps]" +# data: "data csit-vpp-perf-1801-all csit-ligato-perf-1801-all stdev" +# - +# title: "17.10 to 18.01 change [%]" +# data: "operation relative_change 5 6" +# rows: "generated" +# data: +# "performance-improvements" +# filter: "template" +# parameters: +# - "throughput" - type: "table" @@ -945,6 +1115,51 @@ ### P L O T S ### ################################################################################ +# Plots VPP HTTP Server Performance +#- +# type: "plot" +# title: "VPP HTTP Server Performance" +# algorithm: "plot_http_server_performance_box" +# output-file-type: ".html" +# output-file: "{DIR[STATIC,VPP]}/http-server-performance-cps" +# data: +# "plot-vpp-http-server-performance" +# # Keep this formatting, the filter is enclosed with " (quotation mark) and +# # each tag is enclosed with ' (apostrophe). +# filter: "'HTTP' and 'TCP_CPS'" +# parameters: +# - "result" +# - "name" +# traces: +# hoverinfo: "x+y" +# boxpoints: "outliers" +# whiskerwidth: 0 +# layout: +# title: "VPP HTTP Server Performance" +# layout: +# "plot-cps" +# +#- +# type: "plot" +# title: "VPP HTTP Server Performance" +# algorithm: "plot_http_server_performance_box" +# output-file-type: ".html" +# output-file: "{DIR[STATIC,VPP]}/http-server-performance-rps" +# data: +# "plot-vpp-http-server-performance" +# filter: "'HTTP' and 'TCP_RPS'" +# parameters: +# - "result" +# - "name" +# traces: +# hoverinfo: "x+y" +# boxpoints: "outliers" +# whiskerwidth: 0 +# layout: +# title: "VPP HTTP Server Performance" +# layout: +# "plot-rps" + # Plot Throughput Speedup Analysis # L2 - 10ge2p1x520 - NDR