From 1911067935bdf6481fc9b89f40d79ca61f2448d0 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Thu, 20 Jul 2017 14:10:13 +0200 Subject: [PATCH] CSIT-726: Automate data collection for csv files for perf improvments Change-Id: Ia161ecf9af94fd7193bba045be56738b323e4e2e Signed-off-by: Tibor Frank --- .../vpp_performance_tests/csit_release_notes.rst | 8 +- .../performance_improvements/ndr_throughput.csv | 9 - .../ndr_throughput.template | 8 + .../ndr_throughput_others.csv | 16 -- .../ndr_throughput_others.template | 15 ++ .../performance_improvements/pdr_throughput.csv | 9 - .../pdr_throughput.template | 8 + .../pdr_throughput_others.csv | 15 -- .../pdr_throughput_others.template | 14 ++ .../tools/report_gen/run_improvments_tables.py | 212 +++++++++++++++++++++ resources/tools/report_gen/run_report.cfg | 1 + resources/tools/report_gen/run_report.sh | 14 ++ 12 files changed, 276 insertions(+), 53 deletions(-) delete mode 100644 docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv create mode 100644 docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template delete mode 100644 docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv create mode 100644 docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template delete mode 100644 docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv create mode 100644 docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template delete mode 100644 docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv create mode 100644 docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template create mode 100755 resources/tools/report_gen/run_improvments_tables.py diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst index 782fa42840..3daff961b4 100644 --- a/docs/report/vpp_performance_tests/csit_release_notes.rst +++ b/docs/report/vpp_performance_tests/csit_release_notes.rst @@ -101,7 +101,7 @@ Non-Drop Rate Throughput discovery tests: .. csv-table:: :align: center - :header-rows: 1 + :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change :file: ../../../docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv PDR Throughput @@ -111,7 +111,7 @@ Partial Drop Rate thoughput discovery tests with packet Loss Tolerance of 0.5%: .. csv-table:: :align: center - :header-rows: 1 + :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change :file: ../../../docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv Measured improvements are in line with VPP code optimizations listed in @@ -133,7 +133,7 @@ Non-Drop Rate Throughput discovery tests: .. csv-table:: :align: center - :header-rows: 1 + :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change :file: ../../../docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv PDR Throughput @@ -143,7 +143,7 @@ Partial Drop Rate thoughput discovery tests with packet Loss Tolerance of 0.5%: .. csv-table:: :align: center - :header-rows: 1 + :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change :file: ../../../docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv deleted file mode 100644 index 1f3a526ae1..0000000000 --- a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv +++ /dev/null @@ -1,9 +0,0 @@ -VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 [Mpps],VPP-17.07 mean value [Mpps],VPP-17.07 standard deviation [Mpps],17.04 to 17.07 Relative Change -L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,0.5,2.8,3.4,3.4,0.0,0% -L2BD-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,0.4,2.7,3.1,3.2,0.0,3% -IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhost-1vm-ndrdisc,0.3,2.6,3.0,0.0,0.0,0% -IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-ndrdisc,4.4,4.8,5.5,5.7,0.0,4% -IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-ndrdisc,3.0,7.3,8.1,0.0,0.0,0% -IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,6.1,6.1,6.9,0.0,0.0,0% -IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,6.5,6.1,6.9,0.0,0.0,0% -IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-ndrdisc,5.3,4.2,4.6,0.0,0.0,0% diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template new file mode 100644 index 0000000000..a7998a5b12 --- /dev/null +++ b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template @@ -0,0 +1,8 @@ +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-ndrdisc,IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-ndrdisc,3.0,7.3,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,6.1,6.1,6.9 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-iacldstbase-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,6.5,6.1,6.9 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale2m-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale2m-ndrdisc,IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-ndrdisc,5.3,4.2,4.6 +tests.vpp.perf.l2.40ge2p1xl710-eth-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-ndrdisc,L2XC,10ge2p1xl710: 64B-1t1c-eth-l2xcbase-ndrdisc,9.5,12.2,12.4 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,7.1,8.3,9.0 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale2m-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale2m-ndrdisc,IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-ndrdisc,8.5,7.8,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale20k-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale20k-ndrdisc,IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-ndrdisc,6.9,6.5,6.9 diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv deleted file mode 100644 index ce8899f521..0000000000 --- a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv +++ /dev/null @@ -1,16 +0,0 @@ -VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 [Mpps],VPP-17.07 mean value [Mpps],VPP-17.07 standard deviation [Mpps],17.04 to 17.07 Relative Change -L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-ndrdisc,9.4,12.7,13.1,13.8,0.0,5% -L2XC,10ge2p1xl710: 64B-1t1c-eth-l2xcbase-ndrdisc,9.5,12.2,12.4,14.2,0.0,15% -L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-ndrdisc,7.4,8.8,9.3,9.7,0.0,4% -L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-ndrdisc,7.5,8.8,9.2,9.7,0.0,5% -L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-ndrdisc,5.4,6.5,6.8,7.1,0.0,4% -L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-ndrdisc,7.8,10.4,10.8,6.9,0.0,-36% -IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ndrdisc,8.7,9.7,10.6,11.3,0.0,7% -IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,7.1,8.3,9.0,9.9,0.0,10% -IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,6.9,7.6,8.3,8.8,0.0,6% -IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-ndrdisc,8.5,9.0,9.7,10.4,0.0,7% -IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-ndrdisc,8.5,9.0,9.7,10.4,0.0,7% -IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-ndrdisc,8.5,7.8,8.1,8.8,0.0,9% -IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,6.9,7.4,8.1,8.5,0.0,5% -IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-ndrdisc,6.5,5.3,5.3,0.0,0.0,0% -IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-ndrdisc,6.9,6.5,6.9,0.0,0.0,0% diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template new file mode 100644 index 0000000000..f60a56ad15 --- /dev/null +++ b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template @@ -0,0 +1,15 @@ +tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-ndrdisc,L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-ndrdisc,9.4,12.7,13.3 +tests.vpp.perf.l2.10ge2p1x520-dot1ad-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-dot1ad-l2xcbase-ndrdisc,L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-ndrdisc,7.4,8.8,9.3 +tests.vpp.perf.l2.10ge2p1x520-dot1q-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-dot1q-l2xcbase-ndrdisc,L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-ndrdisc,7.5,8.8,9.2 +tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4vxlan-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-ethip4vxlan-l2xcbase-ndrdisc,L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-ndrdisc,5.4,6.5,6.8 +tests.vpp.perf.l2.10ge2p1x520-eth-l2bdbasemaclrn-ndrpdrdisc.tc01-64b-1t1c-eth-l2bdbasemaclrn-ndrdisc,L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-ndrdisc,7.8,10.4,10.8 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-ndrdisc,IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ndrdisc,8.7,9.7,10.6 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-iacldstbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,6.9,7.6,8.3 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale200k-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale200k-ndrdisc,IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-ndrdisc,8.5,9.0,9.7 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale20k-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale20k-ndrdisc,IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-ndrdisc,8.5,9.0,9.7 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,6.9,7.4,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale200k-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale200k-ndrdisc,IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-ndrdisc,6.5,5.3,5.3 +tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4lispip4-ip4base-ndrpdrdisc.tc01-64b-1t1c-ethip4lispip4-ip4base-ndrdisc,IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-ndrdisc,4.4,4.8,5.5 +tests.vpp.perf.vm vhost.10ge2p1x520-dot1q-l2xcbase-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,0.5,2.8,3.2 +tests.vpp.perf.vm vhost.10ge2p1x710-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,L2BD-vhost-VM,10ge2p1x710: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,0.4,2.7,3.2 +tests.vpp.perf.vm vhost.10ge2p1x520-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrdisc,IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrdisc,0.3,2.6,3.1 diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv deleted file mode 100644 index 5ef8324e99..0000000000 --- a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv +++ /dev/null @@ -1,9 +0,0 @@ -VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 [Mpps],VPP-17.07 mean value [Mpps],VPP-17.07 standard deviation [Mpps],17.04 to 17.07 Relative Change -L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,2.6,3.2,3.7,0.0,0.0,0% -L2BD-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,2.1,2.9,3.3,0.0,0.0,0% -IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhost-1vm-pdrdisc,2.0,2.7,3.0,0.0,0.0,0% -IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-pdrdisc,4.6,4.8,5.5,0.0,0.0,0% -IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-pdrdisc,7.7,7.3,8.1,0.0,0.0,0% -IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,6.1,6.1,6.9,0.0,0.0,0% -IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,6.5,6.1,6.9,0.0,0.0,0% -IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-pdrdisc,5.3,4.2,4.6,0.0,0.0,0% diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template new file mode 100644 index 0000000000..245b5dd619 --- /dev/null +++ b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template @@ -0,0 +1,8 @@ +tests.vpp.perf.vm vhost.10ge2p1x520-dot1q-l2xcbase-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,2.6,3.2,3.2 +tests.vpp.perf.vm vhost.10ge2p1x710-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,L2BD-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,2.1,2.9,3.2 +tests.vpp.perf.vm vhost.10ge2p1x520-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-pdrdisc,IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-pdrdisc,2.0,2.7,3.0 +tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4lispip4-ip4base-ndrpdrdisc.tc02-64b-1t1c-ethip4lispip4-ip4base-pdrdisc,IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-pdrdisc,4.6,4.8,5.5 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-pdrdisc,IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-pdrdisc,7.7,7.3,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,6.1,6.1,6.9 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-iacldstbase-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,6.5,6.1,6.9 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale2m-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale2m-pdrdisc,IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-pdrdisc,5.3,4.2,4.6 diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv deleted file mode 100644 index 21aff9455c..0000000000 --- a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv +++ /dev/null @@ -1,15 +0,0 @@ -VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 [Mpps],VPP-17.07 mean value [Mpps],VPP-17.07 standard deviation [Mpps],17.04 to 17.07 Relative Change -L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-pdrdisc,9.4,12.7,13.4,0.0,0.0,0% -L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-pdrdisc,7.4,8.8,9.3,0.0,0.0,0% -L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-pdrdisc,7.5,8.8,9.2,0.0,0.0,0% -L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-pdrdisc,5.4,6.5,6.8,0.0,0.0,0% -L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-pdrdisc,7.8,10.6,10.8,0.0,0.0,0% -IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-pdrdisc,8.7,9.7,10.6,0.0,0.0,0% -IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,7.1,8.3,9.2,0.0,0.0,0% -IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,7.1,7.6,8.3,0.0,0.0,0% -IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-pdrdisc,8.5,9.0,9.7,0.0,0.0,0% -IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-pdrdisc,8.5,9.0,9.7,0.0,0.0,0% -IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-pdrdisc,8.3,8.1,8.1,0.0,0.0,0% -IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,7.1,7.4,8.1,0.0,0.0,0% -IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-pdrdisc,6.9,5.3,5.3,0.0,0.0,0% -IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-pdrdisc,6.9,6.5,6.9,0.0,0.0,0% diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template new file mode 100644 index 0000000000..7905ee899b --- /dev/null +++ b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template @@ -0,0 +1,14 @@ +tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-pdrdisc,L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-pdrdisc,9.4,12.7,13.4 +tests.vpp.perf.l2.10ge2p1x520-dot1ad-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-dot1ad-l2xcbase-pdrdisc,L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-pdrdisc,7.4,8.8,9.3 +tests.vpp.perf.l2.10ge2p1x520-dot1q-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-dot1q-l2xcbase-pdrdisc,L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-pdrdisc,7.5,8.8,9.2 +tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4vxlan-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-ethip4vxlan-l2xcbase-pdrdisc,L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-pdrdisc,5.4,6.5,6.8 +tests.vpp.perf.l2.10ge2p1x520-eth-l2bdbasemaclrn-ndrpdrdisc.tc02-64b-1t1c-eth-l2bdbasemaclrn-pdrdisc,L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-pdrdisc,7.8,10.6,10.9 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-pdrdisc,IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-pdrdisc,8.7,9.7,10.6 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,7.1,8.3,9.2 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-iacldstbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,7.1,7.6,8.3 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale200k-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale200k-pdrdisc,IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-pdrdisc,8.5,9.0,9.7 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale20k-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale20k-pdrdisc,IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-pdrdisc,8.5,9.0,9.7 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale2m-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale2m-pdrdisc,IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-pdrdisc,8.3,8.1,8.3 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,7.1,7.4,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale200k-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale200k-pdrdisc,IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-pdrdisc,6.9,5.3,5.3 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale20k-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale20k-pdrdisc,IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-pdrdisc,6.9,6.5,6.9 diff --git a/resources/tools/report_gen/run_improvments_tables.py b/resources/tools/report_gen/run_improvments_tables.py new file mode 100755 index 0000000000..7bd6775a87 --- /dev/null +++ b/resources/tools/report_gen/run_improvments_tables.py @@ -0,0 +1,212 @@ +#!/usr/bin/python + +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate csv files for the chapter "CSIT Release Notes" from json files +generated by Jenkins' jobs. +""" + +from sys import exit as sys_exit +from os import walk +from os.path import join +from math import sqrt +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from json import load + + +EXT_JSON = ".json" +EXT_TMPL = ".template" +EXT_CSV = ".csv" + + +def get_files(path, extension): + """Generates the list of files to process. + + :param path: Path to files. + :param extension: Extension of files to process. If it is the empty string, + all files will be processed. + :type path: str + :type extension: str + :returns: List of files to process. + :rtype: list + """ + + file_list = list() + for root, _, files in walk(path): + for filename in files: + if extension: + if filename.endswith(extension): + file_list.append(join(root, filename)) + else: + file_list.append(join(root, filename)) + + return file_list + + +def parse_args(): + """Parse arguments from cmd line. + + :returns: Parsed arguments. + :rtype ArgumentParser + """ + + parser = ArgumentParser(description=__doc__, + formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-i", "--input", + required=True, + help="Input folder with data files.") + parser.add_argument("-o", "--output", + required=True, + help="Output folder with csv files and templates for " + "csv files.") + return parser.parse_args() + + +def calculate_stats(data): + """Calculate statistics: + - average, + - standard deviation. + + :param data: Data to process. + :type data: list + :returns: Average and standard deviation. + :rtype: tuple + """ + + if len(data) == 0: + return None, None + + def average(items): + """Calculate average from the items. + + :param items: Average is calculated from these items. + :type items: list + :returns: Average. + :rtype: float + """ + return float(sum(items)) / len(items) + + avg = average(data) + variance = [(x - avg) ** 2 for x in data] + stdev = sqrt(average(variance)) + + return avg, stdev + + +def write_line_to_file(file_handler, item): + """Write a line to the csv file. + + :param file_handler: File handler for the csv file. It must be open for + writing text. + :param item: Item to be written to the file. + :type file_handler: BinaryIO + :type item: dict + """ + + mean = "" if item["mean"] is None else "{:.1f}".format(item["mean"]) + stdev = "" if item["stdev"] is None else "{:.1f}".format(item["stdev"]) + change = "" if item["change"] is None else "{:.0f}%".format(item["change"]) + file_handler.write("{},{},{},{}\n".format(item["old"], mean, stdev, change)) + + +def main(): + """Main function to generate csv files for the chapter "CSIT Release Notes" + from json files generated by Jenkins' jobs. + """ + + args = parse_args() + + json_files = get_files(args.input, EXT_JSON) + tmpl_files = get_files(args.output, EXT_TMPL) + + if len(json_files) == 0: + print("No json data to process.") + exit(1) + + if len(tmpl_files) == 0: + print("No template files to process.") + exit(1) + + # Get information from template files + csv_data = list() + for tmpl_file in tmpl_files: + with open(tmpl_file, mode='r') as file_handler: + for line in file_handler: + line_list = line.split(',') + try: + csv_data.append({ + "ID": line_list[0], + "type": line_list[0].rsplit("-", 1)[-1], + "old": ",".join(line_list[1:])[:-1], + "last_old": line_list[-1][:-1], + "rates": list(), + "mean": None, + "stdev": None, + "change": None}) + except IndexError: + pass + + # Update existing data with the new information from json files + for json_file in json_files: + with open(json_file) as file_handler: + tests_data = load(file_handler) + for item in csv_data: + try: + rate = tests_data["data"][item["ID"]]["throughput"]["value"] + item["rates"].append(rate) + except KeyError: + pass + + # Add statistics + for item in csv_data: + mean, stdev = calculate_stats(item["rates"]) + if mean is not None: + mean = mean / 1000000 + old = float(item["last_old"]) + item["mean"] = mean + item["change"] = ((mean - old) / old) * 100 + item["stdev"] = stdev / 1000000 + + # Sort the list, key = change + csv_data.sort(key=lambda data: data["change"], reverse=True) + + # Write csv files + for tmpl_file in tmpl_files: + csv_file = tmpl_file.replace(EXT_TMPL, EXT_CSV) + with open(csv_file, "w") as file_handler: + for item in csv_data: + if "pdr_" in csv_file \ + and "_others" not in csv_file \ + and item["type"] == "pdrdisc" \ + and item["change"] >= 9.5: + write_line_to_file(file_handler, item) + elif "pdr_" in csv_file \ + and "_others" in csv_file \ + and item["type"] == "pdrdisc" \ + and item["change"] < 9.5: + write_line_to_file(file_handler, item) + elif "ndr_" in csv_file \ + and "_others" not in csv_file \ + and item["type"] == "ndrdisc" \ + and item["change"] >= 9.5: + write_line_to_file(file_handler, item) + elif "ndr_" in csv_file \ + and "_others" in csv_file \ + and item["type"] == "ndrdisc" \ + and item["change"] < 9.5: + write_line_to_file(file_handler, item) + + +if __name__ == "__main__": + sys_exit(main()) diff --git a/resources/tools/report_gen/run_report.cfg b/resources/tools/report_gen/run_report.cfg index 70eb411fe9..642a3e3435 100644 --- a/resources/tools/report_gen/run_report.cfg +++ b/resources/tools/report_gen/run_report.cfg @@ -28,6 +28,7 @@ DIR[DTR,PERF,HC]=${DIR[DTR]}/honeycomb_performance_results DIR[DTR,FUNC,VPP]=${DIR[DTR]}/vpp_functional_results DIR[DTR,FUNC,HC]=${DIR[DTR]}/honeycomb_functional_results DIR[DTR,FUNC,NSHSFC]=${DIR[DTR]}/nshsfc_functional_results +DIR[DTR,PERF,VPP,IMPRV]=${DIR[RST]}/vpp_performance_tests/performance_improvements DIR[DTC]=${DIR[RST]}/test_configuration DIR[DTC,PERF,VPP]=${DIR[DTC]}/vpp_performance_configuration diff --git a/resources/tools/report_gen/run_report.sh b/resources/tools/report_gen/run_report.sh index 4c104928f3..db079773c8 100755 --- a/resources/tools/report_gen/run_report.sh +++ b/resources/tools/report_gen/run_report.sh @@ -143,6 +143,15 @@ then --output ${DIR[DTO,PERF,VPP]}/vpp_performance_operational_data.rst \ --data "SH_RUN" --formatting rst --start 4 --level 2 + blds=${JOB[PERF,VPP,BLD]} + for i in ${blds[@]}; do + unzip -o ${DIR[STATIC,ARCH]}/${JOB[PERF,VPP]}-${i}.zip -d ${DIR[WORKING]}/ + ./run_robot_json_data.py \ + --input ${DIR[WORKING]}/output.xml \ + --output ${DIR[DTR,PERF,VPP,IMPRV]}/${JOB[PERF,VPP]}-${i}.json \ + --vdevice ${i} + done + # DPDK PERF unzip -o ${DIR[STATIC,ARCH]}/${JOB[PERF,DPDK]}-${JOB[PERF,DPDK,FBLD]}.zip -d ${DIR[WORKING]}/ python run_robot_data.py -i ${DIR[WORKING]}/robot-plugin/output.xml \ @@ -171,6 +180,11 @@ then --formatting rst --start 5 --level 2 fi +# Generate tables for performance improvements +./run_improvments_tables.py \ + --input ${DIR[DTR,PERF,VPP,IMPRV]} \ + --output ${DIR[DTR,PERF,VPP,IMPRV]} + # Delete temporary json files find ${DIR[RST]} -name "*.json" -type f -delete -- 2.16.6