CSIT-726: Automate data collection for csv files for perf improvments

Change-Id: Ia161ecf9af94fd7193bba045be56738b323e4e2e
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst
index 782fa42..3daff96 100644 (file)
--- a/docs/report/vpp_performance_tests/csit_release_notes.rst
+++ b/docs/report/vpp_performance_tests/csit_release_notes.rst
@@ -101,7 +101,7 @@ Non-Drop Rate Throughput discovery tests:
 \r
 .. csv-table::\r
     :align: center\r
-    :header-rows: 1\r
+    :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change\r
     :file: ../../../docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv\r
 \r
 PDR Throughput\r
@@ -111,7 +111,7 @@ Partial Drop Rate thoughput discovery tests with packet Loss Tolerance of 0.5%:
 \r
 .. csv-table::\r
     :align: center\r
-    :header-rows: 1\r
+    :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change\r
     :file: ../../../docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv\r
 \r
 Measured improvements are in line with VPP code optimizations listed in\r
@@ -133,7 +133,7 @@ Non-Drop Rate Throughput discovery tests:
 \r
 .. csv-table::\r
     :align: center\r
-    :header-rows: 1\r
+    :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change\r
     :file: ../../../docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv\r
 \r
 PDR Throughput\r
@@ -143,7 +143,7 @@ Partial Drop Rate thoughput discovery tests with packet Loss Tolerance of 0.5%:
 \r
 .. csv-table::\r
     :align: center\r
-    :header-rows: 1\r
+    :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change\r
     :file: ../../../docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv\r
 \r
 \r

diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv
deleted file mode 100644 (file)
index 1f3a526..0000000
--- a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv
+++ /dev/null
@@ -1,9 +0,0 @@
-VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 [Mpps],VPP-17.07 mean value [Mpps],VPP-17.07 standard deviation [Mpps],17.04 to 17.07 Relative Change
-L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,0.5,2.8,3.4,3.4,0.0,0%
-L2BD-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,0.4,2.7,3.1,3.2,0.0,3%
-IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhost-1vm-ndrdisc,0.3,2.6,3.0,0.0,0.0,0%
-IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-ndrdisc,4.4,4.8,5.5,5.7,0.0,4%
-IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-ndrdisc,3.0,7.3,8.1,0.0,0.0,0%
-IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,6.1,6.1,6.9,0.0,0.0,0%
-IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,6.5,6.1,6.9,0.0,0.0,0%
-IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-ndrdisc,5.3,4.2,4.6,0.0,0.0,0%

diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template
new file mode 100644 (file)
index 0000000..a7998a5
--- /dev/null
+++ b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template
@@ -0,0 +1,8 @@
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-ndrdisc,IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-ndrdisc,3.0,7.3,8.1
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,6.1,6.1,6.9
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-iacldstbase-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,6.5,6.1,6.9
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale2m-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale2m-ndrdisc,IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-ndrdisc,5.3,4.2,4.6
+tests.vpp.perf.l2.40ge2p1xl710-eth-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-ndrdisc,L2XC,10ge2p1xl710: 64B-1t1c-eth-l2xcbase-ndrdisc,9.5,12.2,12.4
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,7.1,8.3,9.0
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale2m-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale2m-ndrdisc,IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-ndrdisc,8.5,7.8,8.1
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale20k-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale20k-ndrdisc,IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-ndrdisc,6.9,6.5,6.9

diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv
deleted file mode 100644 (file)
index ce8899f..0000000
--- a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv
+++ /dev/null
@@ -1,16 +0,0 @@
-VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 [Mpps],VPP-17.07 mean value [Mpps],VPP-17.07 standard deviation [Mpps],17.04 to 17.07 Relative Change
-L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-ndrdisc,9.4,12.7,13.1,13.8,0.0,5%
-L2XC,10ge2p1xl710: 64B-1t1c-eth-l2xcbase-ndrdisc,9.5,12.2,12.4,14.2,0.0,15%
-L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-ndrdisc,7.4,8.8,9.3,9.7,0.0,4%
-L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-ndrdisc,7.5,8.8,9.2,9.7,0.0,5%
-L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-ndrdisc,5.4,6.5,6.8,7.1,0.0,4%
-L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-ndrdisc,7.8,10.4,10.8,6.9,0.0,-36%
-IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ndrdisc,8.7,9.7,10.6,11.3,0.0,7%
-IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,7.1,8.3,9.0,9.9,0.0,10%
-IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,6.9,7.6,8.3,8.8,0.0,6%
-IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-ndrdisc,8.5,9.0,9.7,10.4,0.0,7%
-IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-ndrdisc,8.5,9.0,9.7,10.4,0.0,7%
-IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-ndrdisc,8.5,7.8,8.1,8.8,0.0,9%
-IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,6.9,7.4,8.1,8.5,0.0,5%
-IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-ndrdisc,6.5,5.3,5.3,0.0,0.0,0%
-IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-ndrdisc,6.9,6.5,6.9,0.0,0.0,0%

diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template
new file mode 100644 (file)
index 0000000..f60a56a
--- /dev/null
+++ b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template
@@ -0,0 +1,15 @@
+tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-ndrdisc,L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-ndrdisc,9.4,12.7,13.3
+tests.vpp.perf.l2.10ge2p1x520-dot1ad-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-dot1ad-l2xcbase-ndrdisc,L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-ndrdisc,7.4,8.8,9.3
+tests.vpp.perf.l2.10ge2p1x520-dot1q-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-dot1q-l2xcbase-ndrdisc,L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-ndrdisc,7.5,8.8,9.2
+tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4vxlan-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-ethip4vxlan-l2xcbase-ndrdisc,L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-ndrdisc,5.4,6.5,6.8
+tests.vpp.perf.l2.10ge2p1x520-eth-l2bdbasemaclrn-ndrpdrdisc.tc01-64b-1t1c-eth-l2bdbasemaclrn-ndrdisc,L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-ndrdisc,7.8,10.4,10.8
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-ndrdisc,IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ndrdisc,8.7,9.7,10.6
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-iacldstbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,6.9,7.6,8.3
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale200k-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale200k-ndrdisc,IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-ndrdisc,8.5,9.0,9.7
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale20k-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale20k-ndrdisc,IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-ndrdisc,8.5,9.0,9.7
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,6.9,7.4,8.1
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale200k-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale200k-ndrdisc,IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-ndrdisc,6.5,5.3,5.3
+tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4lispip4-ip4base-ndrpdrdisc.tc01-64b-1t1c-ethip4lispip4-ip4base-ndrdisc,IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-ndrdisc,4.4,4.8,5.5
+tests.vpp.perf.vm vhost.10ge2p1x520-dot1q-l2xcbase-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,0.5,2.8,3.2
+tests.vpp.perf.vm vhost.10ge2p1x710-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,L2BD-vhost-VM,10ge2p1x710: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,0.4,2.7,3.2
+tests.vpp.perf.vm vhost.10ge2p1x520-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrdisc,IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrdisc,0.3,2.6,3.1

diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv
deleted file mode 100644 (file)
index 5ef8324..0000000
--- a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv
+++ /dev/null
@@ -1,9 +0,0 @@
-VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 [Mpps],VPP-17.07 mean value [Mpps],VPP-17.07 standard deviation [Mpps],17.04 to 17.07 Relative Change
-L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,2.6,3.2,3.7,0.0,0.0,0%
-L2BD-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,2.1,2.9,3.3,0.0,0.0,0%
-IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhost-1vm-pdrdisc,2.0,2.7,3.0,0.0,0.0,0%
-IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-pdrdisc,4.6,4.8,5.5,0.0,0.0,0%
-IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-pdrdisc,7.7,7.3,8.1,0.0,0.0,0%
-IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,6.1,6.1,6.9,0.0,0.0,0%
-IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,6.5,6.1,6.9,0.0,0.0,0%
-IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-pdrdisc,5.3,4.2,4.6,0.0,0.0,0%

diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template
new file mode 100644 (file)
index 0000000..245b5dd
--- /dev/null
+++ b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template
@@ -0,0 +1,8 @@
+tests.vpp.perf.vm vhost.10ge2p1x520-dot1q-l2xcbase-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,2.6,3.2,3.2
+tests.vpp.perf.vm vhost.10ge2p1x710-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,L2BD-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,2.1,2.9,3.2
+tests.vpp.perf.vm vhost.10ge2p1x520-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-pdrdisc,IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-pdrdisc,2.0,2.7,3.0
+tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4lispip4-ip4base-ndrpdrdisc.tc02-64b-1t1c-ethip4lispip4-ip4base-pdrdisc,IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-pdrdisc,4.6,4.8,5.5
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-pdrdisc,IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-pdrdisc,7.7,7.3,8.1
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,6.1,6.1,6.9
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-iacldstbase-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,6.5,6.1,6.9
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale2m-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale2m-pdrdisc,IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-pdrdisc,5.3,4.2,4.6

diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv
deleted file mode 100644 (file)
index 21aff94..0000000
--- a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv
+++ /dev/null
@@ -1,15 +0,0 @@
-VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 [Mpps],VPP-17.07 mean value [Mpps],VPP-17.07 standard deviation [Mpps],17.04 to 17.07 Relative Change
-L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-pdrdisc,9.4,12.7,13.4,0.0,0.0,0%
-L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-pdrdisc,7.4,8.8,9.3,0.0,0.0,0%
-L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-pdrdisc,7.5,8.8,9.2,0.0,0.0,0%
-L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-pdrdisc,5.4,6.5,6.8,0.0,0.0,0%
-L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-pdrdisc,7.8,10.6,10.8,0.0,0.0,0%
-IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-pdrdisc,8.7,9.7,10.6,0.0,0.0,0%
-IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,7.1,8.3,9.2,0.0,0.0,0%
-IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,7.1,7.6,8.3,0.0,0.0,0%
-IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-pdrdisc,8.5,9.0,9.7,0.0,0.0,0%
-IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-pdrdisc,8.5,9.0,9.7,0.0,0.0,0%
-IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-pdrdisc,8.3,8.1,8.1,0.0,0.0,0%
-IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,7.1,7.4,8.1,0.0,0.0,0%
-IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-pdrdisc,6.9,5.3,5.3,0.0,0.0,0%
-IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-pdrdisc,6.9,6.5,6.9,0.0,0.0,0%

diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template
new file mode 100644 (file)
index 0000000..7905ee8
--- /dev/null
+++ b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template
@@ -0,0 +1,14 @@
+tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-pdrdisc,L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-pdrdisc,9.4,12.7,13.4
+tests.vpp.perf.l2.10ge2p1x520-dot1ad-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-dot1ad-l2xcbase-pdrdisc,L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-pdrdisc,7.4,8.8,9.3
+tests.vpp.perf.l2.10ge2p1x520-dot1q-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-dot1q-l2xcbase-pdrdisc,L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-pdrdisc,7.5,8.8,9.2
+tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4vxlan-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-ethip4vxlan-l2xcbase-pdrdisc,L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-pdrdisc,5.4,6.5,6.8
+tests.vpp.perf.l2.10ge2p1x520-eth-l2bdbasemaclrn-ndrpdrdisc.tc02-64b-1t1c-eth-l2bdbasemaclrn-pdrdisc,L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-pdrdisc,7.8,10.6,10.9
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-pdrdisc,IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-pdrdisc,8.7,9.7,10.6
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,7.1,8.3,9.2
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-iacldstbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,7.1,7.6,8.3
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale200k-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale200k-pdrdisc,IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-pdrdisc,8.5,9.0,9.7
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale20k-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale20k-pdrdisc,IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-pdrdisc,8.5,9.0,9.7
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale2m-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale2m-pdrdisc,IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-pdrdisc,8.3,8.1,8.3
+tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,7.1,7.4,8.1
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale200k-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale200k-pdrdisc,IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-pdrdisc,6.9,5.3,5.3
+tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale20k-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale20k-pdrdisc,IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-pdrdisc,6.9,6.5,6.9

diff --git a/resources/tools/report_gen/run_improvments_tables.py b/resources/tools/report_gen/run_improvments_tables.py
new file mode 100755 (executable)
index 0000000..7bd6775
--- /dev/null
+++ b/resources/tools/report_gen/run_improvments_tables.py
@@ -0,0 +1,212 @@
+#!/usr/bin/python
+
+# Copyright (c) 2017 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generate csv files for the chapter "CSIT Release Notes" from json files
+generated by Jenkins' jobs.
+"""
+
+from sys import exit as sys_exit
+from os import walk
+from os.path import join
+from math import sqrt
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+from json import load
+
+
+EXT_JSON = ".json"
+EXT_TMPL = ".template"
+EXT_CSV = ".csv"
+
+
+def get_files(path, extension):
+    """Generates the list of files to process.
+
+    :param path: Path to files.
+    :param extension: Extension of files to process. If it is the empty string,
+    all files will be processed.
+    :type path: str
+    :type extension: str
+    :returns: List of files to process.
+    :rtype: list
+    """
+
+    file_list = list()
+    for root, _, files in walk(path):
+        for filename in files:
+            if extension:
+                if filename.endswith(extension):
+                    file_list.append(join(root, filename))
+            else:
+                file_list.append(join(root, filename))
+
+    return file_list
+
+
+def parse_args():
+    """Parse arguments from cmd line.
+
+    :returns: Parsed arguments.
+    :rtype ArgumentParser
+    """
+
+    parser = ArgumentParser(description=__doc__,
+                            formatter_class=RawDescriptionHelpFormatter)
+    parser.add_argument("-i", "--input",
+                        required=True,
+                        help="Input folder with data files.")
+    parser.add_argument("-o", "--output",
+                        required=True,
+                        help="Output folder with csv files and templates for "
+                             "csv files.")
+    return parser.parse_args()
+
+
+def calculate_stats(data):
+    """Calculate statistics:
+    - average,
+    - standard deviation.
+
+    :param data: Data to process.
+    :type data: list
+    :returns: Average and standard deviation.
+    :rtype: tuple
+    """
+
+    if len(data) == 0:
+        return None, None
+
+    def average(items):
+        """Calculate average from the items.
+
+        :param items: Average is calculated from these items.
+        :type items: list
+        :returns: Average.
+        :rtype: float
+        """
+        return float(sum(items)) / len(items)
+
+    avg = average(data)
+    variance = [(x - avg) ** 2 for x in data]
+    stdev = sqrt(average(variance))
+
+    return avg, stdev
+
+
+def write_line_to_file(file_handler, item):
+    """Write a line to the csv file.
+
+    :param file_handler: File handler for the csv file. It must be open for
+     writing text.
+    :param item: Item to be written to the file.
+    :type file_handler: BinaryIO
+    :type item: dict
+    """
+
+    mean = "" if item["mean"] is None else "{:.1f}".format(item["mean"])
+    stdev = "" if item["stdev"] is None else "{:.1f}".format(item["stdev"])
+    change = "" if item["change"] is None else "{:.0f}%".format(item["change"])
+    file_handler.write("{},{},{},{}\n".format(item["old"], mean, stdev, change))
+
+
+def main():
+    """Main function to generate csv files for the chapter "CSIT Release Notes"
+    from json files generated by Jenkins' jobs.
+    """
+
+    args = parse_args()
+
+    json_files = get_files(args.input, EXT_JSON)
+    tmpl_files = get_files(args.output, EXT_TMPL)
+
+    if len(json_files) == 0:
+        print("No json data to process.")
+        exit(1)
+
+    if len(tmpl_files) == 0:
+        print("No template files to process.")
+        exit(1)
+
+    # Get information from template files
+    csv_data = list()
+    for tmpl_file in tmpl_files:
+        with open(tmpl_file, mode='r') as file_handler:
+            for line in file_handler:
+                line_list = line.split(',')
+                try:
+                    csv_data.append({
+                        "ID": line_list[0],
+                        "type": line_list[0].rsplit("-", 1)[-1],
+                        "old": ",".join(line_list[1:])[:-1],
+                        "last_old": line_list[-1][:-1],
+                        "rates": list(),
+                        "mean": None,
+                        "stdev": None,
+                        "change": None})
+                except IndexError:
+                    pass
+
+    # Update existing data with the new information from json files
+    for json_file in json_files:
+        with open(json_file) as file_handler:
+            tests_data = load(file_handler)
+            for item in csv_data:
+                try:
+                    rate = tests_data["data"][item["ID"]]["throughput"]["value"]
+                    item["rates"].append(rate)
+                except KeyError:
+                    pass
+
+    # Add statistics
+    for item in csv_data:
+        mean, stdev = calculate_stats(item["rates"])
+        if mean is not None:
+            mean = mean / 1000000
+            old = float(item["last_old"])
+            item["mean"] = mean
+            item["change"] = ((mean - old) / old) * 100
+            item["stdev"] = stdev / 1000000
+
+    # Sort the list, key = change
+    csv_data.sort(key=lambda data: data["change"], reverse=True)
+
+    # Write csv files
+    for tmpl_file in tmpl_files:
+        csv_file = tmpl_file.replace(EXT_TMPL, EXT_CSV)
+        with open(csv_file, "w") as file_handler:
+            for item in csv_data:
+                if "pdr_" in csv_file \
+                        and "_others" not in csv_file \
+                        and item["type"] == "pdrdisc" \
+                        and item["change"] >= 9.5:
+                    write_line_to_file(file_handler, item)
+                elif "pdr_" in csv_file \
+                        and "_others" in csv_file \
+                        and item["type"] == "pdrdisc" \
+                        and item["change"] < 9.5:
+                    write_line_to_file(file_handler, item)
+                elif "ndr_" in csv_file \
+                        and "_others" not in csv_file \
+                        and item["type"] == "ndrdisc" \
+                        and item["change"] >= 9.5:
+                    write_line_to_file(file_handler, item)
+                elif "ndr_" in csv_file \
+                        and "_others" in csv_file \
+                        and item["type"] == "ndrdisc" \
+                        and item["change"] < 9.5:
+                    write_line_to_file(file_handler, item)
+
+
+if __name__ == "__main__":
+    sys_exit(main())

diff --git a/resources/tools/report_gen/run_report.cfg b/resources/tools/report_gen/run_report.cfg
index 70eb411..642a3e3 100644 (file)
--- a/resources/tools/report_gen/run_report.cfg
+++ b/resources/tools/report_gen/run_report.cfg
@@ -28,6 +28,7 @@ DIR[DTR,PERF,HC]=${DIR[DTR]}/honeycomb_performance_results
 DIR[DTR,FUNC,VPP]=${DIR[DTR]}/vpp_functional_results
 DIR[DTR,FUNC,HC]=${DIR[DTR]}/honeycomb_functional_results
 DIR[DTR,FUNC,NSHSFC]=${DIR[DTR]}/nshsfc_functional_results
+DIR[DTR,PERF,VPP,IMPRV]=${DIR[RST]}/vpp_performance_tests/performance_improvements
 
 DIR[DTC]=${DIR[RST]}/test_configuration
 DIR[DTC,PERF,VPP]=${DIR[DTC]}/vpp_performance_configuration

diff --git a/resources/tools/report_gen/run_report.sh b/resources/tools/report_gen/run_report.sh
index 4c10492..db07977 100755 (executable)
--- a/resources/tools/report_gen/run_report.sh
+++ b/resources/tools/report_gen/run_report.sh
@@ -143,6 +143,15 @@ then
         --output ${DIR[DTO,PERF,VPP]}/vpp_performance_operational_data.rst \
         --data "SH_RUN" --formatting rst --start 4 --level 2
 
+    blds=${JOB[PERF,VPP,BLD]}
+    for i in ${blds[@]}; do
+        unzip -o ${DIR[STATIC,ARCH]}/${JOB[PERF,VPP]}-${i}.zip -d ${DIR[WORKING]}/
+        ./run_robot_json_data.py \
+            --input ${DIR[WORKING]}/output.xml \
+            --output ${DIR[DTR,PERF,VPP,IMPRV]}/${JOB[PERF,VPP]}-${i}.json \
+            --vdevice ${i}
+    done
+
     # DPDK PERF
     unzip -o ${DIR[STATIC,ARCH]}/${JOB[PERF,DPDK]}-${JOB[PERF,DPDK,FBLD]}.zip -d ${DIR[WORKING]}/
     python run_robot_data.py -i ${DIR[WORKING]}/robot-plugin/output.xml \
@@ -171,6 +180,11 @@ then
         --formatting rst --start 5 --level 2
 fi
 
+# Generate tables for performance improvements
+./run_improvments_tables.py \
+    --input ${DIR[DTR,PERF,VPP,IMPRV]} \
+    --output ${DIR[DTR,PERF,VPP,IMPRV]}
+
 # Delete temporary json files
 find ${DIR[RST]} -name "*.json" -type f -delete