From eecad36d7d2275fa47fbcab40dbcf56108ab0a51 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Fri, 11 Aug 2017 10:44:36 +0200 Subject: [PATCH] CSIT-755: Presentation and analytics layer - CSIT-760: Configuration - real example - CSIT-774: Implementation - parse configuration - CSIT-779: Implementation - set environment - CSIT-780: Implementation - download data - CSIT-783: Implementation - debug mode - CSIT-761: Implementation - Data pre-processing - parse input files - CSIT-784: Implementation - Data pre-processing - store the data, access to data - CSIT-789: Implementation - Data pre-processing - extract Documentation of the suite - CSIT-757: Low Level Design - CSIT-788: Implementation - Data pre-processing - extract VAT history and show runtime - CSIT-785: Implementation - Data filtering - CSIT-763: Presentation - tables - CSIT-804: Presentation - files - CSIT-762: Presentation - plots - LLD: API + functional diagram - CSIT-807: Element's models - CSIT-813: Process static content - CSIT-812: Report generation - CSIT-764: Integration to CSIT - CSIT-822: Archiving - CSIT-790: Documentation - configuration od the Input data is the same as for 17.07 report Change-Id: I6fd1eb1df4af99eaf91925282cdee1c892698c59 Signed-off-by: Tibor Frank --- .../packet_latency_graphs/ip4.rst | 4 +- .../packet_latency_graphs/l2.rst | 4 +- .../packet_throughput_graphs/ip4.rst | 8 +- .../packet_throughput_graphs/l2.rst | 8 +- docs/report/introduction/csit_tag_description.rst | 2 +- .../vpp_performance_tests/csit_release_notes.rst | 12 +- .../packet_latency_graphs/ip4.rst | 4 +- .../packet_latency_graphs/ip4_tunnels.rst | 4 +- .../packet_latency_graphs/ip6.rst | 4 +- .../packet_latency_graphs/ip6_tunnels.rst | 4 +- .../packet_latency_graphs/ipsec.rst | 4 +- .../packet_latency_graphs/l2.rst | 4 +- .../packet_latency_graphs/vm_vhost.rst | 4 +- .../packet_throughput_graphs/ip4.rst | 8 +- .../packet_throughput_graphs/ip4_tunnels.rst | 8 +- .../packet_throughput_graphs/ip6.rst | 8 +- .../packet_throughput_graphs/ip6_tunnels.rst | 8 +- .../packet_throughput_graphs/ipsec.rst | 8 +- .../packet_throughput_graphs/l2.rst | 6 +- .../packet_throughput_graphs/vm_vhost.rst | 8 +- .../ndr_throughput.template | 9 - .../ndr_throughput_others.template | 15 - .../pdr_throughput.template | 9 - .../pdr_throughput_others.template | 14 - .../tmpl_performance_improvements.csv | 47 + resources/tools/presentation/conf.py | 223 + .../tools/presentation/doc/pal_func_diagram.svg | 1413 ++++++ resources/tools/presentation/doc/pal_lld.rst | 1228 +++++ resources/tools/presentation/environment.py | 157 + resources/tools/presentation/errors.py | 78 + resources/tools/presentation/fdio.svg | 25 + resources/tools/presentation/generator_files.py | 109 + resources/tools/presentation/generator_plots.py | 221 + resources/tools/presentation/generator_report.py | 252 + resources/tools/presentation/generator_tables.py | 251 + resources/tools/presentation/input_data_files.py | 204 + resources/tools/presentation/input_data_parser.py | 860 ++++ resources/tools/presentation/pal.py | 122 + resources/tools/presentation/requirements.txt | 10 + resources/tools/presentation/run_report.sh | 46 + resources/tools/presentation/specification.yaml | 5065 ++++++++++++++++++++ .../tools/presentation/specification_parser.py | 507 ++ resources/tools/presentation/static_content.py | 57 + resources/tools/presentation/utils.py | 104 + 44 files changed, 11037 insertions(+), 109 deletions(-) delete mode 100644 docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template delete mode 100644 docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template delete mode 100644 docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template delete mode 100644 docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template create mode 100644 docs/report/vpp_performance_tests/performance_improvements/tmpl_performance_improvements.csv create mode 100644 resources/tools/presentation/conf.py create mode 100644 resources/tools/presentation/doc/pal_func_diagram.svg create mode 100644 resources/tools/presentation/doc/pal_lld.rst create mode 100644 resources/tools/presentation/environment.py create mode 100644 resources/tools/presentation/errors.py create mode 100644 resources/tools/presentation/fdio.svg create mode 100644 resources/tools/presentation/generator_files.py create mode 100644 resources/tools/presentation/generator_plots.py create mode 100644 resources/tools/presentation/generator_report.py create mode 100644 resources/tools/presentation/generator_tables.py create mode 100644 resources/tools/presentation/input_data_files.py create mode 100644 resources/tools/presentation/input_data_parser.py create mode 100644 resources/tools/presentation/pal.py create mode 100644 resources/tools/presentation/requirements.txt create mode 100755 resources/tools/presentation/run_report.sh create mode 100644 resources/tools/presentation/specification.yaml create mode 100644 resources/tools/presentation/specification_parser.py create mode 100644 resources/tools/presentation/static_content.py create mode 100644 resources/tools/presentation/utils.py diff --git a/docs/report/dpdk_performance_tests/packet_latency_graphs/ip4.rst b/docs/report/dpdk_performance_tests/packet_latency_graphs/ip4.rst index 4449785436..c978d56b80 100644 --- a/docs/report/dpdk_performance_tests/packet_latency_graphs/ip4.rst +++ b/docs/report/dpdk_performance_tests/packet_latency_graphs/ip4.rst @@ -29,7 +29,7 @@ git repository by filtering with specified regex as follows: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -E '64B-1t1c-ethip4-ip4base-l3fwd-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -E '64B-1t1c-ethip4-ip4base-l3fwd-ndrdisc' * :shell: .. only:: latex @@ -62,7 +62,7 @@ git repository by filtering with specified regex as follows: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -E '64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -E '64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc' * :shell: .. only:: latex diff --git a/docs/report/dpdk_performance_tests/packet_latency_graphs/l2.rst b/docs/report/dpdk_performance_tests/packet_latency_graphs/l2.rst index f981df9c89..6bdb86da92 100644 --- a/docs/report/dpdk_performance_tests/packet_latency_graphs/l2.rst +++ b/docs/report/dpdk_performance_tests/packet_latency_graphs/l2.rst @@ -30,7 +30,7 @@ git repository by filtering with specified regex as follows: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-eth-l2xcbase-testpmd-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-eth-l2xcbase-testpmd-ndrdisc' * :shell: .. only:: latex @@ -63,7 +63,7 @@ git repository by filtering with specified regex as follows: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-eth-l2xcbase-testpmd-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-eth-l2xcbase-testpmd-ndrdisc' * :shell: .. only:: latex diff --git a/docs/report/dpdk_performance_tests/packet_throughput_graphs/ip4.rst b/docs/report/dpdk_performance_tests/packet_throughput_graphs/ip4.rst index 11a54cca85..e4552d1326 100644 --- a/docs/report/dpdk_performance_tests/packet_throughput_graphs/ip4.rst +++ b/docs/report/dpdk_performance_tests/packet_throughput_graphs/ip4.rst @@ -34,7 +34,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-ethip4-ip4base-l3fwd-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-ethip4-ip4base-l3fwd-ndrdisc' * :shell: .. only:: latex @@ -68,7 +68,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc' * :shell: .. only:: latex @@ -105,7 +105,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-ethip4-ip4base-l3fwd-pdrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-ethip4-ip4base-l3fwd-pdrdisc' * :shell: .. only:: latex @@ -139,7 +139,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-ethip4-ip4base-l3fwd-pdrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-ethip4-ip4base-l3fwd-pdrdisc' * :shell: .. only:: latex diff --git a/docs/report/dpdk_performance_tests/packet_throughput_graphs/l2.rst b/docs/report/dpdk_performance_tests/packet_throughput_graphs/l2.rst index b63a12f3ad..58248dd0b1 100644 --- a/docs/report/dpdk_performance_tests/packet_throughput_graphs/l2.rst +++ b/docs/report/dpdk_performance_tests/packet_throughput_graphs/l2.rst @@ -34,7 +34,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-eth-l2xcbase-testpmd-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-eth-l2xcbase-testpmd-ndrdisc' * :shell: .. only:: latex @@ -68,7 +68,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-eth-l2xcbase-testpmd-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-eth-l2xcbase-testpmd-ndrdisc' * :shell: .. only:: latex @@ -105,7 +105,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-eth-l2xcbase-testpmd-pdrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-1t1c-eth-l2xcbase-testpmd-pdrdisc' * :shell: .. only:: latex @@ -139,7 +139,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-eth-l2xcbase-testpmd-pdrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/dpdk/perf && grep -P '64B-2t2c-eth-l2xcbase-testpmd-pdrdisc' * :shell: .. only:: latex diff --git a/docs/report/introduction/csit_tag_description.rst b/docs/report/introduction/csit_tag_description.rst index 53f1eb357a..af6e659457 100644 --- a/docs/report/introduction/csit_tag_description.rst +++ b/docs/report/introduction/csit_tag_description.rst @@ -7,5 +7,5 @@ execution. Following sections list currently used CSIT TAGs and their documentation based on the content of `tag documentation rst file`_. -.. include:: ../../tag_documentation.rst +.. include:: ../../../../../../docs/tag_documentation.rst :start-line: 8 diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst index c9e62e8415..812d5f2bed 100644 --- a/docs/report/vpp_performance_tests/csit_release_notes.rst +++ b/docs/report/vpp_performance_tests/csit_release_notes.rst @@ -101,8 +101,7 @@ Non-Drop Rate Throughput discovery tests: .. csv-table:: :align: center - :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change - :file: ../../../docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.csv + :file: performance_improvements/performance_improvements_ndr_top.csv PDR Throughput ~~~~~~~~~~~~~~ @@ -111,8 +110,7 @@ Partial Drop Rate thoughput discovery tests with packet Loss Tolerance of 0.5%: .. csv-table:: :align: center - :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change - :file: ../../../docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.csv + :file: performance_improvements/performance_improvements_pdr_top.csv Measured improvements are in line with VPP code optimizations listed in `VPP-17.07 release notes @@ -133,8 +131,7 @@ Non-Drop Rate Throughput discovery tests: .. csv-table:: :align: center - :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change - :file: ../../../docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.csv + :file: performance_improvements/performance_improvements_ndr_low.csv PDR Throughput ~~~~~~~~~~~~~~ @@ -143,8 +140,7 @@ Partial Drop Rate thoughput discovery tests with packet Loss Tolerance of 0.5%: .. csv-table:: :align: center - :header: VPP Functionality,Test Name,VPP-16.09 [Mpps],VPP-17.01 [Mpps],VPP-17.04 mean [Mpps],VPP-17.07 mean [Mpps],VPP-17.07 stdev [Mpps],17.04 to 17.07 change - :file: ../../../docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.csv + :file: performance_improvements/performance_improvements_pdr_low.csv Known Issues diff --git a/docs/report/vpp_performance_tests/packet_latency_graphs/ip4.rst b/docs/report/vpp_performance_tests/packet_latency_graphs/ip4.rst index 48bdf5e17e..e60d9d6021 100644 --- a/docs/report/vpp_performance_tests/packet_latency_graphs/ip4.rst +++ b/docs/report/vpp_performance_tests/packet_latency_graphs/ip4.rst @@ -29,7 +29,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-1t1c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-1t1c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-ndrdisc' * :shell: .. only:: latex @@ -61,7 +61,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-2t2c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-2t2c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-ndrdisc' * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_latency_graphs/ip4_tunnels.rst b/docs/report/vpp_performance_tests/packet_latency_graphs/ip4_tunnels.rst index 68489922b1..cbe9fe6873 100644 --- a/docs/report/vpp_performance_tests/packet_latency_graphs/ip4_tunnels.rst +++ b/docs/report/vpp_performance_tests/packet_latency_graphs/ip4_tunnels.rst @@ -29,7 +29,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" * :shell: .. only:: latex @@ -61,7 +61,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_latency_graphs/ip6.rst b/docs/report/vpp_performance_tests/packet_latency_graphs/ip6.rst index 1a82aac084..995a2d87fb 100644 --- a/docs/report/vpp_performance_tests/packet_latency_graphs/ip6.rst +++ b/docs/report/vpp_performance_tests/packet_latency_graphs/ip6.rst @@ -29,7 +29,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" * :shell: .. only:: latex @@ -61,7 +61,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_latency_graphs/ip6_tunnels.rst b/docs/report/vpp_performance_tests/packet_latency_graphs/ip6_tunnels.rst index a2fdef9bcb..07f4884fac 100644 --- a/docs/report/vpp_performance_tests/packet_latency_graphs/ip6_tunnels.rst +++ b/docs/report/vpp_performance_tests/packet_latency_graphs/ip6_tunnels.rst @@ -29,7 +29,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" * :shell: .. only:: latex @@ -61,7 +61,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_latency_graphs/ipsec.rst b/docs/report/vpp_performance_tests/packet_latency_graphs/ipsec.rst index d25d56134b..2513fa2343 100644 --- a/docs/report/vpp_performance_tests/packet_latency_graphs/ipsec.rst +++ b/docs/report/vpp_performance_tests/packet_latency_graphs/ipsec.rst @@ -33,7 +33,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-1t1c-.*ipsec.*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-1t1c-.*ipsec.*-ndrdisc" * :shell: .. only:: latex @@ -65,7 +65,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-2t2c-.*ipsec.*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-2t2c-.*ipsec.*-ndrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_latency_graphs/l2.rst b/docs/report/vpp_performance_tests/packet_latency_graphs/l2.rst index 8c340eb969..0017b0ec8c 100644 --- a/docs/report/vpp_performance_tests/packet_latency_graphs/l2.rst +++ b/docs/report/vpp_performance_tests/packet_latency_graphs/l2.rst @@ -46,7 +46,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*ndrdisc" * :shell: .. only:: latex @@ -95,7 +95,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*ndrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_latency_graphs/vm_vhost.rst b/docs/report/vpp_performance_tests/packet_latency_graphs/vm_vhost.rst index 82eb37b3a9..07cf4e7f49 100644 --- a/docs/report/vpp_performance_tests/packet_latency_graphs/vm_vhost.rst +++ b/docs/report/vpp_performance_tests/packet_latency_graphs/vm_vhost.rst @@ -47,7 +47,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-1t1c-.*vhost.*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-1t1c-.*vhost.*-ndrdisc" * :shell: .. only:: latex @@ -96,7 +96,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-2t2c-.*vhost.*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-2t2c-.*vhost.*-ndrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_throughput_graphs/ip4.rst b/docs/report/vpp_performance_tests/packet_throughput_graphs/ip4.rst index 6596f69e71..786b022f6a 100644 --- a/docs/report/vpp_performance_tests/packet_throughput_graphs/ip4.rst +++ b/docs/report/vpp_performance_tests/packet_throughput_graphs/ip4.rst @@ -33,7 +33,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-1t1c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-1t1c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-ndrdisc' * :shell: .. only:: latex @@ -67,7 +67,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-2t2c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-ndrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-2t2c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-ndrdisc' * :shell: .. only:: latex @@ -104,7 +104,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-1t1c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-pdrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-1t1c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-pdrdisc' * :shell: .. only:: latex @@ -138,7 +138,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-2t2c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-pdrdisc' * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4 && grep -P '64B-2t2c-ethip4(udp)*-ip4(base|scale[a-z0-9]*)(-iacl50-state(ful|less)-flows10k.*|-oacl50-state(ful|less)-flows10k.*|-snat.*|-udp.*|-cop.*|-iacldst.*|-ipolice.*)*-pdrdisc' * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_throughput_graphs/ip4_tunnels.rst b/docs/report/vpp_performance_tests/packet_throughput_graphs/ip4_tunnels.rst index 71dc06e9b3..a4112a7a7b 100644 --- a/docs/report/vpp_performance_tests/packet_throughput_graphs/ip4_tunnels.rst +++ b/docs/report/vpp_performance_tests/packet_throughput_graphs/ip4_tunnels.rst @@ -34,7 +34,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" * :shell: .. only:: latex @@ -67,7 +67,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" * :shell: .. only:: latex @@ -104,7 +104,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-pdrdisc" * :shell: .. only:: latex @@ -137,7 +137,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip4_tunnels && grep -E "64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-pdrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_throughput_graphs/ip6.rst b/docs/report/vpp_performance_tests/packet_throughput_graphs/ip6.rst index 959559c407..253993fa7b 100644 --- a/docs/report/vpp_performance_tests/packet_throughput_graphs/ip6.rst +++ b/docs/report/vpp_performance_tests/packet_throughput_graphs/ip6.rst @@ -34,7 +34,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" * :shell: .. only:: latex @@ -68,7 +68,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" * :shell: .. only:: latex @@ -105,7 +105,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*pdrdisc" * :shell: .. only:: latex @@ -139,7 +139,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6 && grep -E "78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*pdrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_throughput_graphs/ip6_tunnels.rst b/docs/report/vpp_performance_tests/packet_throughput_graphs/ip6_tunnels.rst index fa37b85155..677ca0fe0c 100644 --- a/docs/report/vpp_performance_tests/packet_throughput_graphs/ip6_tunnels.rst +++ b/docs/report/vpp_performance_tests/packet_throughput_graphs/ip6_tunnels.rst @@ -34,7 +34,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" * :shell: .. only:: latex @@ -68,7 +68,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" * :shell: .. only:: latex @@ -105,7 +105,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-pdrdisc" * :shell: .. only:: latex @@ -139,7 +139,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/ip6_tunnels && grep -E "78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-pdrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_throughput_graphs/ipsec.rst b/docs/report/vpp_performance_tests/packet_throughput_graphs/ipsec.rst index b59a521227..2512699b53 100644 --- a/docs/report/vpp_performance_tests/packet_throughput_graphs/ipsec.rst +++ b/docs/report/vpp_performance_tests/packet_throughput_graphs/ipsec.rst @@ -36,7 +36,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-1t1c-.*ipsec.*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-1t1c-.*ipsec.*-ndrdisc" * :shell: .. only:: latex @@ -69,7 +69,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-2t2c-.*ipsec.*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-2t2c-.*ipsec.*-ndrdisc" * :shell: .. only:: latex @@ -105,7 +105,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-1t1c-.*ipsec.*-pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-1t1c-.*ipsec.*-pdrdisc" * :shell: .. only:: latex @@ -138,7 +138,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-2t2c-.*ipsec.*-pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/crypto && grep -E "64B-2t2c-.*ipsec.*-pdrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_throughput_graphs/l2.rst b/docs/report/vpp_performance_tests/packet_throughput_graphs/l2.rst index 46c9500a11..cc04191d14 100644 --- a/docs/report/vpp_performance_tests/packet_throughput_graphs/l2.rst +++ b/docs/report/vpp_performance_tests/packet_throughput_graphs/l2.rst @@ -50,7 +50,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*ndrdisc" * :shell: .. only:: latex @@ -100,7 +100,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*ndrdisc" * :shell: .. only:: latex @@ -154,7 +154,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/l2 && grep -E "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-(iacl50-state(ful|less)-flows10k.*|oacl50-state(ful|less)-flows10k.*|eth.*)*pdrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/packet_throughput_graphs/vm_vhost.rst b/docs/report/vpp_performance_tests/packet_throughput_graphs/vm_vhost.rst index 952e37cb69..528a26cc19 100644 --- a/docs/report/vpp_performance_tests/packet_throughput_graphs/vm_vhost.rst +++ b/docs/report/vpp_performance_tests/packet_throughput_graphs/vm_vhost.rst @@ -51,7 +51,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-1t1c-.*vhost.*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-1t1c-.*vhost.*-ndrdisc" * :shell: .. only:: latex @@ -101,7 +101,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-2t2c-.*vhost.*-ndrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-2t2c-.*vhost.*-ndrdisc" * :shell: .. only:: latex @@ -154,7 +154,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-1t1c-.*vhost.*-pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-1t1c-.*vhost.*-pdrdisc" * :shell: .. only:: latex @@ -204,7 +204,7 @@ git repository: .. only:: html - .. program-output:: cd ../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-2t2c-.*vhost.*-pdrdisc" * + .. program-output:: cd ../../../../../ && set -x && cd tests/vpp/perf/vm_vhost && grep -E "64B-2t2c-.*vhost.*-pdrdisc" * :shell: .. only:: latex diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template deleted file mode 100644 index 99be4af8dc..0000000000 --- a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput.template +++ /dev/null @@ -1,9 +0,0 @@ -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-ndrdisc,IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-ndrdisc,3.0,7.3,8.1 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,6.1,6.1,6.9 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-iacldstbase-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,6.5,6.1,6.9 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale2m-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale2m-ndrdisc,IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-ndrdisc,5.3,4.2,4.6 -tests.vpp.perf.l2.40ge2p1xl710-eth-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-ndrdisc,L2XC,10ge2p1xl710: 64B-1t1c-eth-l2xcbase-ndrdisc,9.5,12.2,12.4 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,7.1,8.3,9.0 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale2m-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale2m-ndrdisc,IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-ndrdisc,8.5,7.8,8.1 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale20k-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale20k-ndrdisc,IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-ndrdisc,6.9,6.5,6.9 -tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-eth-2memif-1lxc-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-eth-2memif-1lxc-ndrdisc,LXC MEMIF,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2memif-1lxc-ndrdisc,,, diff --git a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template b/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template deleted file mode 100644 index f60a56ad15..0000000000 --- a/docs/report/vpp_performance_tests/performance_improvements/ndr_throughput_others.template +++ /dev/null @@ -1,15 +0,0 @@ -tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-ndrdisc,L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-ndrdisc,9.4,12.7,13.3 -tests.vpp.perf.l2.10ge2p1x520-dot1ad-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-dot1ad-l2xcbase-ndrdisc,L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-ndrdisc,7.4,8.8,9.3 -tests.vpp.perf.l2.10ge2p1x520-dot1q-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-dot1q-l2xcbase-ndrdisc,L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-ndrdisc,7.5,8.8,9.2 -tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4vxlan-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-ethip4vxlan-l2xcbase-ndrdisc,L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-ndrdisc,5.4,6.5,6.8 -tests.vpp.perf.l2.10ge2p1x520-eth-l2bdbasemaclrn-ndrpdrdisc.tc01-64b-1t1c-eth-l2bdbasemaclrn-ndrdisc,L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-ndrdisc,7.8,10.4,10.8 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-ndrdisc,IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ndrdisc,8.7,9.7,10.6 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-iacldstbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,6.9,7.6,8.3 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale200k-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale200k-ndrdisc,IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-ndrdisc,8.5,9.0,9.7 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale20k-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale20k-ndrdisc,IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-ndrdisc,8.5,9.0,9.7 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,6.9,7.4,8.1 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale200k-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale200k-ndrdisc,IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-ndrdisc,6.5,5.3,5.3 -tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4lispip4-ip4base-ndrpdrdisc.tc01-64b-1t1c-ethip4lispip4-ip4base-ndrdisc,IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-ndrdisc,4.4,4.8,5.5 -tests.vpp.perf.vm vhost.10ge2p1x520-dot1q-l2xcbase-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,0.5,2.8,3.2 -tests.vpp.perf.vm vhost.10ge2p1x710-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,L2BD-vhost-VM,10ge2p1x710: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,0.4,2.7,3.2 -tests.vpp.perf.vm vhost.10ge2p1x520-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrdisc,IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrdisc,0.3,2.6,3.1 diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template deleted file mode 100644 index 27c93ea1a7..0000000000 --- a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput.template +++ /dev/null @@ -1,9 +0,0 @@ -tests.vpp.perf.vm vhost.10ge2p1x520-dot1q-l2xcbase-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,2.6,3.2,3.2 -tests.vpp.perf.vm vhost.10ge2p1x710-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,L2BD-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,2.1,2.9,3.2 -tests.vpp.perf.vm vhost.10ge2p1x520-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-pdrdisc,IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-pdrdisc,2.0,2.7,3.0 -tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4lispip4-ip4base-ndrpdrdisc.tc02-64b-1t1c-ethip4lispip4-ip4base-pdrdisc,IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-pdrdisc,4.6,4.8,5.5 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-pdrdisc,IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-pdrdisc,7.7,7.3,8.1 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,6.1,6.1,6.9 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-iacldstbase-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,6.5,6.1,6.9 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale2m-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale2m-pdrdisc,IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-pdrdisc,5.3,4.2,4.6 -tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-eth-2memif-1lxc-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-eth-2memif-1lxc-pdrdisc,LXC MEMIF,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2memif-1lxc-pdrdisc,,, diff --git a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template b/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template deleted file mode 100644 index 7905ee899b..0000000000 --- a/docs/report/vpp_performance_tests/performance_improvements/pdr_throughput_others.template +++ /dev/null @@ -1,14 +0,0 @@ -tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-pdrdisc,L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-pdrdisc,9.4,12.7,13.4 -tests.vpp.perf.l2.10ge2p1x520-dot1ad-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-dot1ad-l2xcbase-pdrdisc,L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-pdrdisc,7.4,8.8,9.3 -tests.vpp.perf.l2.10ge2p1x520-dot1q-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-dot1q-l2xcbase-pdrdisc,L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-pdrdisc,7.5,8.8,9.2 -tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4vxlan-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-ethip4vxlan-l2xcbase-pdrdisc,L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-pdrdisc,5.4,6.5,6.8 -tests.vpp.perf.l2.10ge2p1x520-eth-l2bdbasemaclrn-ndrpdrdisc.tc02-64b-1t1c-eth-l2bdbasemaclrn-pdrdisc,L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-pdrdisc,7.8,10.6,10.9 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-pdrdisc,IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-pdrdisc,8.7,9.7,10.6 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,7.1,8.3,9.2 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-iacldstbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,7.1,7.6,8.3 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale200k-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale200k-pdrdisc,IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-pdrdisc,8.5,9.0,9.7 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale20k-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale20k-pdrdisc,IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-pdrdisc,8.5,9.0,9.7 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale2m-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale2m-pdrdisc,IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-pdrdisc,8.3,8.1,8.3 -tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,7.1,7.4,8.1 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale200k-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale200k-pdrdisc,IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-pdrdisc,6.9,5.3,5.3 -tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale20k-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale20k-pdrdisc,IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-pdrdisc,6.9,6.5,6.9 diff --git a/docs/report/vpp_performance_tests/performance_improvements/tmpl_performance_improvements.csv b/docs/report/vpp_performance_tests/performance_improvements/tmpl_performance_improvements.csv new file mode 100644 index 0000000000..fc466465bb --- /dev/null +++ b/docs/report/vpp_performance_tests/performance_improvements/tmpl_performance_improvements.csv @@ -0,0 +1,47 @@ +tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-ndrdisc,L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-ndrdisc,9.4,12.7,13.3 +tests.vpp.perf.l2.10ge2p1x520-dot1ad-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-dot1ad-l2xcbase-ndrdisc,L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-ndrdisc,7.4,8.8,9.3 +tests.vpp.perf.l2.10ge2p1x520-dot1q-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-dot1q-l2xcbase-ndrdisc,L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-ndrdisc,7.5,8.8,9.2 +tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4vxlan-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-ethip4vxlan-l2xcbase-ndrdisc,L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-ndrdisc,5.4,6.5,6.8 +tests.vpp.perf.l2.10ge2p1x520-eth-l2bdbasemaclrn-ndrpdrdisc.tc01-64b-1t1c-eth-l2bdbasemaclrn-ndrdisc,L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-ndrdisc,7.8,10.4,10.8 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-ndrdisc,IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ndrdisc,8.7,9.7,10.6 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-iacldstbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-ndrdisc,6.9,7.6,8.3 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale200k-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale200k-ndrdisc,IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-ndrdisc,8.5,9,9.7 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale20k-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale20k-ndrdisc,IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-ndrdisc,8.5,9,9.7 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-ndrdisc,6.9,7.4,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale200k-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale200k-ndrdisc,IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-ndrdisc,6.5,5.3,5.3 +tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4lispip4-ip4base-ndrpdrdisc.tc01-64b-1t1c-ethip4lispip4-ip4base-ndrdisc,IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-ndrdisc,4.4,4.8,5.5 +tests.vpp.perf.vm vhost.10ge2p1x520-dot1q-l2xcbase-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-ndrdisc,0.5,2.8,3.2 +tests.vpp.perf.vm vhost.10ge2p1x710-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,L2BD-vhost-VM,10ge2p1x710: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc,0.4,2.7,3.2 +tests.vpp.perf.vm vhost.10ge2p1x520-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrdisc,IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrdisc,0.3,2.6,3.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-ndrdisc,IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-ndrdisc,3,7.3,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-ndrdisc,6.1,6.1,6.9 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-iacldstbase-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-ndrdisc,6.5,6.1,6.9 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale2m-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale2m-ndrdisc,IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-ndrdisc,5.3,4.2,4.6 +tests.vpp.perf.l2.40ge2p1xl710-eth-l2xcbase-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-ndrdisc,L2XC,10ge2p1xl710: 64B-1t1c-eth-l2xcbase-ndrdisc,9.5,12.2,12.4 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-ndrdisc,7.1,8.3,9 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale2m-ndrpdrdisc.tc01-64b-1t1c-ethip4-ip4scale2m-ndrdisc,IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-ndrdisc,8.5,7.8,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale20k-ndrpdrdisc.tc01-78b-1t1c-ethip6-ip6scale20k-ndrdisc,IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-ndrdisc,6.9,6.5,6.9 +tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-eth-2memif-1lxc-ndrpdrdisc.tc01-64b-1t1c-eth-l2xcbase-eth-2memif-1lxc-ndrdisc,LXC MEMIF,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2memif-1lxc-ndrdisc,,, +tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-pdrdisc,L2XC,10ge2p1x520: 64B-1t1c-eth-l2xcbase-pdrdisc,9.4,12.7,13.4 +tests.vpp.perf.l2.10ge2p1x520-dot1ad-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-dot1ad-l2xcbase-pdrdisc,L2XC dot1ad,10ge2p1x520: 64B-1t1c-dot1ad-l2xcbase-pdrdisc,7.4,8.8,9.3 +tests.vpp.perf.l2.10ge2p1x520-dot1q-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-dot1q-l2xcbase-pdrdisc,L2XC dot1q,10ge2p1x520: 64B-1t1c-dot1q-l2xcbase-pdrdisc,7.5,8.8,9.2 +tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4vxlan-l2xcbase-ndrpdrdisc.tc02-64b-1t1c-ethip4vxlan-l2xcbase-pdrdisc,L2XC VxLAN,10ge2p1x520: 64B-1t1c-ethip4vxlan-l2xcbase-pdrdisc,5.4,6.5,6.8 +tests.vpp.perf.l2.10ge2p1x520-eth-l2bdbasemaclrn-ndrpdrdisc.tc02-64b-1t1c-eth-l2bdbasemaclrn-pdrdisc,L2BD,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-pdrdisc,7.8,10.6,10.9 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-pdrdisc,IPv4,10ge2p1x520: 64B-1t1c-ethip4-ip4base-pdrdisc,8.7,9.7,10.6 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,IPv4 COP,10ge2p1x520: 64B-1t1c-ethip4-ip4base-copwhtlistbase-pdrdisc,7.1,8.3,9.2 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-iacldstbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,IPv4 iAcl,10ge2p1x520: 64B-1t1c-ethip4-ip4base-iacldstbase-pdrdisc,7.1,7.6,8.3 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale200k-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale200k-pdrdisc,IPv4 FIB 200k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale200k-pdrdisc,8.5,9,9.7 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale20k-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale20k-pdrdisc,IPv4 FIB 20k,10ge2p1x520: 64B-1t1c-ethip4-ip4scale20k-pdrdisc,8.5,9,9.7 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4scale2m-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4scale2m-pdrdisc,IPv4 FIB 2M,10ge2p1x520: 64B-1t1c-ethip4-ip4scale2m-pdrdisc,8.3,8.1,8.3 +tests.vpp.perf.ip4.10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,IPv4 Policer,10ge2p1x520: 64B-1t1c-ethip4-ip4base-ipolicemarkbase-pdrdisc,7.1,7.4,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale200k-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale200k-pdrdisc,IPv6 FIB 200k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale200k-pdrdisc,6.9,5.3,5.3 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale20k-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale20k-pdrdisc,IPv6 FIB 20k,10ge2p1x520: 78B-1t1c-ethip6-ip6scale20k-pdrdisc,6.9,6.5,6.9 +tests.vpp.perf.vm vhost.10ge2p1x520-dot1q-l2xcbase-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,L2XC-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2vhost-1vm-pdrdisc,2.6,3.2,3.2 +tests.vpp.perf.vm vhost.10ge2p1x710-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,L2BD-vhost-VM,10ge2p1x520: 64B-1t1c-eth-l2bdbasemaclrn-eth-2vhost-1vm-pdrdisc,2.1,2.9,3.2 +tests.vpp.perf.vm vhost.10ge2p1x520-ethip4-ip4base-eth-2vhostvr1024-1vm-ndrpdrdisc.tc02-64b-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-pdrdisc,IPv4 vhost,10ge2p1x520: 64B-1t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-pdrdisc,2,2.7,3 +tests.vpp.perf.ip4 tunnels.10ge2p1x520-ethip4lispip4-ip4base-ndrpdrdisc.tc02-64b-1t1c-ethip4lispip4-ip4base-pdrdisc,IPv4 LISP,10ge2p1x520: 64B-1t1c-ethip4lispip4-ip4base-pdrdisc,4.6,4.8,5.5 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-pdrdisc,IPv6,10ge2p1x520: 78B-1t1c-ethip6-ip6base-pdrdisc,7.7,7.3,8.1 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,IPv6 COP,10ge2p1x520: 78B-1t1c-ethip6-ip6base-copwhtlistbase-pdrdisc,6.1,6.1,6.9 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6base-iacldstbase-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,IPv6 iAcl,10ge2p1x520: 78B-1t1c-ethip6-ip6base-iacldstbase-pdrdisc,6.5,6.1,6.9 +tests.vpp.perf.ip6.10ge2p1x520-ethip6-ip6scale2m-ndrpdrdisc.tc02-78b-1t1c-ethip6-ip6scale2m-pdrdisc,IPv6 FIB 2M,10ge2p1x520: 78B-1t1c-ethip6-ip6scale2m-pdrdisc,5.3,4.2,4.6 +tests.vpp.perf.l2.10ge2p1x520-eth-l2xcbase-eth-2memif-1lxc-ndrpdrdisc.tc02-64b-1t1c-eth-l2xcbase-eth-2memif-1lxc-pdrdisc,LXC MEMIF,10ge2p1x520: 64B-1t1c-eth-l2xcbase-eth-2memif-1lxc-pdrdisc,,, diff --git a/resources/tools/presentation/conf.py b/resources/tools/presentation/conf.py new file mode 100644 index 0000000000..2c942c45a8 --- /dev/null +++ b/resources/tools/presentation/conf.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +# +# CSIT 17.01 report documentation build configuration file, created by +# sphinx-quickstart on Sun Jan 15 09:49:36 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinxcontrib.programoutput', + 'sphinx.ext.ifconfig'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = ['.rst', '.md'] + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'FD.io CSIT' +copyright = u'2017, FD.io' +author = u'FD.io CSIT' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +#version = u'' +# The full version, including alpha/beta/rc tags. +#release = u'' + +rst_epilog = """ +.. |release-1| replace:: rls1704 +.. |vpp-release| replace:: VPP-17.07 release +.. |vpp-release-1| replace:: VPP-17.04 release +.. |dpdk-release| replace:: DPDK 17.05 +.. |trex-release| replace:: TRex v2.25 +.. |virl-image-ubuntu| replace:: ubuntu-16.04.1_2017-02-23_1.8 +.. |virl-image-centos| replace:: centos-7.3-1611_2017-02-23_1.4 + +.. _tag documentation rst file: https://git.fd.io/csit/tree/docs/tag_documentation.rst?h=rls1707 +.. _TRex intallation: https://git.fd.io/csit/tree/resources/tools/trex/trex_installer.sh?h=rls1707 +.. _TRex driver: https://git.fd.io/csit/tree/resources/tools/trex/trex_stateless_profile.py?h=rls1707 +.. _VIRL topologies directory: https://git.fd.io/csit/tree/resources/tools/virl/topologies/?h=rls1707 +.. _VIRL images lists: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/ubuntu/lists/?h=rls1707 +.. _VIRL nested: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/nested/?h=rls1707 +.. _CSIT Honeycomb Functional Tests Documentation: https://docs.fd.io/csit/rls1707/doc/tests.vpp.func.honeycomb.html +.. _CSIT Honeycomb Performance Tests Documentation: https://docs.fd.io/csit/rls1707/doc/tests.vpp.perf.honeycomb.html +.. _CSIT DPDK Performance Tests Documentation: https://docs.fd.io/csit/rls1707/doc/tests.dpdk.perf.html +.. _CSIT VPP Functional Tests Documentation: https://docs.fd.io/csit/rls1707/doc/tests.vpp.func.html +.. _CSIT VPP Performance Tests Documentation: https://docs.fd.io/csit/rls1707/doc/tests.vpp.perf.html +.. _CSIT NSH_SFC Functional Tests Documentation: https://docs.fd.io/csit/rls1707/doc/tests.nsh_sfc.func.html +.. _VPP test framework documentation: https://docs.fd.io/vpp/17.07/vpp_make_test/html/ +.. _FD.io test executor vpp performance jobs: https://jenkins.fd.io/view/csit/job/csit-vpp-perf-1707-all +.. _FD.io test executor vpp functional jobs: https://jenkins.fd.io/view/csit/job/csit-vpp-functional-1707-ubuntu1604-virl/lastSuccessfulBuild +.. _FD.io test executor dpdk performance jobs: https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-1707-all +.. _FD.io test executor Honeycomb functional jobs: https://jenkins.fd.io/view/csit/job/hc2vpp-csit-integration-1707-ubuntu1604/lastSuccessfulBuild +.. _FD.io test executor honeycomb performance jobs: https://jenkins.fd.io/view/hc2vpp/job/hc2vpp-csit-perf-master-ubuntu1604/lastSuccessfulBuild +.. _FD.io test executor NSH_SFC functional jobs: https://jenkins.fd.io/view/csit/job/csit-nsh_sfc-verify-func-1707-ubuntu1604-virl/lastSuccessfulBuild +.. _FD.io VPP compile job: https://jenkins.fd.io/view/vpp/job/vpp-merge-1707-ubuntu1604/ +.. _CSIT Testbed Setup: https://git.fd.io/csit/tree/resources/tools/testbed-setup/README.md?h=rls1707 +""" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_theme_path = ['env/lib/python2.7/site-packages/sphinx_rtd_theme'] + +# html_static_path = ['_build/_static'] +html_static_path = ['_tmp/src/_static'] + +html_context = { + 'css_files': [ + '_static/theme_overrides.css', # overrides for wide tables in RTD theme + ], + } + +# -- Options for LaTeX output --------------------------------------------- + +latex_engine = 'pdflatex' + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + 'papersize': 'a4paper', + + # The font size ('10pt', '11pt' or '12pt'). + # + #'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + 'preamble': r''' + \usepackage{pdfpages} + \usepackage{svg} + \usepackage{charter} + \usepackage[defaultsans]{lato} + \usepackage{inconsolata} + \usepackage{csvsimple} + ''', + + # Latex figure (float) alignment + # + 'figure_align': 'H', + + # Latex font setup + # + 'fontpkg': r''' + \renewcommand{\familydefault}{\sfdefault} + ''', + + # Latex other setup + # + 'extraclassoptions': 'openany', + 'sphinxsetup': r''' + TitleColor={RGB}{225,38,40}, + InnerLinkColor={RGB}{62,62,63}, + OuterLinkColor={RGB}{225,38,40}, + shadowsep=0pt, + shadowsize=0pt, + shadowrule=0pt + ''' +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'csit.tex', u'CSIT REPORT', + u'', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = 'fdio.pdf' + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = True + +# If true, show page references after internal links. +# +latex_show_pagerefs = True + +# If true, show URL addresses after external links. +# +latex_show_urls = 'footnote' + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True diff --git a/resources/tools/presentation/doc/pal_func_diagram.svg b/resources/tools/presentation/doc/pal_func_diagram.svg new file mode 100644 index 0000000000..c74cb79185 --- /dev/null +++ b/resources/tools/presentation/doc/pal_func_diagram.svg @@ -0,0 +1,1413 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Specification.YAML + + + + + + + + Data to process.xml + + + + + + + + Static content.rst + + + + + + + + + + read_specification + + + + + + + + + + read_data + + + + + + + + Specification + + + + + + + + Input data + + + + + + + + + + filter_data + + + + + + + + + + filter_data + + + + + + + + + + generate_files + + + + + + + + Tables + + + + + + + + Plots + + + + + + + + Files + + + + + + + + + + generate_report + + + + + + + + Report + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + generate_tables + + + + + + + + + + generate_plots + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + sL1 - Data + + + + + + sL2 - Data processing + + + + + + sL3 - Data presentation + + + + + + sL4 - Report generation + + + + + + + \ No newline at end of file diff --git a/resources/tools/presentation/doc/pal_lld.rst b/resources/tools/presentation/doc/pal_lld.rst new file mode 100644 index 0000000000..9c2446ea91 --- /dev/null +++ b/resources/tools/presentation/doc/pal_lld.rst @@ -0,0 +1,1228 @@ +=================================================== +Presentation and Analytics Layer - Low Level Design +=================================================== + +Overview +-------- + +The presentation and analytics layer (PAL) is the fourth layer of CSIT +hierarchy. The model of presentation and analytics layer consists of four +sub-layers, bottom up: + + - sL1 - Data - input data to be processed: + + - Static content - .rst text files, .svg static figures, and other files + stored in the CSIT git repository. + - Data to process - .xml files generated by Jenkins jobs executing tests, + stored as robot results files (output.xml). + - Specification - .yaml file with the models of report elements (tables, + plots, layout, ...) generated by this tool. There is also the configuration + of the tool and the specification of input data (jobs and builds). + + - sL2 - Data processing + + - The data are read from the specified input files (.xml) and stored as + multi-indexed `pandas.Series `_. + - This layer provides also interface to input data and filtering of the input + data. + + - sL3 - Data presentation - This layer generates the elements specified in the + specification file: + + - Tables: .csv files linked to static .rst files. + - Plots: .html files generated using plot.ly linked to static .rst files. + + - sL4 - Report generation - Sphinx generates required formats and versions: + + - formats: html, pdf + - versions: minimal, full (TODO: define the names and scope of versions) + + +Data +---- + +Report Specification +```````````````````` + +The report specification file defines which data is used and which outputs are +generated. It is human readable and structured. It is easy to add / remove / +change items. The specification includes: + + - Specification of the environment. + - Configuration of debug mode (optional). + - Specification of input data (jobs, builds, files, ...). + - Specification of the output. + - What and how is generated: + - What: plots, tables. + - How: specification of all properties and parameters. + - .yaml format. + +Structure of the specification file +''''''''''''''''''''''''''''''''''' + +The specification file is organized as a list of dictionaries distinguished by +the type: + +:: + + - + type: "environment" + - + type: "debug" + - + type: "static" + - + type: "input" + - + type: "output" + - + type: "table" + - + type: "plot" + - + type: "file" + +Each type represents a section. The sections "environment", "debug", "static", +"input" and "output" are listed only once in the specification; "table", "file" +and "plot" can be there multiple times. + +Sections "debug", "table", "file" and "plot" are optional. + +Table(s), files(s) and plot(s) are referred as "elements" in this text. It is +possible to define and implement other elements if needed. + + +Section: Environment +'''''''''''''''''''' + +This section has the following parts: + + - type: "environment" - says that this is the section "environment". + - configuration - configuration of the PAL. + - paths - paths used by the PAL. + - urls - urls pointing to the data sources. + - make-dirs - a list of the directories to be created by the PAL while + preparing the environment. + - remove-dirs - a list of the directories to be removed while cleaning the + environment. + - build-dirs - a list of the directories where the results are stored. + +The structure of the section "Environment" is as follows (example): +:: + + - + type: "environment" + configuration: + # Debug mode: + # - Skip: + # - Download of input data files + # - Do: + # - Read data from given zip / xml files + # - Set the configuration as it is done in normal mode + # If the section "type: debug" is missing, CFG[DEBUG] is set to 0. + CFG[DEBUG]: 0 + + paths: + # Top level directories: + ## Working directory + DIR[WORKING]: "_tmp" + ## Build directories + DIR[BUILD,HTML]: "_build" + DIR[BUILD,LATEX]: "_build_latex" + + # Static .rst files + DIR[RST]: "../../../docs/report" + + # Working directories + ## Input data files (.zip, .xml) + DIR[WORKING,DATA]: "{DIR[WORKING]}/data" + ## Static source files from git + DIR[WORKING,SRC]: "{DIR[WORKING]}/src" + DIR[WORKING,SRC,STATIC]: "{DIR[WORKING,SRC]}/_static" + + # Static html content + DIR[STATIC]: "{DIR[BUILD,HTML]}/_static" + DIR[STATIC,VPP]: "{DIR[STATIC]}/vpp" + DIR[STATIC,DPDK]: "{DIR[STATIC]}/dpdk" + DIR[STATIC,ARCH]: "{DIR[STATIC]}/archive" + + # Detailed test results + DIR[DTR]: "{DIR[WORKING,SRC]}/detailed_test_results" + DIR[DTR,PERF,DPDK]: "{DIR[DTR]}/dpdk_performance_results" + DIR[DTR,PERF,VPP]: "{DIR[DTR]}/vpp_performance_results" + DIR[DTR,PERF,HC]: "{DIR[DTR]}/honeycomb_performance_results" + DIR[DTR,FUNC,VPP]: "{DIR[DTR]}/vpp_functional_results" + DIR[DTR,FUNC,HC]: "{DIR[DTR]}/honeycomb_functional_results" + DIR[DTR,FUNC,NSHSFC]: "{DIR[DTR]}/nshsfc_functional_results" + DIR[DTR,PERF,VPP,IMPRV]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_improvements" + + # Detailed test configurations + DIR[DTC]: "{DIR[WORKING,SRC]}/test_configuration" + DIR[DTC,PERF,VPP]: "{DIR[DTC]}/vpp_performance_configuration" + DIR[DTC,FUNC,VPP]: "{DIR[DTC]}/vpp_functional_configuration" + + # Detailed tests operational data + DIR[DTO]: "{DIR[WORKING,SRC]}/test_operational_data" + DIR[DTO,PERF,VPP]: "{DIR[DTO]}/vpp_performance_operational_data" + + # .css patch file to fix tables generated by Sphinx + DIR[CSS_PATCH_FILE]: "{DIR[STATIC]}/theme_overrides.css" + DIR[CSS_PATCH_FILE2]: "{DIR[WORKING,SRC,STATIC]}/theme_overrides.css" + + urls: + URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" + URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job" + + make-dirs: + # List the directories which are created while preparing the environment. + # All directories MUST be defined in "paths" section. + - "DIR[WORKING,DATA]" + - "DIR[STATIC,VPP]" + - "DIR[STATIC,DPDK]" + - "DIR[STATIC,ARCH]" + - "DIR[BUILD,LATEX]" + - "DIR[WORKING,SRC]" + - "DIR[WORKING,SRC,STATIC]" + + remove-dirs: + # List the directories which are deleted while cleaning the environment. + # All directories MUST be defined in "paths" section. + #- "DIR[BUILD,HTML]" + + build-dirs: + # List the directories where the results (build) is stored. + # All directories MUST be defined in "paths" section. + - "DIR[BUILD,HTML]" + - "DIR[BUILD,LATEX]" + +It is possible to use defined items in the definition of other items, e.g.: + +:: + + DIR[WORKING,DATA]: "{DIR[WORKING]}/data" + +will be automatically changed to + +:: + + DIR[WORKING,DATA]: "_tmp/data" + + +Section: Debug mode +''''''''''''''''''' + +This section is optional as it configures the debug mode. It is used if one +does not want to download input data files and use local files instead. + +If the debug mode is configured, the "input" section is ignored. + +This section has the following parts: + + - type: "debug" - says that this is the section "debug". + - general: + + - input-format - xml or zip. + - extract - if "zip" is defined as the input format, this file is extracted + from the zip file, otherwise this parameter is ignored. + + - builds - list of builds from which the data is used. Must include a job + name as a key and then a list of builds and their output files. + +The structure of the section "Debug" is as follows (example): + +:: + + - + type: "debug" + general: + input-format: "zip" # zip or xml + extract: "robot-plugin/output.xml" # Only for zip + builds: + # The files must be in the directory DIR[WORKING,DATA] + csit-dpdk-perf-1707-all: + - + build: 10 + file: "csit-dpdk-perf-1707-all__10.xml" + - + build: 9 + file: "csit-dpdk-perf-1707-all__9.xml" + csit-nsh_sfc-verify-func-1707-ubuntu1604-virl: + - + build: 2 + file: "csit-nsh_sfc-verify-func-1707-ubuntu1604-virl-2.xml" + csit-vpp-functional-1707-ubuntu1604-virl: + - + build: lastSuccessfulBuild + file: "csit-vpp-functional-1707-ubuntu1604-virl-lastSuccessfulBuild.xml" + hc2vpp-csit-integration-1707-ubuntu1604: + - + build: lastSuccessfulBuild + file: "hc2vpp-csit-integration-1707-ubuntu1604-lastSuccessfulBuild.xml" + csit-vpp-perf-1707-all: + - + build: 16 + file: "csit-vpp-perf-1707-all__16__output.xml" + - + build: 17 + file: "csit-vpp-perf-1707-all__17__output.xml" + + +Section: Static +''''''''''''''' + +This section defines the static content which is stored in git and will be used +as a source to generate the report. + +This section has these parts: + + - type: "static" - says that this section is the "static". + - src-path - path to the static content. + - dst-path - destination path where the static content is copied and then + processed. + +:: + - + type: "static" + src-path: "{DIR[RST]}" + dst-path: "{DIR[WORKING,SRC]}" + + +Section: Input +'''''''''''''' + +This section defines the data used to generate elements. It is mandatory +if the debug mode is not used. + +This section has the following parts: + + - type: "input" - says that this section is the "input". + - general - parameters common to all builds: + + - file-name: file to be downloaded. + - file-format: format of the downloaded file, ".zip" or ".xml" are supported. + - download-path: path to be added to url pointing to the file, e.g.: + "{job}/{build}/robot/report/*zip*/{filename}"; {job}, {build} and + {filename} are replaced by proper values defined in this section. + - extract: file to be extracted from downloaded zip file, e.g.: "output.xml"; + if xml file is downloaded, this parameter is ignored. + + - builds - list of jobs (keys) and numbers of builds which output data will be + downloaded. + +The structure of the section "Input" is as follows (example from 17.07 report): + +:: + + - + type: "input" # Ignored in debug mode + general: + file-name: "robot-plugin.zip" + file-format: ".zip" + download-path: "{job}/{build}/robot/report/*zip*/{filename}" + extract: "robot-plugin/output.xml" + builds: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + - 22 + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + csit-vpp-functional-1707-ubuntu1604-virl: + - lastSuccessfulBuild + hc2vpp-csit-perf-master-ubuntu1604: + - 8 + - 9 + hc2vpp-csit-integration-1707-ubuntu1604: + - lastSuccessfulBuild + csit-nsh_sfc-verify-func-1707-ubuntu1604-virl: + - 2 + + +Section: Output +''''''''''''''' + +This section specifies which format(s) will be generated (html, pdf) and which +versions will be generated for each format. + +This section has the following parts: + + - type: "output" - says that this section is the "output". + - format: html or pdf. + - version: defined for each format separately. + +The structure of the section "Output" is as follows (example): + +:: + + - + type: "output" + format: + html: + - full + pdf: + - full + - minimal + +TODO: define the names of versions + + +Content of "minimal" version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +TODO: define the name and content of this version + + +Section: Table +'''''''''''''' + +This section defines a table to be generated. There can be 0 or more "table" +sections. + +This section has the following parts: + + - type: "table" - says that this section defines a table. + - title: Title of the table. + - algorithm: Algorithm which is used to generate the table. The other + parameters in this section must provide all information needed by the used + algorithm. + - template: (optional) a .csv file used as a template while generating the + table. + - output-file-ext: extension of the output file. + - output-file: file which the table will be written to. + - columns: specification of table columns: + + - title: The title used in the table header. + - data: Specification of the data, it has two parts - command and arguments: + + - command: + + - template - take the data from template, arguments: + + - number of column in the template. + + - data - take the data from the input data, arguments: + + - jobs and builds which data will be used. + + - operation - performs an operation with the data already in the table, + arguments: + + - operation to be done, e.g.: mean, stdev, relative_change (compute + the relative change between two columns) and display number of data + samples ~= number of test jobs. The operations are implemented in the + utils.py + TODO: Move from utils,py to e.g. operations.py + - numbers of columns which data will be used (optional). + + - data: Specify the jobs and builds which data is used to generate the table. + - filter: filter based on tags applied on the input data, if "template" is + used, filtering is based on the template. + - parameters: Only these parameters will be put to the output data structure. + +The structure of the section "Table" is as follows (example of +"table_performance_improvements"): + +:: + + - + type: "table" + title: "Performance improvements" + algorithm: "table_performance_improvements" + template: "{DIR[DTR,PERF,VPP,IMPRV]}/tmpl_performance_improvements.csv" + output-file-ext: ".csv" + output-file: "{DIR[DTR,PERF,VPP,IMPRV]}/performance_improvements" + columns: + - + title: "VPP Functionality" + data: "template 1" + - + title: "Test Name" + data: "template 2" + - + title: "VPP-16.09 mean [Mpps]" + data: "template 3" + - + title: "VPP-17.01 mean [Mpps]" + data: "template 4" + - + title: "VPP-17.04 mean [Mpps]" + data: "template 5" + - + title: "VPP-17.07 mean [Mpps]" + data: "data csit-vpp-perf-1707-all mean" + - + title: "VPP-17.07 stdev [Mpps]" + data: "data csit-vpp-perf-1707-all stdev" + - + title: "17.04 to 17.07 change [%]" + data: "operation relative_change 5 4" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "template" + parameters: + - "throughput" + +Example of "table_details" which generates "Detailed Test Results - VPP +Performance Results": + +:: + + - + type: "table" + title: "Detailed Test Results - VPP Performance Results" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[WORKING]}/vpp_performance_results" + columns: + - + title: "Name" + data: "data test_name" + - + title: "Documentation" + data: "data test_documentation" + - + title: "Status" + data: "data test_msg" + data: + csit-vpp-perf-1707-all: + - 17 + filter: "all" + parameters: + - "parent" + - "doc" + - "msg" + +Example of "table_details" which generates "Test configuration - VPP Performance +Test Configs": + +:: + + - + type: "table" + title: "Test configuration - VPP Performance Test Configs" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[WORKING]}/vpp_test_configuration" + columns: + - + title: "Name" + data: "data name" + - + title: "VPP API Test (VAT) Commands History - Commands Used Per Test Case" + data: "data show-run" + data: + csit-vpp-perf-1707-all: + - 17 + filter: "all" + parameters: + - "parent" + - "name" + - "show-run" + + +Section: Plot +''''''''''''' + +This section defines a plot to be generated. There can be 0 or more "plot" +sections. + +This section has these parts: + + - type: "plot" - says that this section defines a plot. + - title: Plot title used in the logs. Title which is displayed is in the + section "layout". + - output-file-type: format of the output file. + - output-file: file which the plot will be written to. + - algorithm: Algorithm used to generate the plot. The other parameters in this + section must provide all information needed by plot.ly to generate the plot. + For example: + + - traces + - layout + + - These parameters are transparently passed to plot.ly. + + - data: Specify the jobs and numbers of builds which data is used to generate + the plot. + - filter: filter applied on the input data. + - parameters: Only these parameters will be put to the output data structure. + +The structure of the section "Plot" is as follows (example of a plot showing +throughput in a chart box-with-whiskers): + +:: + + - + type: "plot" + title: "VPP Performance 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + # Keep this formatting, the filter is enclosed with " (quotation mark) and + # each tag is enclosed with ' (apostrophe). + filter: "'64B' and 'BASE' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +The structure of the section "Plot" is as follows (example of a plot showing +latency in a box chart): + +:: + + - + type: "plot" + title: "VPP Latency 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'BASE' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + + +Section: file +''''''''''''' + +This section defines a file to be generated. There can be 0 or more "file" +sections. + +This section has the following parts: + + - type: "file" - says that this section defines a file. + - title: Title of the table. + - algorithm: Algorithm which is used to generate the file. The other + parameters in this section must provide all information needed by the used + algorithm. + - output-file-ext: extension of the output file. + - output-file: file which the file will be written to. + - file-header: The header of the generated .rst file. + - dir-tables: The directory with the tables. + - data: Specify the jobs and builds which data is used to generate the table. + - filter: filter based on tags applied on the input data, if "all" is + used, no filtering is done. + - parameters: Only these parameters will be put to the output data structure. + - chapters: the hierarchy of chapters in the generated file. + - start-level: the level of the the top-level chapter. + +The structure of the section "file" is as follows (example): + +:: + + - + type: "file" + title: "VPP Performance Results" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTR,PERF,VPP]}/vpp_performance_results" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTR,PERF,VPP]}" + data: + csit-vpp-perf-1707-all: + - 22 + filter: "all" + parameters: + - "name" + - "doc" + - "level" + data-start-level: 2 # 0, 1, 2, ... + chapters-start-level: 2 # 0, 1, 2, ... + + +Static content +`````````````` + + - Manually created / edited files. + - .rst files, static .csv files, static pictures (.svg), ... + - Stored in CSIT git repository. + +No more details about the static content in this document. + + +Data to process +``````````````` + +The PAL processes tests results and other information produced by Jenkins jobs. +The data are now stored as robot results in Jenkins (TODO: store the data in +nexus) either as .zip and / or .xml files. + + +Data processing +--------------- + +As the first step, the data are downloaded and stored locally (typically on a +Jenkins slave). If .zip files are used, the given .xml files are extracted for +further processing. + +Parsing of the .xml files is performed by a class derived from +"robot.api.ResultVisitor", only necessary methods are overridden. All and only +necessary data is extracted from .xml file and stored in a structured form. + +The parsed data are stored as the multi-indexed pandas.Series data type. Its +structure is as follows: + +:: + + + + + + + +"job name", "build", "metadata", "suites", "tests" are indexes to access the +data. For example: + +:: + + data = + + job 1 name: + build 1: + metadata: metadata + suites: suites + tests: tests + ... + build N: + metadata: metadata + suites: suites + tests: tests + ... + job M name: + build 1: + metadata: metadata + suites: suites + tests: tests + ... + build N: + metadata: metadata + suites: suites + tests: tests + +Using indexes data["job 1 name"]["build 1"]["tests"] (e.g.: +data["csit-vpp-perf-1704-all"]["17"]["tests"]) we get a list of all tests with +all tests data. + +Data will not be accessible directly using indexes, but using getters and +filters. + +**Structure of metadata:** + +:: + + "metadata": { + "version": "VPP version", + "job": "Jenkins job name" + "build": "Information about the build" + }, + +**Structure of suites:** + +:: + + "suites": { + "Suite name 1": { + "doc": "Suite 1 documentation" + "parent": "Suite 1 parent" + } + "Suite name N": { + "doc": "Suite N documentation" + "parent": "Suite N parent" + } + +**Structure of tests:** + +Performance tests: + +:: + + "tests": { + "ID": { + "name": "Test name", + "parent": "Name of the parent of the test", + "doc": "Test documentation" + "msg": "Test message" + "tags": ["tag 1", "tag 2", "tag n"], + "type": "PDR" | "NDR", + "throughput": { + "value": int, + "unit": "pps" | "bps" | "percentage" + }, + "latency": { + "direction1": { + "100": { + "min": int, + "avg": int, + "max": int + }, + "50": { # Only for NDR + "min": int, + "avg": int, + "max": int + }, + "10": { # Only for NDR + "min": int, + "avg": int, + "max": int + } + }, + "direction2": { + "100": { + "min": int, + "avg": int, + "max": int + }, + "50": { # Only for NDR + "min": int, + "avg": int, + "max": int + }, + "10": { # Only for NDR + "min": int, + "avg": int, + "max": int + } + } + }, + "lossTolerance": "lossTolerance" # Only for PDR + "vat-history": "DUT1 and DUT2 VAT History" + }, + "show-run": "Show Run" + }, + "ID" { + # next test + } + +Functional tests: + +:: + + "tests": { + "ID": { + "name": "Test name", + "parent": "Name of the parent of the test", + "doc": "Test documentation" + "msg": "Test message" + "tags": ["tag 1", "tag 2", "tag n"], + "vat-history": "DUT1 and DUT2 VAT History" + "show-run": "Show Run" + "status": "PASS" | "FAIL" + }, + "ID" { + # next test + } + } + +Note: ID is the lowercase full path to the test. + + +Data filtering +`````````````` + +The first step when generating an element is getting the data needed to +construct the element. The data are filtered from the processed input data. + +The data filtering is based on: + + - job name(s). + - build number(s). + - tag(s). + - required data - only this data is included in the output. + +WARNING: The filtering is based on tags, so be careful with tagging. + +For example, the element which specification includes: + +:: + + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: + - "'64B' and 'BASE' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + +will be constructed using data from the job "csit-vpp-perf-1707-all", for all listed +builds and the tests with the list of tags matching the filter conditions. + +The output data structure for filtered test data is: + +:: + + - job 1 + - build 1 + - test 1 + - parameter 1 + - parameter 2 + ... + - parameter n + ... + - test n + ... + ... + - build n + ... + - job n + + +Data analytics +`````````````` + +Data analytics part implements: + + - methods to compute statistical data from the filtered input data. + - trending. + - etc. + + +Advanced data analytics +``````````````````````` + +As the next steps, advanced data analytics (ADA) will be implemented using +machine learning (ML) and artificial intelligence (AI). + +TODO: + + - describe the concept of ADA. + - add specification. + + +Data presentation +----------------- + +Generates the plots and tables according to the report models per +specification file. The elements are generated using algorithms and data +specified in their models. + + +Tables +`````` + + - tables are generated by algorithms implemented in PAL, the model includes the + algorithm and all necessary information. + - output format: csv + - generated tables are stored in specified directories and linked to .rst files. + + +Plots +````` + + - `plot.ly `_ is currently used to generate plots, the model + includes the type of plot and all the necessary information to render it. + - output format: html. + - generated plots are stored in specified directories and linked to .rst files. + + +Report generation +----------------- + +Report is generated using Sphinx and Read_the_Docs template. PAL generates html +and pdf formats. It is possible to define the content of the report by specifying +the version (TODO: define the names and content of versions). + + +The process +``````````` + +1. Read the specification. +2. Read the input data. +3. Process the input data. +4. For element (plot, table, file) defined in specification: + + a. Get the data needed to construct the element using a filter. + b. Generate the element. + c. Store the element. + +5. Generate the report. +6. Store the report (Nexus). + +The process is model driven. The elements’ models (tables, plots, files and +report itself) are defined in the specification file. Script reads the elements’ +models from specification file and generates the elements. + +It is easy to add elements to be generated, if a new kind of element is +required, only a new algorithm is implemented and integrated. + + +API +--- + +List of modules, classes, methods and functions +``````````````````````````````````````````````` + +:: + + specification_parser.py + + class Specification + + Methods: + read_specification + set_input_state + set_input_file_name + + Getters: + specification + environment + debug + is_debug + input + builds + output + tables + plots + files + static + + + input_data_parser.py + + class InputData + + Methods: + read_data + filter_data + + Getters: + data + metadata + suites + tests + + + environment.py + + Functions: + clean_environment + + class Environment + + Methods: + set_environment + + Getters: + environment + + + input_data_files.py + + Functions: + download_data_files + unzip_files + + + generator_tables.py + + Functions: + generate_tables + + Functions implementing algorithms to generate particular types of + tables (called by the function "generate_tables"): + table_details + table_performance_improvements + + + generator_plots.py + + Functions: + generate_plots + + Functions implementing algorithms to generate particular types of + plots (called by the function "generate_plots"): + plot_performance_box + plot_latency_box + + + generator_files.py + + Functions: + generate_files + + Functions implementing algorithms to generate particular types of + files (called by the function "generate_files"): + file_test_results + + + report.py + + Functions: + generate_report + + Functions implementing algorithms to generate particular types of + report (called by the function "generate_report"): + generate_html_report + generate_pdf_report + + Other functions called by the function "generate_report": + archive_input_data + archive_report + + +PAL functional diagram +`````````````````````` + +.. figure:: pal_func_diagram.svg + :alt: PAL functional diagram + :align: center + + +How to add an element +````````````````````` + +Element can be added by adding its model to the specification file. If the +element will be generated by an existing algorithm, only its parameters must be +set. + +If a brand new type of element will be added, also the algorithm must be +implemented. +The algorithms are implemented in the files which names start with "generator". +The name of the function implementing the algorithm and the name of algorithm in +the specification file had to be the same. diff --git a/resources/tools/presentation/environment.py b/resources/tools/presentation/environment.py new file mode 100644 index 0000000000..05376e0e09 --- /dev/null +++ b/resources/tools/presentation/environment.py @@ -0,0 +1,157 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Environment + +Setting of the environment according to the specification specified in the +specification YAML file. +""" + +import os +import shutil +import logging + +from errors import PresentationError + + +class Environment(object): + """Setting of the environment: + - set environment variables, + - create directories. + """ + + def __init__(self, env, force=False): + """Initialization. + + :param env: Environment specification. + :param force: If True, remove old build(s) if present. + :type env: dict + :type force: bool + """ + + self._env = env + self._force = force + + @property + def environment(self): + """Getter. + + :returns: Environment settings. + :rtype: dict + """ + return self._env + + def _set_environment_variables(self): + """Set environment variables. + """ + logging.info("Setting the environment variables ...") + # logging.debug("Environment variables before:\n{}".format(os.environ)) + + count = 1 + + for var, value in self._env["configuration"].items(): + logging.debug(" {:3d} Setting the variable {} = {}". + format(count, var, value)) + os.environ[var] = str(value) + count += 1 + + for var, value in self._env["paths"].items(): + logging.debug(" {:3d} Setting the variable {} = {}". + format(count, var, value)) + os.environ[var] = str(value) + count += 1 + + for var, value in self._env["urls"].items(): + logging.debug(" {:3d} Setting the variable {} = {}". + format(count, var, value)) + os.environ[var] = str(value) + count += 1 + + # logging.debug("Environment variables after:\n{}".format(os.environ)) + logging.info("Done.") + + def _make_dirs(self): + """Create the directories specified in the 'make-dirs' part of + 'environment' section in the specification file. + + :raises: PresentationError if it is not possible to remove or create a + directory. + """ + + if self._force: + logging.info("Removing old build(s) ...") + for directory in self._env["build-dirs"]: + dir_to_remove = self._env["paths"][directory] + if os.path.isdir(dir_to_remove): + try: + shutil.rmtree(dir_to_remove) + logging.info(" Removed: {}".format(dir_to_remove)) + except OSError: + raise PresentationError("Cannot remove the directory " + "'{}'".format(dir_to_remove)) + logging.info("Done.") + + logging.info("Making directories ...") + + for directory in self._env["make-dirs"]: + dir_to_make = self._env["paths"][directory] + try: + if os.path.isdir(dir_to_make): + logging.warning("The directory '{}' exists, skipping.". + format(dir_to_make)) + else: + os.makedirs(dir_to_make) + logging.info(" Created: {}".format(dir_to_make)) + except OSError: + raise PresentationError("Cannot make the directory '{}'". + format(dir_to_make)) + + logging.info("Done.") + + def set_environment(self): + """Set the environment. + """ + + self._set_environment_variables() + self._make_dirs() + + +def clean_environment(env): + """Clean the environment. + + :param env: Environment specification. + :type env: dict + :raises: PresentationError if it is not possible to remove a directory. + """ + + logging.info("Cleaning the environment ...") + + if not env["remove-dirs"]: # None or empty + logging.info(" No directories to remove.") + return + + for directory in env["remove-dirs"]: + dir_to_remove = env["paths"][directory] + logging.info(" Removing the working directory {} ...". + format(dir_to_remove)) + if os.path.isdir(dir_to_remove): + try: + shutil.rmtree(dir_to_remove) + except OSError: + raise PresentationError("Cannot remove the directory '{}'". + format(dir_to_remove)) + else: + logging.warning("The directory '{}' does not exist.". + format(dir_to_remove)) + + logging.info("Done.") diff --git a/resources/tools/presentation/errors.py b/resources/tools/presentation/errors.py new file mode 100644 index 0000000000..0d8d5b9b33 --- /dev/null +++ b/resources/tools/presentation/errors.py @@ -0,0 +1,78 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of exceptions used in the Presentation and analytics layer. +""" + +import sys +import logging + + +class PresentationError(Exception): + """Exception(s) raised by the presentation module. + + When raising this exception, put this information to the message in this + order: + - short description of the encountered problem (parameter msg), + - relevant messages if there are any collected, e.g., from caught + exception (optional parameter details), + - relevant data if there are any collected (optional parameter details). + """ + + log_exception = {"DEBUG": logging.debug, + "INFO": logging.info, + "WARNING": logging.warning, + "ERROR": logging.error, + "CRITICAL": logging.critical} + + def __init__(self, msg, details='', level="CRITICAL"): + """Sets the exception message and the level. + + :param msg: Short description of the encountered problem. + :param details: Relevant messages if there are any collected, e.g., + from caught exception (optional parameter details), or relevant data if + there are any collected (optional parameter details). + :param level: Level of the error, possible choices are: "DEBUG", "INFO", + "WARNING", "ERROR" and "CRITICAL". + :type msg: str + :type details: str + :type level: str + """ + + super(PresentationError, self).__init__() + self._msg = msg + self._details = details + self._level = level + + try: + self.log_exception[self._level](self._msg) + if self._details: + self.log_exception[self._level](self._details) + except KeyError: + print("Wrong log level.") + sys.exit(1) + + def __repr__(self): + return repr(self._msg) + + def __str__(self): + return str(self._msg) + + @property + def level(self): + """Getter - logging level. + + :returns: Logging level. + :rtype: str + """ + return self._level diff --git a/resources/tools/presentation/fdio.svg b/resources/tools/presentation/fdio.svg new file mode 100644 index 0000000000..32dd070d36 --- /dev/null +++ b/resources/tools/presentation/fdio.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + diff --git a/resources/tools/presentation/generator_files.py b/resources/tools/presentation/generator_files.py new file mode 100644 index 0000000000..0989fc76c0 --- /dev/null +++ b/resources/tools/presentation/generator_files.py @@ -0,0 +1,109 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Algorithms to generate files. +""" + + +import logging + +from utils import get_files, get_rst_title_char + + +def generate_files(spec, data): + """Generate all files specified in the specification file. + + :param spec: Specification read from the specification file. + :param data: Data to process. + :type spec: Specification + :type data: InputData + """ + + logging.info("Generating the files ...") + for file_spec in spec.files: + try: + eval(file_spec["algorithm"])(file_spec, data) + except NameError: + logging.error("The algorithm '{0}' is not defined.". + format(file_spec["algorithm"])) + logging.info("Done.") + + +def file_test_results(file_spec, input_data): + """Generate the file(s) with algorithm: file_test_results specified in the + specification file. + + :param file_spec: File to generate. + :param input_data: Data to process. + :type file_spec: pandas.Series + :type input_data: InputData + """ + + def tests_in_suite(suite_name, tests): + """Check if the suite includes tests. + + :param suite_name: Name of the suite to be checked. + :param tests: Set of tests + :type suite_name: str + :type tests: pandas.Series + :returns: True if the suite includes tests. + :rtype: bool + """ + + for key in tests.keys(): + if suite_name == tests[key]["parent"]: + return True + return False + + file_name = "{0}{1}".format(file_spec["output-file"], + file_spec["output-file-ext"]) + rst_header = file_spec["file-header"] + rst_include_table = ("\n.. csv-table::\n" + " :header-rows: 1\n" + " :widths: auto\n" + " :align: center\n" + " :file: {file}\n\n") + + logging.info(" Generating the file {0} ...".format(file_name)) + + table_lst = get_files(file_spec["dir-tables"], ".csv", full_path=False) + if len(table_lst) == 0: + logging.error(" No tables to include in '{0}'. Skipping.". + format(file_spec["dir-tables"])) + return None + + job = file_spec["data"].keys()[0] + build = str(file_spec["data"][job][0]) + + logging.info(" Writing file '{0}'".format(file_name)) + + suites = input_data.suites(job, build)[file_spec["data-start-level"]:] + suites.sort_index(inplace=True) + + with open(file_name, "w") as file_handler: + file_handler.write(rst_header) + for suite_longname, suite in suites.iteritems(): + suite_name = suite["name"] + file_handler.write("\n{0}\n{1}\n".format( + suite_name, get_rst_title_char( + suite["level"] - file_spec["data-start-level"] - 1) * + len(suite_name))) + file_handler.write("\n{0}\n".format( + suite["doc"].replace('|br|', '\n\n -'))) + if tests_in_suite(suite_name, input_data.tests(job, build)): + for tbl_file in table_lst: + if suite_name in tbl_file: + file_handler.write(rst_include_table. + format(file=tbl_file)) + + logging.info(" Done.") diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py new file mode 100644 index 0000000000..9ef8574907 --- /dev/null +++ b/resources/tools/presentation/generator_plots.py @@ -0,0 +1,221 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Algorithms to generate plots. +""" + + +import logging +import pandas as pd +import plotly.offline as ploff +import plotly.graph_objs as plgo +from plotly.exceptions import PlotlyError + +from utils import mean + + +def generate_plots(spec, data): + """Generate all plots specified in the specification file. + + :param spec: Specification read from the specification file. + :param data: Data to process. + :type spec: Specification + :type data: InputData + """ + + logging.info("Generating the plots ...") + for index, plot in enumerate(spec.plots): + try: + logging.info(" Plot nr {0}:".format(index + 1)) + eval(plot["algorithm"])(plot, data) + except NameError: + logging.error("The algorithm '{0}' is not defined.". + format(plot["algorithm"])) + logging.info("Done.") + + +def plot_performance_box(plot, input_data): + """Generate the plot(s) with algorithm: table_detailed_test_results + specified in the specification file. + + :param plot: Plot to generate. + :param input_data: Data to process. + :type plot: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the plot {0} ...". + format(plot.get("title", ""))) + + # Transform the data + data = input_data.filter_data(plot) + if data is None: + logging.error("No data.") + return + + # Prepare the data for the plot + y_vals = dict() + for job in data: + for build in job: + for test in build: + if y_vals.get(test["parent"], None) is None: + y_vals[test["parent"]] = list() + try: + y_vals[test["parent"]].append(test["throughput"]["value"]) + except (KeyError, TypeError): + y_vals[test["parent"]].append(None) + + # Add None to the lists with missing data + max_len = 0 + for val in y_vals.values(): + if len(val) > max_len: + max_len = len(val) + for key, val in y_vals.items(): + if len(val) < max_len: + val.extend([None for _ in range(max_len - len(val))]) + + # Add plot traces + traces = list() + df = pd.DataFrame(y_vals) + df.head() + for i, col in enumerate(df.columns): + name = "{0}. {1}".format(i + 1, col.lower().replace('-ndrpdrdisc', '')) + traces.append(plgo.Box(x=[str(i + 1) + '.'] * len(df[col]), + y=df[col], + name=name, + **plot["traces"])) + + try: + # Create plot + plpl = plgo.Figure(data=traces, layout=plot["layout"]) + + # Export Plot + logging.info(" Writing file '{0}{1}'.". + format(plot["output-file"], plot["output-file-type"])) + ploff.plot(plpl, + show_link=False, auto_open=False, + filename='{0}{1}'.format(plot["output-file"], + plot["output-file-type"])) + except PlotlyError as err: + logging.error(" Finished with error: {}". + format(str(err).replace("\n", " "))) + return + + logging.info(" Done.") + + +def plot_latency_box(plot, input_data): + """Generate the plot(s) with algorithm: plot_latency_box + specified in the specification file. + + :param plot: Plot to generate. + :param input_data: Data to process. + :type plot: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the plot {0} ...". + format(plot.get("title", ""))) + + # Transform the data + data = input_data.filter_data(plot) + if data is None: + logging.error("No data.") + return + + # Prepare the data for the plot + y_tmp_vals = dict() + for job in data: + for build in job: + for test in build: + if y_tmp_vals.get(test["parent"], None) is None: + y_tmp_vals[test["parent"]] = [ + list(), # direction1, min + list(), # direction1, avg + list(), # direction1, max + list(), # direction2, min + list(), # direction2, avg + list() # direction2, max + ] + try: + y_tmp_vals[test["parent"]][0].append( + test["latency"]["direction1"]["50"]["min"]) + y_tmp_vals[test["parent"]][1].append( + test["latency"]["direction1"]["50"]["avg"]) + y_tmp_vals[test["parent"]][2].append( + test["latency"]["direction1"]["50"]["max"]) + y_tmp_vals[test["parent"]][3].append( + test["latency"]["direction2"]["50"]["min"]) + y_tmp_vals[test["parent"]][4].append( + test["latency"]["direction2"]["50"]["avg"]) + y_tmp_vals[test["parent"]][5].append( + test["latency"]["direction2"]["50"]["max"]) + except (KeyError, TypeError): + pass + + y_vals = dict() + for key, values in y_tmp_vals.items(): + y_vals[key] = list() + for val in values: + if val: + average = mean(val) + else: + average = None + y_vals[key].append(average) + y_vals[key].append(average) # Twice for plot.ly + + # Add plot traces + traces = list() + try: + df = pd.DataFrame(y_vals) + df.head() + except ValueError as err: + logging.error(" Finished with error: {}". + format(str(err).replace("\n", " "))) + return + + for i, col in enumerate(df.columns): + name = "{0}. {1}".format(i + 1, col.lower().replace('-ndrpdrdisc', '')) + traces.append(plgo.Box(x=['TGint1-to-SUT1-to-SUT2-to-TGint2', + 'TGint1-to-SUT1-to-SUT2-to-TGint2', + 'TGint1-to-SUT1-to-SUT2-to-TGint2', + 'TGint1-to-SUT1-to-SUT2-to-TGint2', + 'TGint1-to-SUT1-to-SUT2-to-TGint2', + 'TGint1-to-SUT1-to-SUT2-to-TGint2', + 'TGint2-to-SUT2-to-SUT1-to-TGint1', + 'TGint2-to-SUT2-to-SUT1-to-TGint1', + 'TGint2-to-SUT2-to-SUT1-to-TGint1', + 'TGint2-to-SUT2-to-SUT1-to-TGint1', + 'TGint2-to-SUT2-to-SUT1-to-TGint1', + 'TGint2-to-SUT2-to-SUT1-to-TGint1'], + y=df[col], + name=name, + **plot["traces"])) + + try: + # Create plot + logging.info(" Writing file '{0}{1}'.". + format(plot["output-file"], plot["output-file-type"])) + plpl = plgo.Figure(data=traces, layout=plot["layout"]) + + # Export Plot + ploff.plot(plpl, + show_link=False, auto_open=False, + filename='{0}{1}'.format(plot["output-file"], + plot["output-file-type"])) + except PlotlyError as err: + logging.error(" Finished with error: {}". + format(str(err).replace("\n", " "))) + return + + logging.info(" Done.") diff --git a/resources/tools/presentation/generator_report.py b/resources/tools/presentation/generator_report.py new file mode 100644 index 0000000000..8a3f8334a7 --- /dev/null +++ b/resources/tools/presentation/generator_report.py @@ -0,0 +1,252 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Report generation. +""" + +import subprocess +import logging +import datetime + +from os import makedirs, environ +from os.path import isdir +from shutil import copy, Error, make_archive + +from utils import get_files +from errors import PresentationError + + +# .css file for the html format of the report +THEME_OVERRIDES = """/* override table width restrictions */ +@media screen and (min-width: 767px) { + .wy-table-responsive table td, .wy-table-responsive table th { + white-space: normal !important; + } + + .wy-table-responsive { + font-size: small; + margin-bottom: 24px; + max-width: 100%; + overflow: visible !important; + } +} +.rst-content blockquote { + margin-left: 0px; + line-height: 18px; + margin-bottom: 0px; +} +""" + +# Command to build the html format of the report +HTML_BUILDER = 'sphinx-build -v -c . -a ' \ + '-b html -E ' \ + '-D release={release} ' \ + '-D version="{release} report - {date}" ' \ + '{working_dir} ' \ + '{build_dir}/' + +# Command to build the pdf format of the report +PDF_BUILDER = 'sphinx-build -v -c . -a ' \ + '-b latex -E ' \ + '-D release={release} ' \ + '-D version="{release} report - {date}" ' \ + '{working_dir} ' \ + '{build_dir}' + + +def generate_report(release, spec): + """Generate all formats and versions of the report. + + :param release: Release string of the product. + :param spec: Specification read from the specification file. + :type release: str + :type spec: Specification + """ + + logging.info("Generating the report ...") + + report = { + "html": generate_html_report, + "pdf": generate_pdf_report + } + + for report_format, versions in spec.output.items(): + report[report_format](release, spec, versions) + + archive_input_data(spec) + archive_report(spec) + + logging.info("Done.") + + +def generate_html_report(release, spec, versions): + """Generate html format of the report. + + :param release: Release string of the product. + :param spec: Specification read from the specification file. + :param versions: List of versions to generate. + :type release: str + :type spec: Specification + :type versions: list + """ + + logging.info(" Generating the html report, give me a few minutes, please " + "...") + + cmd = HTML_BUILDER.format( + release=release, + date=datetime.date.today().strftime('%d-%b-%Y'), + working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"], + build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"]) + _execute_command(cmd) + + with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \ + css_file: + css_file.write(THEME_OVERRIDES) + + with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \ + css_file: + css_file.write(THEME_OVERRIDES) + + logging.info(" Done.") + + +def generate_pdf_report(release, spec, versions): + """Generate html format of the report. + + :param release: Release string of the product. + :param spec: Specification read from the specification file. + :param versions: List of versions to generate. Not implemented yet. + :type release: str + :type spec: Specification + :type versions: list + """ + + logging.info(" Generating the pdf report, give me a few minutes, please " + "...") + + convert_plots = "xvfb-run -a wkhtmltopdf {html} {pdf}.pdf" + + # Convert PyPLOT graphs in HTML format to PDF. + plots = get_files(spec.environment["paths"]["DIR[STATIC,VPP]"], "html") + for plot in plots: + file_name = "{0}".format(plot.rsplit(".", 1)[0]) + cmd = convert_plots.format(html=plot, pdf=file_name) + _execute_command(cmd) + + # Generate the LaTeX documentation + build_dir = spec.environment["paths"]["DIR[BUILD,LATEX]"] + cmd = PDF_BUILDER.format( + release=release, + date=datetime.date.today().strftime('%d-%b-%Y'), + working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"], + build_dir=build_dir) + _execute_command(cmd) + + # Build pdf documentation + archive_dir = spec.environment["paths"]["DIR[STATIC,ARCH]"] + cmds = [ + 'cd {build_dir} && ' + 'pdflatex -shell-escape -interaction nonstopmode csit.tex || true'. + format(build_dir=build_dir), + 'cd {build_dir} && ' + 'pdflatex -interaction nonstopmode csit.tex || true'. + format(build_dir=build_dir), + 'cd {build_dir} && ' + 'cp csit.pdf ../{archive_dir}/csit_{release}.pdf'. + format(build_dir=build_dir, + archive_dir=archive_dir, + release=release) + ] + + for cmd in cmds: + _execute_command(cmd) + + logging.info(" Done.") + + +def archive_report(spec): + """Archive the report. + + :param spec: Specification read from the specification file. + :type spec: Specification + """ + + logging.info(" Archiving the report ...") + + make_archive("csit.report", + "gztar", + base_dir=spec.environment["paths"]["DIR[BUILD,HTML]"]) + + logging.info(" Done.") + + +def archive_input_data(spec): + """Archive the report. + + :param spec: Specification read from the specification file. + :type spec: Specification + :raises PresentationError: If it is not possible to archive the input data. + """ + + logging.info(" Archiving the input data files ...") + + if spec.is_debug: + extension = spec.debug["input-format"] + else: + extension = spec.input["file-format"] + data_files = get_files(spec.environment["paths"]["DIR[WORKING,DATA]"], + extension=extension) + dst = spec.environment["paths"]["DIR[STATIC,ARCH]"] + logging.info(" Destination: {0}".format(dst)) + + try: + if not isdir(dst): + makedirs(dst) + + for data_file in data_files: + logging.info(" Copying the file: {0} ...".format(data_file)) + copy(data_file, dst) + + except (Error, OSError) as err: + raise PresentationError("Not possible to archive the input data.", + str(err)) + + logging.info(" Done.") + + +def _execute_command(cmd): + """Execute the command in a subprocess and log the stdout and stderr. + + :param cmd: Command to execute. + :type cmd: str + :returns: Return code of the executed command. + :rtype: int + """ + + env = environ.copy() + proc = subprocess.Popen( + [cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + env=env) + + stdout, stderr = proc.communicate() + + logging.info(stdout) + logging.info(stderr) + + if proc.returncode != 0: + logging.error(" Command execution failed.") + return proc.returncode diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py new file mode 100644 index 0000000000..367e8c9878 --- /dev/null +++ b/resources/tools/presentation/generator_tables.py @@ -0,0 +1,251 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Algorithms to generate tables. +""" + + +import logging +from string import replace + +from errors import PresentationError +from utils import mean, stdev, relative_change + + +def generate_tables(spec, data): + """Generate all tables specified in the specification file. + + :param spec: Specification read from the specification file. + :param data: Data to process. + :type spec: Specification + :type data: InputData + """ + + logging.info("Generating the tables ...") + for table in spec.tables: + try: + eval(table["algorithm"])(table, data) + except NameError: + logging.error("The algorithm '{0}' is not defined.". + format(table["algorithm"])) + logging.info("Done.") + + +def table_details(table, input_data): + """Generate the table(s) with algorithm: table_detailed_test_results + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + # Transform the data + data = input_data.filter_data(table) + + # Prepare the header of the tables + header = list() + for column in table["columns"]: + header.append('"{0}"'.format(str(column["title"]).replace('"', '""'))) + + # Generate the data for the table according to the model in the table + # specification + + job = table["data"].keys()[0] + build = str(table["data"][job][0]) + for suite_longname, suite in input_data.suites(job, build).iteritems(): + # Generate data + suite_name = suite["name"] + table_lst = list() + for test in data[job][build].keys(): + if data[job][build][test]["parent"] in suite_name: + row_lst = list() + for column in table["columns"]: + try: + col_data = str(data[job][build][test][column["data"]. + split(" ")[1]]).replace('"', '""') + if column["data"].split(" ")[1] in ("vat-history", + "show-run"): + col_data = replace(col_data, " |br| ", "", + maxreplace=1) + col_data = " |prein| {0} |preout| ".\ + format(col_data[:-5]) + row_lst.append('"{0}"'.format(col_data)) + except KeyError: + row_lst.append("No data") + table_lst.append(row_lst) + + # Write the data to file + if table_lst: + file_name = "{0}_{1}{2}".format(table["output-file"], suite_name, + table["output-file-ext"]) + logging.info(" Writing file: '{}'".format(file_name)) + with open(file_name, "w") as file_handler: + file_handler.write(",".join(header) + "\n") + for item in table_lst: + file_handler.write(",".join(item) + "\n") + + logging.info(" Done.") + + +def table_performance_improvements(table, input_data): + """Generate the table(s) with algorithm: table_performance_improvements + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + def _write_line_to_file(file_handler, data): + """Write a line to the .csv file. + + :param file_handler: File handler for the csv file. It must be open for + writing text. + :param data: Item to be written to the file. + :type file_handler: BinaryIO + :type data: list + """ + + line_lst = list() + for item in data: + if isinstance(item["data"], str): + line_lst.append(item["data"]) + elif isinstance(item["data"], float): + line_lst.append("{:.1f}".format(item["data"])) + file_handler.write(",".join(line_lst) + "\n") + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + # Read the template + file_name = table.get("template", None) + if file_name: + try: + tmpl = _read_csv_template(file_name) + except PresentationError: + logging.error(" The template '{0}' does not exist. Skipping the " + "table.".format(file_name)) + return None + else: + logging.error("The template is not defined. Skipping the table.") + return None + + # Transform the data + data = input_data.filter_data(table) + + # Prepare the header of the tables + header = list() + for column in table["columns"]: + header.append(column["title"]) + + # Generate the data for the table according to the model in the table + # specification + tbl_lst = list() + for tmpl_item in tmpl: + tbl_item = list() + for column in table["columns"]: + cmd = column["data"].split(" ")[0] + args = column["data"].split(" ")[1:] + if cmd == "template": + try: + val = float(tmpl_item[int(args[0])]) + except ValueError: + val = tmpl_item[int(args[0])] + tbl_item.append({"data": val}) + elif cmd == "data": + job = args[0] + operation = args[1] + data_lst = list() + for build in data[job]: + try: + data_lst.append(float(build[tmpl_item[0]]["throughput"] + ["value"]) / 1000000) + except (KeyError, TypeError): + # No data, ignore + pass + if data_lst: + tbl_item.append({"data": eval(operation)(data_lst)}) + elif cmd == "operation": + operation = args[0] + nr1 = tbl_item[int(args[1])]["data"] + nr2 = tbl_item[int(args[2])]["data"] + if nr1 and nr2: + tbl_item.append({"data": eval(operation)(nr1, nr2)}) + else: + tbl_item.append({"data": None}) + else: + logging.error("Not supported command {0}. Skipping the table.". + format(cmd)) + return None + tbl_lst.append(tbl_item) + + # Sort the table according to the relative change + tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True) + + # Create the tables and write them to the files + file_names = [ + "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]), + "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]), + "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]), + "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"]) + ] + + for file_name in file_names: + logging.info(" Writing the file '{0}'".format(file_name)) + with open(file_name, "w") as file_handler: + file_handler.write(",".join(header) + "\n") + for item in tbl_lst: + if "ndr_top" in file_name \ + and "ndr" in item[1]["data"] \ + and item[-1]["data"] >= 10: + _write_line_to_file(file_handler, item) + elif "pdr_top" in file_name \ + and "pdr" in item[1]["data"] \ + and item[-1]["data"] >= 10: + _write_line_to_file(file_handler, item) + elif "ndr_low" in file_name \ + and "ndr" in item[1]["data"] \ + and item[-1]["data"] < 10: + _write_line_to_file(file_handler, item) + elif "pdr_low" in file_name \ + and "pdr" in item[1]["data"] \ + and item[-1]["data"] < 10: + _write_line_to_file(file_handler, item) + + logging.info(" Done.") + + +def _read_csv_template(file_name): + """Read the template from a .csv file. + + :param file_name: Name / full path / relative path of the file to read. + :type file_name: str + :returns: Data from the template as list (lines) of lists (items on line). + :rtype: list + :raises: PresentationError if it is not possible to read the file. + """ + + try: + with open(file_name, 'r') as csv_file: + tmpl_data = list() + for line in csv_file: + tmpl_data.append(line[:-1].split(",")) + return tmpl_data + except IOError as err: + raise PresentationError(str(err), level="ERROR") diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py new file mode 100644 index 0000000000..d57d1545ea --- /dev/null +++ b/resources/tools/presentation/input_data_files.py @@ -0,0 +1,204 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Inputs +Download all data. +""" + +import logging + +from os import rename, remove +from os.path import join, getsize +from shutil import move +from zipfile import ZipFile, is_zipfile, BadZipfile +from httplib import responses +from requests import get, codes, RequestException, Timeout, TooManyRedirects, \ + HTTPError, ConnectionError + +from errors import PresentationError + + +# Chunk size used for file download +CHUNK_SIZE = 512 + +# Separator used in file names +SEPARATOR = "__" + + +def download_data_files(spec): + """Download all data specified in the specification file in the section + type: input --> builds. + + :param spec: Specification. + :type spec: Specification + :raises: PresentationError if there is no url defined for the job. + """ + + for job, builds in spec.builds.items(): + for build in builds: + if job.startswith("csit-"): + url = spec.environment["urls"]["URL[JENKINS,CSIT]"] + elif job.startswith("hc2vpp-"): + url = spec.environment["urls"]["URL[JENKINS,HC]"] + else: + raise PresentationError("No url defined for the job '{}'.". + format(job)) + file_name = spec.input["file-name"] + full_name = spec.input["download-path"].\ + format(job=job, build=build["build"], filename=file_name) + url = "{0}/{1}".format(url, full_name) + new_name = join( + spec.environment["paths"]["DIR[WORKING,DATA]"], + "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR, + build=build["build"], + name=file_name)) + + logging.info("Downloading the file '{0}' to '{1}'.". + format(url, new_name)) + + status = "failed" + try: + response = get(url, stream=True) + code = response.status_code + if code != codes["OK"]: + logging.error("{0}: {1}".format(code, responses[code])) + spec.set_input_state(job, build["build"], "not found") + continue + + file_handle = open(new_name, "wb") + for chunk in response.iter_content(chunk_size=CHUNK_SIZE): + if chunk: + file_handle.write(chunk) + file_handle.close() + + expected_length = None + try: + expected_length = int(response.headers["Content-Length"]) + logging.debug(" Expected file size: {0}B". + format(expected_length)) + except KeyError: + logging.debug(" No information about expected size.") + + real_length = getsize(new_name) + logging.debug(" Downloaded size: {0}B".format(real_length)) + + if expected_length: + if real_length == expected_length: + status = "downloaded" + logging.info("{0}: {1}".format(code, responses[code])) + else: + logging.error("The file size differs from the expected " + "size.") + else: + status = "downloaded" + logging.info("{0}: {1}".format(code, responses[code])) + + except ConnectionError as err: + logging.error("Not possible to connect to '{0}'.".format(url)) + logging.debug(err) + except HTTPError as err: + logging.error("Invalid HTTP response from '{0}'.".format(url)) + logging.debug(err) + except TooManyRedirects as err: + logging.error("Request exceeded the configured number " + "of maximum re-directions.") + logging.debug(err) + except Timeout as err: + logging.error("Request timed out.") + logging.debug(err) + except RequestException as err: + logging.error("Unexpected HTTP request exception.") + logging.debug(err) + except (IOError, ValueError, KeyError) as err: + logging.error("Download failed.") + logging.debug("Reason: {0}".format(err)) + + spec.set_input_state(job, build["build"], status) + spec.set_input_file_name(job, build["build"], new_name) + + if status == "failed": + logging.info("Removing the file '{0}'".format(new_name)) + try: + remove(new_name) + except OSError as err: + logging.warning(str(err)) + spec.set_input_file_name(job, build["build"], None) + + unzip_files(spec) + + +def unzip_files(spec): + """Unzip downloaded zip files + + :param spec: Specification. + :type spec: Specification + :raises: PresentationError if the zip file does not exist or it is not a + zip file. + """ + + if spec.is_debug: + data_file = spec.debug["extract"] + else: + data_file = spec.input["extract"] + + for job, builds in spec.builds.items(): + for build in builds: + if build["status"] == "failed" or build["status"] == "not found": + continue + try: + status = "failed" + directory = spec.environment["paths"]["DIR[WORKING,DATA]"] + file_name = join(build["file-name"]) + + if build["status"] == "downloaded" and is_zipfile(file_name): + logging.info("Unziping: '{0}' from '{1}'.". + format(data_file, file_name)) + new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2], + SEPARATOR, + data_file.split("/")[-1]) + try: + with ZipFile(file_name, 'r') as zip_file: + zip_file.extract(data_file, directory) + logging.info("Moving {0} to {1} ...". + format(join(directory, data_file), + directory)) + move(join(directory, data_file), directory) + logging.info("Renaming the file '{0}' to '{1}'". + format(join(directory, + data_file.split("/")[-1]), + new_name)) + rename(join(directory, data_file.split("/")[-1]), + new_name) + status = "unzipped" + spec.set_input_state(job, build["build"], status) + spec.set_input_file_name(job, build["build"], + new_name) + except (BadZipfile, RuntimeError) as err: + logging.error("Failed to unzip the file '{0}': {1}.". + format(file_name, str(err))) + except OSError as err: + logging.error("Failed to rename the file '{0}': {1}.". + format(data_file, str(err))) + finally: + if status == "failed": + spec.set_input_file_name(job, build["build"], + None) + else: + raise PresentationError("The file '{0}' does not exist or " + "it is not a zip file". + format(file_name)) + + spec.set_input_state(job, build["build"], status) + + except KeyError: + pass diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py new file mode 100644 index 0000000000..1b8c3ccb47 --- /dev/null +++ b/resources/tools/presentation/input_data_parser.py @@ -0,0 +1,860 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Data pre-processing + +- extract data from output.xml files generated by Jenkins jobs and store in + pandas' Series, +- provide access to the data. +""" + +import re +import pandas as pd +import logging + +from robot.api import ExecutionResult, ResultVisitor +from collections import OrderedDict +from string import replace + + +class ExecutionChecker(ResultVisitor): + """Class to traverse through the test suite structure. + + The functionality implemented in this class generates a json structure: + + Performance tests: + + { + "metadata": { # Optional + "version": "VPP version", + "job": "Jenkins job name", + "build": "Information about the build" + }, + "suites": { + "Suite name 1": { + "doc": "Suite 1 documentation", + "parent": "Suite 1 parent", + "level": "Level of the suite in the suite hierarchy" + } + "Suite name N": { + "doc": "Suite N documentation", + "parent": "Suite 2 parent", + "level": "Level of the suite in the suite hierarchy" + } + } + "tests": { + "ID": { + "name": "Test name", + "parent": "Name of the parent of the test", + "doc": "Test documentation" + "msg": "Test message" + "tags": ["tag 1", "tag 2", "tag n"], + "type": "PDR" | "NDR", + "throughput": { + "value": int, + "unit": "pps" | "bps" | "percentage" + }, + "latency": { + "direction1": { + "100": { + "min": int, + "avg": int, + "max": int + }, + "50": { # Only for NDR + "min": int, + "avg": int, + "max": int + }, + "10": { # Only for NDR + "min": int, + "avg": int, + "max": int + } + }, + "direction2": { + "100": { + "min": int, + "avg": int, + "max": int + }, + "50": { # Only for NDR + "min": int, + "avg": int, + "max": int + }, + "10": { # Only for NDR + "min": int, + "avg": int, + "max": int + } + } + }, + "lossTolerance": "lossTolerance", # Only for PDR + "vat-history": "DUT1 and DUT2 VAT History" + }, + "show-run": "Show Run" + }, + "ID" { + # next test + } + } + } + + Functional tests: + + + { + "metadata": { # Optional + "version": "VPP version", + "job": "Jenkins job name", + "build": "Information about the build" + }, + "suites": { + "Suite name 1": { + "doc": "Suite 1 documentation", + "parent": "Suite 1 parent", + "level": "Level of the suite in the suite hierarchy" + } + "Suite name N": { + "doc": "Suite N documentation", + "parent": "Suite 2 parent", + "level": "Level of the suite in the suite hierarchy" + } + } + "tests": { + "ID": { + "name": "Test name", + "parent": "Name of the parent of the test", + "doc": "Test documentation" + "msg": "Test message" + "tags": ["tag 1", "tag 2", "tag n"], + "vat-history": "DUT1 and DUT2 VAT History" + "show-run": "Show Run" + "status": "PASS" | "FAIL" + }, + "ID" { + # next test + } + } + } + + .. note:: ID is the lowercase full path to the test. + """ + + REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)') + + REGEX_LAT_NDR = re.compile(r'^[\D\d]*' + r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+/-?\d+)\',' + r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n' + r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\',' + r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n' + r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\',' + r'\s\'(-?\d+/-?\d+/-?\d+)\'\]') + + REGEX_LAT_PDR = re.compile(r'^[\D\d]*' + r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\',' + r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*') + + REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s' + r'[\D\d]*') + + REGEX_VERSION = re.compile(r"(stdout: 'vat# vat# Version:)(\s*)(.*)") + + def __init__(self, **metadata): + """Initialisation. + + :param metadata: Key-value pairs to be included in "metadata" part of + JSON structure. + :type metadata: dict + """ + + # Type of message to parse out from the test messages + self._msg_type = None + + # VPP version + self._version = None + + # Number of VAT History messages found: + # 0 - no message + # 1 - VAT History of DUT1 + # 2 - VAT History of DUT2 + self._lookup_kw_nr = 0 + self._vat_history_lookup_nr = 0 + + # Number of Show Running messages found + # 0 - no message + # 1 - Show run message found + self._show_run_lookup_nr = 0 + + # Test ID of currently processed test- the lowercase full path to the + # test + self._test_ID = None + + # The main data structure + self._data = { + "metadata": OrderedDict(), + "suites": OrderedDict(), + "tests": OrderedDict() + } + + # Save the provided metadata + for key, val in metadata.items(): + self._data["metadata"][key] = val + + # Dictionary defining the methods used to parse different types of + # messages + self.parse_msg = { + "setup-version": self._get_version, + "teardown-vat-history": self._get_vat_history, + "teardown-show-runtime": self._get_show_run + } + + @property + def data(self): + """Getter - Data parsed from the XML file. + + :returns: Data parsed from the XML file. + :rtype: dict + """ + return self._data + + def _get_version(self, msg): + """Called when extraction of VPP version is required. + + :param msg: Message to process. + :type msg: Message + :returns: Nothing. + """ + + if msg.message.count("stdout: 'vat# vat# Version:"): + self._version = str(re.search(self.REGEX_VERSION, msg.message). + group(3)) + self._data["metadata"]["version"] = self._version + self._msg_type = None + + logging.debug(" VPP version: {0}".format(self._version)) + + def _get_vat_history(self, msg): + """Called when extraction of VAT command history is required. + + :param msg: Message to process. + :type msg: Message + :returns: Nothing. + """ + if msg.message.count("VAT command history:"): + self._vat_history_lookup_nr += 1 + if self._vat_history_lookup_nr == 1: + self._data["tests"][self._test_ID]["vat-history"] = str() + else: + self._msg_type = None + text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} " + "VAT command history:", "", msg.message, count=1). \ + replace("\n\n", "\n").replace('\n', ' |br| ').\ + replace('\r', '').replace('"', "'") + + self._data["tests"][self._test_ID]["vat-history"] += " |br| " + self._data["tests"][self._test_ID]["vat-history"] += \ + "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text + + def _get_show_run(self, msg): + """Called when extraction of VPP operational data (output of CLI command + Show Runtime) is required. + + :param msg: Message to process. + :type msg: Message + :returns: Nothing. + """ + if msg.message.count("vat# Thread "): + self._show_run_lookup_nr += 1 + if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1: + self._data["tests"][self._test_ID]["show-run"] = str() + if self._lookup_kw_nr > 1: + self._msg_type = None + if self._show_run_lookup_nr == 1: + text = msg.message.replace("vat# ", "").\ + replace("return STDOUT ", "").replace("\n\n", "\n").\ + replace('\n', ' |br| ').\ + replace('\r', '').replace('"', "'") + try: + self._data["tests"][self._test_ID]["show-run"] += " |br| " + self._data["tests"][self._test_ID]["show-run"] += \ + "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text + except KeyError: + pass + + def _get_latency(self, msg, test_type): + """Get the latency data from the test message. + + :param msg: Message to be parsed. + :param test_type: Type of the test - NDR or PDR. + :type msg: str + :type test_type: str + :returns: Latencies parsed from the message. + :rtype: dict + """ + + if test_type == "NDR": + groups = re.search(self.REGEX_LAT_NDR, msg) + groups_range = range(1, 7) + elif test_type == "PDR": + groups = re.search(self.REGEX_LAT_PDR, msg) + groups_range = range(1, 3) + else: + return {} + + latencies = list() + for idx in groups_range: + try: + lat = [int(item) for item in str(groups.group(idx)).split('/')] + except (AttributeError, ValueError): + lat = [-1, -1, -1] + latencies.append(lat) + + keys = ("min", "avg", "max") + latency = { + "direction1": { + }, + "direction2": { + } + } + + latency["direction1"]["100"] = dict(zip(keys, latencies[0])) + latency["direction2"]["100"] = dict(zip(keys, latencies[1])) + if test_type == "NDR": + latency["direction1"]["50"] = dict(zip(keys, latencies[2])) + latency["direction2"]["50"] = dict(zip(keys, latencies[3])) + latency["direction1"]["10"] = dict(zip(keys, latencies[4])) + latency["direction2"]["10"] = dict(zip(keys, latencies[5])) + + return latency + + def visit_suite(self, suite): + """Implements traversing through the suite and its direct children. + + :param suite: Suite to process. + :type suite: Suite + :returns: Nothing. + """ + if self.start_suite(suite) is not False: + suite.suites.visit(self) + suite.tests.visit(self) + self.end_suite(suite) + + def start_suite(self, suite): + """Called when suite starts. + + :param suite: Suite to process. + :type suite: Suite + :returns: Nothing. + """ + + try: + parent_name = suite.parent.name + except AttributeError: + return + + doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\ + replace('\r', '').replace('*[', ' |br| *[').replace("*", "**") + doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1) + + self._data["suites"][suite.longname.lower().replace('"', "'"). + replace(" ", "_")] = { + "name": suite.name.lower(), + "doc": doc_str, + "parent": parent_name, + "level": len(suite.longname.split(".")) + } + + suite.keywords.visit(self) + + def end_suite(self, suite): + """Called when suite ends. + + :param suite: Suite to process. + :type suite: Suite + :returns: Nothing. + """ + pass + + def visit_test(self, test): + """Implements traversing through the test. + + :param test: Test to process. + :type test: Test + :returns: Nothing. + """ + if self.start_test(test) is not False: + test.keywords.visit(self) + self.end_test(test) + + def start_test(self, test): + """Called when test starts. + + :param test: Test to process. + :type test: Test + :returns: Nothing. + """ + + tags = [str(tag) for tag in test.tags] + test_result = dict() + test_result["name"] = test.name.lower() + test_result["parent"] = test.parent.name.lower() + test_result["tags"] = tags + doc_str = test.doc.replace('"', "'").replace('\n', ' '). \ + replace('\r', '').replace('[', ' |br| [') + test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1) + test_result["msg"] = test.message.replace('\n', ' |br| '). \ + replace('\r', '').replace('"', "'") + if test.status == "PASS" and "NDRPDRDISC" in tags: + + if "NDRDISC" in tags: + test_type = "NDR" + elif "PDRDISC" in tags: + test_type = "PDR" + else: + return + + try: + rate_value = str(re.search( + self.REGEX_RATE, test.message).group(1)) + except AttributeError: + rate_value = "-1" + try: + rate_unit = str(re.search( + self.REGEX_RATE, test.message).group(2)) + except AttributeError: + rate_unit = "-1" + + test_result["type"] = test_type + test_result["throughput"] = dict() + test_result["throughput"]["value"] = int(rate_value.split('.')[0]) + test_result["throughput"]["unit"] = rate_unit + test_result["latency"] = self._get_latency(test.message, test_type) + if test_type == "PDR": + test_result["lossTolerance"] = str(re.search( + self.REGEX_TOLERANCE, test.message).group(1)) + else: + test_result["status"] = test.status + + self._test_ID = test.longname.lower() + self._data["tests"][self._test_ID] = test_result + + def end_test(self, test): + """Called when test ends. + + :param test: Test to process. + :type test: Test + :returns: Nothing. + """ + pass + + def visit_keyword(self, keyword): + """Implements traversing through the keyword and its child keywords. + + :param keyword: Keyword to process. + :type keyword: Keyword + :returns: Nothing. + """ + if self.start_keyword(keyword) is not False: + self.end_keyword(keyword) + + def start_keyword(self, keyword): + """Called when keyword starts. Default implementation does nothing. + + :param keyword: Keyword to process. + :type keyword: Keyword + :returns: Nothing. + """ + try: + if keyword.type == "setup": + self.visit_setup_kw(keyword) + elif keyword.type == "teardown": + self._lookup_kw_nr = 0 + self.visit_teardown_kw(keyword) + except AttributeError: + pass + + def end_keyword(self, keyword): + """Called when keyword ends. Default implementation does nothing. + + :param keyword: Keyword to process. + :type keyword: Keyword + :returns: Nothing. + """ + pass + + def visit_setup_kw(self, setup_kw): + """Implements traversing through the teardown keyword and its child + keywords. + + :param setup_kw: Keyword to process. + :type setup_kw: Keyword + :returns: Nothing. + """ + for keyword in setup_kw.keywords: + if self.start_setup_kw(keyword) is not False: + self.visit_setup_kw(keyword) + self.end_setup_kw(keyword) + + def start_setup_kw(self, setup_kw): + """Called when teardown keyword starts. Default implementation does + nothing. + + :param setup_kw: Keyword to process. + :type setup_kw: Keyword + :returns: Nothing. + """ + if setup_kw.name.count("Vpp Show Version Verbose") \ + and not self._version: + self._msg_type = "setup-version" + setup_kw.messages.visit(self) + + def end_setup_kw(self, setup_kw): + """Called when keyword ends. Default implementation does nothing. + + :param setup_kw: Keyword to process. + :type setup_kw: Keyword + :returns: Nothing. + """ + pass + + def visit_teardown_kw(self, teardown_kw): + """Implements traversing through the teardown keyword and its child + keywords. + + :param teardown_kw: Keyword to process. + :type teardown_kw: Keyword + :returns: Nothing. + """ + for keyword in teardown_kw.keywords: + if self.start_teardown_kw(keyword) is not False: + self.visit_teardown_kw(keyword) + self.end_teardown_kw(keyword) + + def start_teardown_kw(self, teardown_kw): + """Called when teardown keyword starts. Default implementation does + nothing. + + :param teardown_kw: Keyword to process. + :type teardown_kw: Keyword + :returns: Nothing. + """ + + if teardown_kw.name.count("Show Vat History On All Duts"): + self._vat_history_lookup_nr = 0 + self._msg_type = "teardown-vat-history" + elif teardown_kw.name.count("Vpp Show Runtime"): + self._lookup_kw_nr += 1 + self._show_run_lookup_nr = 0 + self._msg_type = "teardown-show-runtime" + + if self._msg_type: + teardown_kw.messages.visit(self) + + def end_teardown_kw(self, teardown_kw): + """Called when keyword ends. Default implementation does nothing. + + :param teardown_kw: Keyword to process. + :type teardown_kw: Keyword + :returns: Nothing. + """ + pass + + def visit_message(self, msg): + """Implements visiting the message. + + :param msg: Message to process. + :type msg: Message + :returns: Nothing. + """ + if self.start_message(msg) is not False: + self.end_message(msg) + + def start_message(self, msg): + """Called when message starts. Get required information from messages: + - VPP version. + + :param msg: Message to process. + :type msg: Message + :returns: Nothing. + """ + + if self._msg_type: + self.parse_msg[self._msg_type](msg) + + def end_message(self, msg): + """Called when message ends. Default implementation does nothing. + + :param msg: Message to process. + :type msg: Message + :returns: Nothing. + """ + pass + + +class InputData(object): + """Input data + + The data is extracted from output.xml files generated by Jenkins jobs and + stored in pandas' DataFrames. + + The data structure: + - job name + - build number + - metadata + - job + - build + - vpp version + - suites + - tests + - ID: test data (as described in ExecutionChecker documentation) + """ + + def __init__(self, spec): + """Initialization. + + :param spec: Specification. + :type spec: Specification + """ + + # Specification: + self._cfg = spec + + # Data store: + self._input_data = None + + @property + def data(self): + """Getter - Input data. + + :returns: Input data + :rtype: pandas.Series + """ + return self._input_data + + def metadata(self, job, build): + """Getter - metadata + + :param job: Job which metadata we want. + :param build: Build which metadata we want. + :type job: str + :type build: str + :returns: Metadata + :rtype: pandas.Series + """ + + return self.data[job][build]["metadata"] + + def suites(self, job, build): + """Getter - suites + + :param job: Job which suites we want. + :param build: Build which suites we want. + :type job: str + :type build: str + :returns: Suites. + :rtype: pandas.Series + """ + + return self.data[job][str(build)]["suites"] + + def tests(self, job, build): + """Getter - tests + + :param job: Job which tests we want. + :param build: Build which tests we want. + :type job: str + :type build: str + :returns: Tests. + :rtype: pandas.Series + """ + + return self.data[job][build]["tests"] + + @staticmethod + def _parse_tests(job, build): + """Process data from robot output.xml file and return JSON structured + data. + + :param job: The name of job which build output data will be processed. + :param build: The build which output data will be processed. + :type job: str + :type build: dict + :returns: JSON data structure. + :rtype: dict + """ + + with open(build["file-name"], 'r') as data_file: + result = ExecutionResult(data_file) + checker = ExecutionChecker(job=job, build=build) + result.visit(checker) + + return checker.data + + def read_data(self): + """Parse input data from input files and store in pandas' Series. + """ + + logging.info("Parsing input files ...") + + job_data = dict() + for job, builds in self._cfg.builds.items(): + logging.info(" Extracting data from the job '{0}' ...'". + format(job)) + builds_data = dict() + for build in builds: + if build["status"] == "failed" \ + or build["status"] == "not found": + continue + logging.info(" Extracting data from the build '{0}'". + format(build["build"])) + logging.info(" Processing the file '{0}'". + format(build["file-name"])) + data = InputData._parse_tests(job, build) + + build_data = pd.Series({ + "metadata": pd.Series(data["metadata"].values(), + index=data["metadata"].keys()), + "suites": pd.Series(data["suites"].values(), + index=data["suites"].keys()), + "tests": pd.Series(data["tests"].values(), + index=data["tests"].keys()), + }) + builds_data[str(build["build"])] = build_data + logging.info(" Done.") + + job_data[job] = pd.Series(builds_data.values(), + index=builds_data.keys()) + logging.info(" Done.") + + self._input_data = pd.Series(job_data.values(), index=job_data.keys()) + logging.info("Done.") + + @staticmethod + def _end_of_tag(tag_filter, start=0, closer="'"): + """Return the index of character in the string which is the end of tag. + + :param tag_filter: The string where the end of tag is being searched. + :param start: The index where the searching is stated. + :param closer: The character which is the tag closer. + :type tag_filter: str + :type start: int + :type closer: str + :returns: The index of the tag closer. + :rtype: int + """ + + try: + idx_opener = tag_filter.index(closer, start) + return tag_filter.index(closer, idx_opener + 1) + except ValueError: + return None + + @staticmethod + def _condition(tag_filter): + """Create a conditional statement from the given tag filter. + + :param tag_filter: Filter based on tags from the element specification. + :type tag_filter: str + :returns: Conditional statement which can be evaluated. + :rtype: str + """ + + index = 0 + while True: + index = InputData._end_of_tag(tag_filter, index) + if index is None: + return tag_filter + index += 1 + tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:] + + def filter_data(self, element, params=None, data_set="tests"): + """Filter required data from the given jobs and builds. + + The output data structure is: + + - job 1 + - build 1 + - test (suite) 1 ID: + - param 1 + - param 2 + ... + - param n + ... + - test (suite) n ID: + ... + ... + - build n + ... + - job n + + :param element: Element which will use the filtered data. + :param params: Parameters which will be included in the output. If None, + all parameters are included. + :param data_set: The set of data to be filtered: tests, suites, + metadata. + :type element: pandas.Series + :type params: list + :type data_set: str + :returns: Filtered data. + :rtype pandas.Series + """ + + logging.info(" Creating the data set for the {0} '{1}'.". + format(element["type"], element.get("title", ""))) + + try: + if element["filter"] in ("all", "template"): + cond = "True" + else: + cond = InputData._condition(element["filter"]) + logging.debug(" Filter: {0}".format(cond)) + except KeyError: + logging.error(" No filter defined.") + return None + + if params is None: + params = element.get("parameters", None) + + data = pd.Series() + try: + for job, builds in element["data"].items(): + data[job] = pd.Series() + for build in builds: + data[job][str(build)] = pd.Series() + for test_ID, test_data in \ + self.data[job][str(build)][data_set].iteritems(): + if eval(cond, {"tags": test_data.get("tags", "")}): + data[job][str(build)][test_ID] = pd.Series() + if params is None: + for param, val in test_data.items(): + data[job][str(build)][test_ID][param] = val + else: + for param in params: + try: + data[job][str(build)][test_ID][param] =\ + test_data[param] + except KeyError: + data[job][str(build)][test_ID][param] =\ + "No Data" + return data + + except (KeyError, IndexError, ValueError) as err: + logging.error(" Missing mandatory parameter in the element " + "specification.", err) + return None + except SyntaxError: + logging.error(" The filter '{0}' is not correct. Check if all " + "tags are enclosed by apostrophes.".format(cond)) + return None diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py new file mode 100644 index 0000000000..86b709c5be --- /dev/null +++ b/resources/tools/presentation/pal.py @@ -0,0 +1,122 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CSIT Presentation and analytics layer. +""" + +import sys +import argparse +import logging + +from errors import PresentationError +from environment import Environment, clean_environment +from specification_parser import Specification +from input_data_files import download_data_files, unzip_files +from input_data_parser import InputData +from generator_tables import generate_tables +from generator_plots import generate_plots +from generator_files import generate_files +from static_content import prepare_static_content +from generator_report import generate_report + + +def parse_args(): + """Parse arguments from cmd line. + + :returns: Parsed arguments. + :rtype: ArgumentParser + """ + + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse. + RawDescriptionHelpFormatter) + parser.add_argument("-s", "--specification", + required=True, + type=argparse.FileType('r'), + help="Specification YAML file.") + parser.add_argument("-r", "--release", + default="master", + type=str, + help="Release string of the product.") + parser.add_argument("-l", "--logging", + choices=["DEBUG", "INFO", "WARNING", + "ERROR", "CRITICAL"], + default="ERROR", + help="Logging level.") + parser.add_argument("-f", "--force", + action='store_true', + help="Force removing the old build(s) if present.") + + return parser.parse_args() + + +def main(): + """Main function.""" + + log_levels = {"NOTSET": logging.NOTSET, + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + "CRITICAL": logging.CRITICAL} + + args = parse_args() + logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s', + datefmt='%Y/%m/%d %H:%M:%S', + level=log_levels[args.logging]) + + logging.info("Application started.") + try: + spec = Specification(args.specification) + spec.read_specification() + except PresentationError: + logging.critical("Finished with error.") + sys.exit(1) + + try: + env = Environment(spec.environment, args.force) + env.set_environment() + + if spec.is_debug: + if spec.debug["input-format"] == "zip": + unzip_files(spec) + else: + download_data_files(spec) + + prepare_static_content(spec) + + data = InputData(spec) + data.read_data() + + generate_tables(spec, data) + generate_plots(spec, data) + generate_files(spec, data) + generate_report(args.release, spec) + + logging.info("Successfully finished.") + + except (KeyError, ValueError, PresentationError) as err: + logging.info("Finished with an error.") + logging.critical(str(err)) + except Exception as err: + logging.info("Finished with an error.") + logging.critical(str(err)) + + finally: + if spec is not None and not spec.is_debug: + clean_environment(spec.environment) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/resources/tools/presentation/requirements.txt b/resources/tools/presentation/requirements.txt new file mode 100644 index 0000000000..5765dc4ef0 --- /dev/null +++ b/resources/tools/presentation/requirements.txt @@ -0,0 +1,10 @@ +Sphinx +sphinx-rtd-theme +robotframework==2.9.2 +sphinxcontrib-programoutput +PyYAML +pytz +python-dateutil +numpy +pandas +plotly diff --git a/resources/tools/presentation/run_report.sh b/resources/tools/presentation/run_report.sh new file mode 100755 index 0000000000..34d6c5d7be --- /dev/null +++ b/resources/tools/presentation/run_report.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -x + +RELEASE=$1 + +# set default values in config array +typeset -A CFG +typeset -A DIR + +DIR[WORKING]=_tmp +CFG[BLD_LATEX]=1 + +# Install system dependencies +sudo apt-get -y update +sudo apt-get -y install libxml2 libxml2-dev libxslt-dev build-essential \ + zlib1g-dev unzip + +if [[ ${CFG[BLD_LATEX]} -eq 1 ]] ; +then + sudo apt-get -y install xvfb texlive-latex-recommended \ + texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra latexmk wkhtmltopdf inkscape + sudo sed -i.bak 's/^\(main_memory\s=\s\).*/\110000000/' /usr/share/texlive/texmf-dist/web2c/texmf.cnf +fi + +# Clean-up when finished +trap 'rm -rf ${DIR[WORKING]}; exit' EXIT +trap 'rm -rf ${DIR[WORKING]}; exit' ERR + +# Create working directories +mkdir ${DIR[WORKING]} + +# Create virtual environment +virtualenv ${DIR[WORKING]}/env +. ${DIR[WORKING]}/env/bin/activate + +# Install python dependencies: +pip install -r requirements.txt + +export PYTHONPATH=`pwd` + +python pal.py \ + --specification specification.yaml \ + --release ${RELEASE} \ + --logging INFO \ + --force diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml new file mode 100644 index 0000000000..d5968baf44 --- /dev/null +++ b/resources/tools/presentation/specification.yaml @@ -0,0 +1,5065 @@ +- + type: "environment" + configuration: + # Debug mode: + # - Skip: + # - Download of input data files + # - Do: + # - Read data from given zip / xml files + # - Set the configuration as it is done in normal mode + # If the section "type: debug" is missing, CFG[DEBUG] is set to 0. + CFG[DEBUG]: 0 + + paths: + # Top level directories: + ## Working directory + DIR[WORKING]: "_tmp" + ## Build directories + DIR[BUILD,HTML]: "_build" + DIR[BUILD,LATEX]: "_build_latex" + + # Static .rst files + DIR[RST]: "../../../docs/report" + + # Working directories + ## Input data files (.zip, .xml) + DIR[WORKING,DATA]: "{DIR[WORKING]}/data" + ## Static source files from git + DIR[WORKING,SRC]: "{DIR[WORKING]}/src" + DIR[WORKING,SRC,STATIC]: "{DIR[WORKING,SRC]}/_static" + + # Static html content + DIR[STATIC]: "{DIR[BUILD,HTML]}/_static" + DIR[STATIC,VPP]: "{DIR[STATIC]}/vpp" + DIR[STATIC,DPDK]: "{DIR[STATIC]}/dpdk" + DIR[STATIC,ARCH]: "{DIR[STATIC]}/archive" + + # Detailed test results + DIR[DTR]: "{DIR[WORKING,SRC]}/detailed_test_results" + DIR[DTR,PERF,DPDK]: "{DIR[DTR]}/dpdk_performance_results" + DIR[DTR,PERF,VPP]: "{DIR[DTR]}/vpp_performance_results" + DIR[DTR,PERF,HC]: "{DIR[DTR]}/honeycomb_performance_results" + DIR[DTR,FUNC,VPP]: "{DIR[DTR]}/vpp_functional_results" + DIR[DTR,FUNC,HC]: "{DIR[DTR]}/honeycomb_functional_results" + DIR[DTR,FUNC,NSHSFC]: "{DIR[DTR]}/nshsfc_functional_results" + DIR[DTR,PERF,VPP,IMPRV]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_improvements" + + # Detailed test configurations + DIR[DTC]: "{DIR[WORKING,SRC]}/test_configuration" + DIR[DTC,PERF,VPP]: "{DIR[DTC]}/vpp_performance_configuration" + DIR[DTC,FUNC,VPP]: "{DIR[DTC]}/vpp_functional_configuration" + + # Detailed tests operational data + DIR[DTO]: "{DIR[WORKING,SRC]}/test_operational_data" + DIR[DTO,PERF,VPP]: "{DIR[DTO]}/vpp_performance_operational_data" + + # .css patch file to fix tables generated by Sphinx + DIR[CSS_PATCH_FILE]: "{DIR[STATIC]}/theme_overrides.css" + DIR[CSS_PATCH_FILE2]: "{DIR[WORKING,SRC,STATIC]}/theme_overrides.css" + + urls: + URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" + URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job" + + make-dirs: + # List the directories which are created while preparing the environment. + # All directories MUST be defined in "paths" section. + - "DIR[WORKING,DATA]" + - "DIR[STATIC,VPP]" + - "DIR[STATIC,DPDK]" + - "DIR[STATIC,ARCH]" + - "DIR[BUILD,LATEX]" + - "DIR[WORKING,SRC]" + - "DIR[WORKING,SRC,STATIC]" + + remove-dirs: + # List the directories which are deleted while cleaning the environment. + # All directories MUST be defined in "paths" section. + #- "DIR[BUILD,HTML]" + + build-dirs: + # List the directories where the results (build) is stored. + # All directories MUST be defined in "paths" section. + - "DIR[BUILD,HTML]" + - "DIR[BUILD,LATEX]" + +- + type: "debug" + general: + input-format: "xml" # zip or xml + extract: "robot-plugin/output.xml" # Only for zip + builds: + # The files must be in the directory DIR[WORKING,DATA] + csit-dpdk-perf-1707-all: + - + build: 10 + file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__10.xml" + - + build: 9 + file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__9.xml" + csit-nsh_sfc-verify-func-1707-ubuntu1604-virl: + - + build: 2 + file: "{DIR[WORKING,DATA]}/csit-nsh_sfc-verify-func-1707-ubuntu1604-virl-2.xml" + csit-vpp-functional-1707-ubuntu1604-virl: + - + build: lastSuccessfulBuild + file: "{DIR[WORKING,DATA]}/csit-vpp-functional-1707-ubuntu1604-virl-lastSuccessfulBuild.xml" + hc2vpp-csit-integration-1707-ubuntu1604: + - + build: lastSuccessfulBuild + file: "{DIR[WORKING,DATA]}/hc2vpp-csit-integration-1707-ubuntu1604-lastSuccessfulBuild.xml" + csit-vpp-perf-1707-all: + - + build: 16 + file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__16__output.xml" + - + build: 17 + file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__17__output.xml" + +- + type: "static" + src-path: "{DIR[RST]}" + dst-path: "{DIR[WORKING,SRC]}" + +- + type: "input" # Ignored in debug mode + general: + file-name: "robot-plugin.zip" + file-format: ".zip" + download-path: "{job}/{build}/robot/report/*zip*/{filename}" + extract: "robot-plugin/output.xml" + builds: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + - 22 + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + csit-vpp-functional-1707-ubuntu1604-virl: + - lastSuccessfulBuild + hc2vpp-csit-perf-master-ubuntu1604: + - 8 + - 9 + hc2vpp-csit-integration-1707-ubuntu1604: + - lastSuccessfulBuild + csit-nsh_sfc-verify-func-1707-ubuntu1604-virl: + - 2 + +- + type: "output" + format: + html: + - full + pdf: + - full + +################################################################################ +### T A B L E S ### +################################################################################ + +- + type: "table" + title: "Performance improvements" + algorithm: "table_performance_improvements" + template: "{DIR[DTR,PERF,VPP,IMPRV]}/tmpl_performance_improvements.csv" + output-file-ext: ".csv" + output-file: "{DIR[DTR,PERF,VPP,IMPRV]}/performance_improvements" + columns: + - + title: "VPP Functionality" + data: "template 1" + - + title: "Test Name" + data: "template 2" + - + title: "VPP-16.09 mean [Mpps]" + data: "template 3" + - + title: "VPP-17.01 mean [Mpps]" + data: "template 4" + - + title: "VPP-17.04 mean [Mpps]" + data: "template 5" + - + title: "VPP-17.07 mean [Mpps]" + data: "data csit-vpp-perf-1707-all mean" + - + title: "VPP-17.07 stdev [Mpps]" + data: "data csit-vpp-perf-1707-all stdev" + - + title: "17.04 to 17.07 change [%]" + data: "operation relative_change 5 4" + rows: "generated" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "template" + parameters: + - "throughput" + +- + type: "table" + title: "Detailed Test Results - VPP Performance Results" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[DTR,PERF,VPP]}/vpp_performance_results" + columns: + - + title: "Name" + data: "data name" + - + title: "Documentation" + data: "data doc" + - + title: "Status" + data: "data msg" + rows: "generated" + data: + csit-vpp-perf-1707-all: + - 22 + filter: "all" + parameters: + - "name" + - "parent" + - "doc" + - "msg" + +- + type: "table" + title: "Test configuration - VPP Performance Test Configs" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[DTC,PERF,VPP]}/vpp_test_configuration" + columns: + - + title: "Name" + data: "data name" + - + title: "VPP API Test (VAT) Commands History - Commands Used Per Test Case" + data: "data vat-history" + rows: "generated" + data: + csit-vpp-perf-1707-all: + - 22 + filter: "all" + parameters: + - "parent" + - "name" + - "vat-history" + +- + type: "table" + title: "Test Operational Data - VPP Performance Operational Data" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[DTO,PERF,VPP]}/vpp_test_operational" + columns: + - + title: "Name" + data: "data name" + - + title: "VPP Operational Data - Outputs of 'show runtime' at NDR packet rate" + data: "data show-run" + rows: "generated" + data: + csit-vpp-perf-1707-all: + - 22 + filter: "all" + parameters: + - "parent" + - "name" + - "show-run" + +- + type: "table" + title: "Detailed Test Results - VPP Functional Results" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[DTR,FUNC,VPP]}/vpp_functional_results" + columns: + - + title: "Name" + data: "data name" + - + title: "Documentation" + data: "data doc" + - + title: "Status" + data: "data status" + rows: "generated" + data: + csit-vpp-functional-1707-ubuntu1604-virl: + - "lastSuccessfulBuild" + filter: "all" + parameters: + - "name" + - "parent" + - "doc" + - "status" + +- + type: "table" + title: "Test configuration - VPP Functional Test Configs" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[DTC,FUNC,VPP]}/vpp_functional_configuration" + columns: + - + title: "Name" + data: "data name" + - + title: "VPP API Test (VAT) Commands History - Commands Used Per Test Case" + data: "data vat-history" + rows: "generated" + data: + csit-vpp-functional-1707-ubuntu1604-virl: + - "lastSuccessfulBuild" + filter: "all" + parameters: + - "parent" + - "name" + - "vat-history" + +- + type: "table" + title: "Detailed Test Results - DPDK Performance Results" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[DTR,PERF,DPDK]}/dpdk_performance_results" + columns: + - + title: "Name" + data: "data name" + - + title: "Documentation" + data: "data doc" + - + title: "Status" + data: "data msg" + rows: "generated" + data: + csit-dpdk-perf-1707-all: + - 10 + filter: "all" + parameters: + - "name" + - "parent" + - "doc" + - "msg" + +- + type: "table" + title: "Detailed Test Results - Honeycomb Functional Results" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[DTR,FUNC,HC]}/hc_functional_results" + columns: + - + title: "Name" + data: "data name" + - + title: "Documentation" + data: "data doc" + - + title: "Status" + data: "data status" + rows: "generated" + data: + hc2vpp-csit-integration-1707-ubuntu1604: + - "lastSuccessfulBuild" + filter: "all" + parameters: + - "name" + - "parent" + - "doc" + - "status" + +- + type: "table" + title: "Detailed Test Results - NSH SFC Functional Results" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[DTR,FUNC,NSHSFC]}/nsh_sfc_functional_results" + columns: + - + title: "Name" + data: "data name" + - + title: "Documentation" + data: "data doc" + - + title: "Status" + data: "data status" + rows: "generated" + data: + csit-nsh_sfc-verify-func-1707-ubuntu1604-virl: + - 2 + filter: "all" + parameters: + - "name" + - "parent" + - "doc" + - "status" + +################################################################################ +### F I L E S ### +################################################################################ + +- + type: "file" + title: "VPP Performance Results" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTR,PERF,VPP]}/vpp_performance_results" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTR,PERF,VPP]}" + data: + csit-vpp-perf-1707-all: + - 22 + filter: "all" + parameters: + - "name" + - "doc" + - "level" + data-start-level: 2 # 0, 1, 2, ... + chapters-start-level: 2 # 0, 1, 2, ... + +- + type: "file" + title: "VPP Performance Configuration" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTC,PERF,VPP]}/vpp_performance_configuration" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTC,PERF,VPP]}" + data: + csit-vpp-perf-1707-all: + - 22 + filter: "all" + parameters: + - "name" + - "doc" + - "level" + data-start-level: 2 # 0, 1, 2, ... + chapters-start-level: 2 # 0, 1, 2, ... + +- + type: "file" + title: "VPP Performance Operational Data" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTO,PERF,VPP]}/vpp_performance_operational_data" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTO,PERF,VPP]}" + data: + csit-vpp-perf-1707-all: + - 22 + filter: "all" + parameters: + - "name" + - "doc" + - "level" + data-start-level: 2 # 0, 1, 2, ... + chapters-start-level: 2 # 0, 1, 2, ... + +- + type: "file" + title: "VPP Functional Results" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTR,FUNC,VPP]}/vpp_functional_results" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTR,FUNC,VPP]}" + data: + csit-vpp-functional-1707-ubuntu1604-virl: + - "lastSuccessfulBuild" + filter: "all" + parameters: + - "name" + - "doc" + - "level" + data-start-level: 3 # 0, 1, 2, ... + chapters-start-level: 2 # 0, 1, 2, ... + +- + type: "file" + title: "VPP Functional Configuration" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTC,FUNC,VPP]}/vpp_functional_configuration" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTC,FUNC,VPP]}" + data: + csit-vpp-functional-1707-ubuntu1604-virl: + - "lastSuccessfulBuild" + filter: "all" + parameters: + - "name" + - "doc" + - "level" + data-start-level: 3 # 0, 1, 2, ... + chapters-start-level: 2 # 0, 1, 2, ... + +- + type: "file" + title: "DPDK Performance Results" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTR,PERF,DPDK]}/dpdk_performance_results" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTR,PERF,DPDK]}" + data: + csit-dpdk-perf-1707-all: + - 10 + filter: "all" + parameters: + - "name" + - "doc" + - "level" + chapters: + - "suites" + data-start-level: 2 # 0, 1, 2, ... + chapters-start-level: 2 # 0, 1, 2, ... + +- + type: "file" + title: "Honeycomb Functional Results" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTR,FUNC,HC]}/honeycomb_functional_results" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTR,FUNC,HC]}" + data: + hc2vpp-csit-integration-1707-ubuntu1604: + - "lastSuccessfulBuild" + filter: "all" + parameters: + - "name" + - "doc" + - "level" + chapters: + - "suites" + data-start-level: 3 # 0, 1, 2, ... + chapters-start-level: 2 # 0, 1, 2, ... + +- + type: "file" + title: "NSH SFC Functional Results" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTR,FUNC,NSHSFC]}/nshsfc_functional_results" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTR,FUNC,NSHSFC]}" + data: + csit-nsh_sfc-verify-func-1707-ubuntu1604-virl: + - 2 + filter: "all" + parameters: + - "name" + - "doc" + - "level" + chapters: + - "suites" + data-start-level: 2 # 0, 1, 2, ... + chapters-start-level: 2 # 0, 1, 2, ... + +################################################################################ +### P L O T S ### +################################################################################ + +# Plot packets per second + +# VPP L2 sel1 +- + type: "plot" + title: "VPP Performance 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + # Keep this formatting, the filter is enclosed with " (quotation mark) and + # each tag is enclosed with ' (apostrophe). + filter: "'64B' and 'BASE' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel1-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'BASE' and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'BASE' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel1-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'BASE' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP L2 sel2 +- + type: "plot" + title: "VPP Performance 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel2-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'FEATURE' and 'ACL50' and '10k_FLOWS' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [0, 8000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel2-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'FEATURE' and 'ACL50' and '10k_FLOWS' and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [5000000, 12000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel2-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'FEATURE' and 'ACL50' and '10k_FLOWS' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [0, 8000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel2-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'FEATURE' and 'ACL50' and '10k_FLOWS' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [5000000, 12000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP IP4 +- + type: "plot" + title: "VPP Performance 64B-1t1c-ethip4-ip4[a-z0-9]+-[a-z-]*ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-ethip4-ip4-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and ('BASE' or 'SCALE' or 'FEATURE') and 'NDRDISC' and '1T1C' and 'IP4FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-ethip4-ip4[a-z0-9]+-[a-z-]*ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-ethip4-ip4[a-z0-9]+-[a-z-]*ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-ethip4-ip4-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and ('BASE' or 'SCALE' or 'FEATURE') and 'NDRDISC' and '2T2C' and 'IP4FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-ethip4-ip4[a-z0-9]+-[a-z-]*ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-1t1c-ethip4-ip4[a-z0-9]+-[a-z-]*pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-ethip4-ip4-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and ('BASE' or 'SCALE' or 'FEATURE') and 'PDRDISC' and not 'NDRDISC' and '1T1C' and 'IP4FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-ethip4-ip4[a-z0-9]+-[a-z-]*pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-ethip4-ip4[a-z0-9]+-[a-z-]*pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-ethip4-ip4-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and ('BASE' or 'SCALE' or 'FEATURE') and 'PDRDISC' and not 'NDRDISC' and '2T2C' and 'IP4FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-ethip4-ip4[a-z0-9]+-[a-z-]*pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP IP6 +- + type: "plot" + title: "VPP Performance 78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-1t1c-ethip6-ip6-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and ('BASE' or 'SCALE' or 'FEATURE') and 'NDRDISC' and '1T1C' and 'IP6FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-2t2c-ethip6-ip6-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and ('BASE' or 'SCALE' or 'FEATURE') and 'NDRDISC' and '2T2C' and 'IP6FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-1t1c-ethip6-ip6-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and ('BASE' or 'SCALE' or 'FEATURE') and 'PDRDISC' and not 'NDRDISC' and '1T1C' and 'IP6FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-2t2c-ethip6-ip6-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and ('BASE' or 'SCALE' or 'FEATURE') and 'PDRDISC' and not 'NDRDISC' and '2T2C' and 'IP6FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP IP4_overlay +- + type: "plot" + title: "VPP Performance 64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-ethip4-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'ENCAP' and 'NDRDISC' and '1T1C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST' and not 'IPSECHW'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-ethip4-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'ENCAP' and 'NDRDISC' and '2T2C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST' and not 'IPSECHW'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-ethip4-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'ENCAP' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST' and not 'IPSECHW'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-ethip4-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'ENCAP' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST' and not 'IPSECHW'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP IP6_overlay +- + type: "plot" + title: "VPP Performance 78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-1t1c-ethip6-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and 'ENCAP' and 'NDRDISC' and '1T1C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-2t2c-ethip6-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and 'ENCAP' and 'NDRDISC' and '2T2C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-1t1c-ethip6-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and 'ENCAP' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-2t2c-ethip6-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and 'ENCAP' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP VM VHOST +- + type: "plot" + title: "VPP Performance 64B-1t1c-.*vhost.*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-vhost-sel1-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'NDRDISC' and '1T1C' and 'VHOST' and not ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD')" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-.*vhost.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-.*vhost.*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-vhost-sel1-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'NDRDISC' and '2T2C' and 'VHOST' and not ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD')" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-.*vhost.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-1t1c-.*vhost.*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-vhost-sel1-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and 'VHOST' and not ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD')" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-.*vhost.*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-.*vhost.*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-vhost-sel1-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and 'VHOST' and not ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD')" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-.*vhost.*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP VM VHOST SELECTION +- + type: "plot" + title: "VPP Performance 64B-1t1c-.*vhost.*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-vhost-sel2-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'NDRDISC' and '1T1C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-.*vhost.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-.*vhost.*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-vhost-sel2-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'NDRDISC' and '2T2C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-.*vhost.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-1t1c-.*vhost.*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-vhost-sel2-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-.*vhost.*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-.*vhost.*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-vhost-sel2-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-.*vhost.*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP CRYPTO +- + type: "plot" + title: "VPP Performance 64B-1t1c-.*ipsec.*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-ipsechw-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and not 'VHOST' and 'IP4FWD' and 'NDRDISC' and '1T1C' and 'IPSECHW' and ('IPSECTRAN' or 'IPSECTUN')" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-.*ipsec.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-.*ipsec.*-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-ipsechw-ndrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and not 'VHOST' and 'IP4FWD' and 'NDRDISC' and '2T2C' and 'IPSECHW' and ('IPSECTRAN' or 'IPSECTUN')" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-.*ipsec.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-1t1c-.*ipsec.*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-ipsechw-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and not 'VHOST' and 'IP4FWD' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and 'IPSECHW' and ('IPSECTRAN' or 'IPSECTUN')" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-.*ipsec.*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Performance 64B-2t2c-.*ipsec.*-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-ipsechw-pdrdisc" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and not 'VHOST' and 'IP4FWD' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and 'IPSECHW' and ('IPSECTRAN' or 'IPSECTUN')" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-.*ipsec.*-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# DPDK +- + type: "plot" + title: "DPDK Performance 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-1t1c-l2-ndrdisc" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Performance 64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-2t2c-l2-ndrdisc" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Performance 64B-1t1c-ethip4-ip4base-l3fwd-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-1t1c-ipv4-ndrdisc" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'NDRDISC' and '1T1C' and 'IP4FWD'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-ethip4-ip4base-l3fwd-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [2000000, 12000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Performance 64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-2t2c-ipv4-ndrdisc" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'NDRDISC' and '2T2C' and 'IP4FWD'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [2000000, 12000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Performance 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-1t1c-l2-pdrdisc" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Performance 64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-2t2c-l2-pdrdisc" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Performance 64B-1t1c-ethip4-ip4base-l3fwd-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-1t1c-ipv4-pdrdisc" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'PDRDISC' and not 'NDRDISC' and '1T1C' and 'IP4FWD'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-ethip4-ip4base-l3fwd-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [20000000, 30000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Performance 64B-2t2c-ethip4-ip4base-l3fwd-pdrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-2t2c-ipv4-pdrdisc" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'PDRDISC' and not 'NDRDISC' and '2T2C' and 'IP4FWD'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-2t2c-ethip4-ip4base-l3fwd-pdrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [20000000, 30000000] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# Plot latency + +# VPP L2 sel1 +- + type: "plot" + title: "VPP Latency 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'BASE' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Latency 64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel1-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'BASE' and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP L2 sel2 +- + type: "plot" + title: "VPP Latency 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel2-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and ('FEATURE' and 'ACL50' and '10k_FLOWS') and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Latency 64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-l2-sel2-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and ('FEATURE' and 'ACL50' and '10k_FLOWS') and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP IP4 +- + type: "plot" + title: "VPP Latency 64B-1t1c-ethip4-ip4[a-z0-9]+-[a-z-]*ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-ethip4-ip4-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and ('BASE' or 'SCALE' or 'FEATURE') and 'NDRDISC' and '1T1C' and 'IP4FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-ethip4-ip4[a-z0-9]+-[a-z-]*ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Latency 64B-2t2c-ethip4-ip4[a-z0-9]+-[a-z-]*ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-ethip4-ip4-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and ('BASE' or 'SCALE' or 'FEATURE') and 'NDRDISC' and '2T2C' and 'IP4FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-2t2c-ethip4-ip4[a-z0-9]+-[a-z-]*ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP IP6 +- + type: "plot" + title: "VPP Latency 78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-1t1c-ethip6-ip6-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and ('BASE' or 'SCALE' or 'FEATURE') and 'NDRDISC' and '1T1C' and 'IP6FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "78B-1t1c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Latency 78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-2t2c-ethip6-ip6-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and ('BASE' or 'SCALE' or 'FEATURE') and 'NDRDISC' and '2T2C' and 'IP6FWD' and not 'IPSEC' and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "78B-2t2c-ethip6-ip6[a-z0-9]+-[a-z-]*ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP IP4_overlay +- + type: "plot" + title: "VPP Latency 64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-ethip4-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'ENCAP' and 'NDRDISC' and '1T1C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST' and not 'IPSECHW'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Latency 64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-ethip4-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'ENCAP' and 'NDRDISC' and '2T2C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST' and not 'IPSECHW'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-2t2c-ethip4[a-z0-9]+-[a-z0-9]*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP IP6_overlay +- + type: "plot" + title: "VPP Latency 78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-1t1c-ethip6-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and 'ENCAP' and 'NDRDISC' and '1T1C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "78B-1t1c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Latency 78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/78B-2t2c-ethip6-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'78B' and 'ENCAP' and 'NDRDISC' and '2T2C' and ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "78B-2t2c-ethip6[a-z0-9]+-[a-z0-9]*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP VM VHOST +- + type: "plot" + title: "VPP Latency 64B-1t1c-.*vhost.*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-vhost-sel1-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'NDRDISC' and '1T1C' and 'VHOST' and not ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD')" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-.*vhost.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Latency 64B-2t2c-.*vhost.*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-vhost-sel1-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'NDRDISC' and '2T2C' and 'VHOST' and not ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD')" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-2t2c-.*vhost.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP VM VHOST selection +- + type: "plot" + title: "VPP Latency 64B-1t1c-.*vhost.*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-vhost-sel2-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'NDRDISC' and '1T1C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-.*vhost.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Latency 64B-2t2c-.*vhost.*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-vhost-sel2-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and 'NDRDISC' and '2T2C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-2t2c-.*vhost.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# VPP CRYPTO +- + type: "plot" + title: "VPP Latency 64B-1t1c-.*ipsec.*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-ipsechw-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and not 'VHOST' and 'IP4FWD' and 'NDRDISC' and '1T1C' and 'IPSECHW' and ('IPSECTRAN' or 'IPSECTUN')" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-.*ipsec.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "VPP Latency 64B-2t2c-.*ipsec.*-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-2t2c-ipsechw-ndrdisc-lat50" + data: + csit-vpp-perf-1707-all: + - 9 + - 10 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 21 + filter: "'64B' and not 'VHOST' and 'IP4FWD' and 'NDRDISC' and '2T2C' and 'IPSECHW' and ('IPSECTRAN' or 'IPSECTUN')" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-2t2c-.*ipsec.*-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +# DPDK +- + type: "plot" + title: "DPDK Latency 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-1t1c-l2-ndrdisc-lat50" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Latency 64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-2t2c-l2-ndrdisc-lat50" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'NDRDISC' and '2T2C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Latency 64B-1t1c-ethip4-ip4base-l3fwd-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-1t1c-ipv4-ndrdisc-lat50" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'NDRDISC' and '1T1C' and 'IP4FWD'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-1t1c-ethip4-ip4base-l3fwd-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +- + type: "plot" + title: "DPDK Latency 64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc" + algorithm: "plot_latency_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,DPDK]}/64B-2t2c-ipv4-ndrdisc-lat50" + data: + csit-dpdk-perf-1707-all: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + filter: "'64B' and 'BASE' and 'NDRDISC' and '2T2C' and 'IP4FWD'" + parameters: + - "latency" + - "parent" + traces: + boxmean: False + layout: + title: "64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc" + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: "" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Latency min/avg/max [uSec]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py new file mode 100644 index 0000000000..ec663f99d9 --- /dev/null +++ b/resources/tools/presentation/specification_parser.py @@ -0,0 +1,507 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Specification + +Parsing of the specification YAML file. +""" + + +import logging +from yaml import load, YAMLError +from pprint import pformat + +from errors import PresentationError + + +class Specification(object): + """Specification of Presentation and analytics layer. + + - based on specification specified in the specification YAML file + - presentation and analytics layer is model driven + """ + + # Tags are used in specification YAML file and replaced while the file is + # parsed. + TAG_OPENER = "{" + TAG_CLOSER = "}" + + def __init__(self, cfg_file): + """Initialization. + + :param cfg_file: File handler for the specification YAML file. + :type cfg_file: BinaryIO + """ + self._cfg_file = cfg_file + self._cfg_yaml = None + + self._specification = {"environment": dict(), + "debug": dict(), + "static": dict(), + "input": dict(), + "output": dict(), + "tables": list(), + "plots": list(), + "files": list()} + + @property + def specification(self): + """Getter - specification. + + :returns: Specification. + :rtype: dict + """ + return self._specification + + @property + def environment(self): + """Getter - environment. + + :returns: Environment specification. + :rtype: dict + """ + return self._specification["environment"] + + @property + def static(self): + """Getter - static content. + + :returns: Static content specification. + :rtype: dict + """ + return self._specification["static"] + + @property + def debug(self): + """Getter - debug + + :returns: Debug specification + :rtype: dict + """ + return self._specification["debug"] + + @property + def is_debug(self): + """Getter - debug mode + + :returns: True if debug mode is on, otherwise False. + :rtype: bool + """ + + try: + if self.environment["configuration"]["CFG[DEBUG]"] == 1: + return True + else: + return False + except KeyError: + return False + + @property + def input(self): + """Getter - specification - inputs. + - jobs and builds. + + :returns: Inputs. + :rtype: dict + """ + return self._specification["input"] + + @property + def builds(self): + """Getter - builds defined in specification. + + :returns: Builds defined in the specification. + :rtype: dict + """ + return self.input["builds"] + + @property + def output(self): + """Getter - specification - output formats and versions to be generated. + - formats: html, pdf + - versions: full, ... + + :returns: Outputs to be generated. + :rtype: dict + """ + return self._specification["output"] + + @property + def tables(self): + """Getter - tables to be generated. + + :returns: List of specifications of tables to be generated. + :rtype: list + """ + return self._specification["tables"] + + @property + def plots(self): + """Getter - plots to be generated. + + :returns: List of specifications of plots to be generated. + :rtype: list + """ + return self._specification["plots"] + + @property + def files(self): + """Getter - files to be generated. + + :returns: List of specifications of files to be generated. + :rtype: list + """ + return self._specification["files"] + + def set_input_state(self, job, build_nr, state): + """Set the state of input + + :param job: + :param build_nr: + :param state: + :return: + """ + + try: + for build in self._specification["input"]["builds"][job]: + if build["build"] == build_nr: + build["status"] = state + break + else: + raise PresentationError("Build '{}' is not defined for job '{}'" + " in specification file.". + format(build_nr, job)) + except KeyError: + raise PresentationError("Job '{}' and build '{}' is not defined in " + "specification file.".format(job, build_nr)) + + def set_input_file_name(self, job, build_nr, file_name): + """Set the state of input + + :param job: + :param build_nr: + :param file_name: + :return: + """ + + try: + for build in self._specification["input"]["builds"][job]: + if build["build"] == build_nr: + build["file-name"] = file_name + break + else: + raise PresentationError("Build '{}' is not defined for job '{}'" + " in specification file.". + format(build_nr, job)) + except KeyError: + raise PresentationError("Job '{}' and build '{}' is not defined in " + "specification file.".format(job, build_nr)) + + def _get_type_index(self, item_type): + """Get index of item type (environment, input, output, ...) in + specification YAML file. + + :param item_type: Item type: Top level items in specification YAML file, + e.g.: environment, input, output. + :type item_type: str + :returns: Index of the given item type. + :rtype: int + """ + + index = 0 + for item in self._cfg_yaml: + if item["type"] == item_type: + return index + index += 1 + return None + + def _find_tag(self, text): + """Find the first tag in the given text. The tag is enclosed by the + TAG_OPENER and TAG_CLOSER. + + :param text: Text to be searched. + :type text: str + :returns: The tag, or None if not found. + :rtype: str + """ + try: + start = text.index(self.TAG_OPENER) + end = text.index(self.TAG_CLOSER, start + 1) + 1 + return text[start:end] + except ValueError: + return None + + def _replace_tags(self, data, src_data=None): + """Replace tag(s) in the data by their values. + + :param data: The data where the tags will be replaced by their values. + :param src_data: Data where the tags are defined. It is dictionary where + the key is the tag and the value is the tag value. If not given, 'data' + is used instead. + :type data: str or dict + :type src_data: dict + :returns: Data with the tags replaced. + :rtype: str or dict + :raises: PresentationError if it is not possible to replace the tag or + the data is not the supported data type (str, dict). + """ + + if src_data is None: + src_data = data + + if isinstance(data, str): + tag = self._find_tag(data) + if tag is not None: + data = data.replace(tag, src_data[tag[1:-1]]) + + elif isinstance(data, dict): + counter = 0 + for key, value in data.items(): + tag = self._find_tag(value) + if tag is not None: + try: + data[key] = value.replace(tag, src_data[tag[1:-1]]) + counter += 1 + except KeyError: + raise PresentationError("Not possible to replace the " + "tag '{}'".format(tag)) + if counter: + self._replace_tags(data, src_data) + else: + raise PresentationError("Replace tags: Not supported data type.") + + return data + + def _parse_env(self): + """Parse environment specification in the specification YAML file. + """ + + logging.info("Parsing specification file: environment ...") + + idx = self._get_type_index("environment") + if idx is None: + return None + + try: + self._specification["environment"]["configuration"] = \ + self._cfg_yaml[idx]["configuration"] + except KeyError: + self._specification["environment"]["configuration"] = None + + try: + self._specification["environment"]["paths"] = \ + self._replace_tags(self._cfg_yaml[idx]["paths"]) + except KeyError: + self._specification["environment"]["paths"] = None + + try: + self._specification["environment"]["urls"] = \ + self._replace_tags(self._cfg_yaml[idx]["urls"]) + except KeyError: + self._specification["environment"]["urls"] = None + + try: + self._specification["environment"]["make-dirs"] = \ + self._cfg_yaml[idx]["make-dirs"] + except KeyError: + self._specification["environment"]["make-dirs"] = None + + try: + self._specification["environment"]["remove-dirs"] = \ + self._cfg_yaml[idx]["remove-dirs"] + except KeyError: + self._specification["environment"]["remove-dirs"] = None + + try: + self._specification["environment"]["build-dirs"] = \ + self._cfg_yaml[idx]["build-dirs"] + except KeyError: + self._specification["environment"]["build-dirs"] = None + + logging.info("Done.") + + def _parse_debug(self): + """Parse debug specification in the specification YAML file. + """ + + if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1: + return None + + logging.info("Parsing specification file: debug ...") + + idx = self._get_type_index("debug") + if idx is None: + self.environment["configuration"]["CFG[DEBUG]"] = 0 + return None + + try: + for key, value in self._cfg_yaml[idx]["general"].items(): + self._specification["debug"][key] = value + + self._specification["input"]["builds"] = dict() + for job, builds in self._cfg_yaml[idx]["builds"].items(): + if builds: + self._specification["input"]["builds"][job] = list() + for build in builds: + self._specification["input"]["builds"][job].\ + append({"build": build["build"], + "status": "downloaded", + "file-name": self._replace_tags( + build["file"], + self.environment["paths"])}) + else: + logging.warning("No build is defined for the job '{}'. " + "Trying to continue without it.". + format(job)) + + except KeyError: + raise PresentationError("No data to process.") + + def _parse_input(self): + """Parse input specification in the specification YAML file. + + :raises: PresentationError if there are no data to process. + """ + + logging.info("Parsing specification file: input ...") + + idx = self._get_type_index("input") + if idx is None: + raise PresentationError("No data to process.") + + try: + for key, value in self._cfg_yaml[idx]["general"].items(): + self._specification["input"][key] = value + self._specification["input"]["builds"] = dict() + for job, builds in self._cfg_yaml[idx]["builds"].items(): + if builds: + self._specification["input"]["builds"][job] = list() + for build in builds: + self._specification["input"]["builds"][job].\ + append({"build": build, "status": None}) + else: + logging.warning("No build is defined for the job '{}'. " + "Trying to continue without it.". + format(job)) + except KeyError: + raise PresentationError("No data to process.") + + logging.info("Done.") + + def _parse_output(self): + """Parse output specification in the specification YAML file. + + :raises: PresentationError if there is no output defined. + """ + + logging.info("Parsing specification file: output ...") + + idx = self._get_type_index("output") + if idx is None: + raise PresentationError("No output defined.") + + try: + self._specification["output"] = self._cfg_yaml[idx]["format"] + except KeyError: + raise PresentationError("No output defined.") + + logging.info("Done.") + + def _parse_static(self): + """Parse specification of the static content in the specification YAML + file. + """ + + logging.info("Parsing specification file: static content ...") + + idx = self._get_type_index("static") + if idx is None: + logging.warning("No static content specified.") + + for key, value in self._cfg_yaml[idx].items(): + if isinstance(value, str): + try: + self._cfg_yaml[idx][key] = self._replace_tags( + value, self._specification["environment"]["paths"]) + except KeyError: + pass + + self._specification["static"] = self._cfg_yaml[idx] + + logging.info("Done.") + + def _parse_elements(self): + """Parse elements (tables, plots) specification in the specification + YAML file. + """ + + logging.info("Parsing specification file: elements ...") + + count = 1 + for element in self._cfg_yaml: + try: + element["output-file"] = self._replace_tags( + element["output-file"], + self._specification["environment"]["paths"]) + except KeyError: + pass + if element["type"] == "table": + logging.info(" {:3d} Processing a table ...".format(count)) + try: + element["template"] = self._replace_tags( + element["template"], + self._specification["environment"]["paths"]) + except KeyError: + pass + self._specification["tables"].append(element) + count += 1 + elif element["type"] == "plot": + logging.info(" {:3d} Processing a plot ...".format(count)) + self._specification["plots"].append(element) + count += 1 + elif element["type"] == "file": + logging.info(" {:3d} Processing a file ...".format(count)) + try: + element["dir-tables"] = self._replace_tags( + element["dir-tables"], + self._specification["environment"]["paths"]) + except KeyError: + pass + self._specification["files"].append(element) + count += 1 + + logging.info("Done.") + + def read_specification(self): + """Parse specification in the specification YAML file. + + :raises: PresentationError if an error occurred while parsing the + specification file. + """ + try: + self._cfg_yaml = load(self._cfg_file) + except YAMLError as err: + raise PresentationError(msg="An error occurred while parsing the " + "specification file.", + details=str(err)) + + self._parse_env() + self._parse_debug() + if not self.debug: + self._parse_input() + self._parse_output() + self._parse_static() + self._parse_elements() + + logging.debug("Specification: \n{}". + format(pformat(self._specification))) diff --git a/resources/tools/presentation/static_content.py b/resources/tools/presentation/static_content.py new file mode 100644 index 0000000000..fe2d0724bf --- /dev/null +++ b/resources/tools/presentation/static_content.py @@ -0,0 +1,57 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Static content + +Process the static content stored in the git. +""" + +import logging + +from os import makedirs +from os.path import isdir +from shutil import rmtree, copytree, Error + +from errors import PresentationError + + +def prepare_static_content(spec): + """Prepare the static content which is stored in the git. + + :param spec: Specification read from the specification file. + :type spec: Specification + :raises PresentationError: If it is not possible to process the static + content. + """ + + src = spec.static["src-path"] + dst = spec.static["dst-path"] + + # Copy all the static content to the build directory: + logging.info("Copying the static content ...") + logging.info(" Source: {0}".format(src)) + logging.info(" Destination: {0}".format(dst)) + + try: + if isdir(dst): + rmtree(dst) + + copytree(src, dst) + + makedirs(spec.environment["paths"]["DIR[WORKING,SRC,STATIC]"]) + + except (Error, OSError) as err: + raise PresentationError("Not possible to process the static content.", + str(err)) + + logging.info("Done.") diff --git a/resources/tools/presentation/utils.py b/resources/tools/presentation/utils.py new file mode 100644 index 0000000000..f423cd22a7 --- /dev/null +++ b/resources/tools/presentation/utils.py @@ -0,0 +1,104 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""General purpose utilities. +""" + +from os import walk +from os.path import join +from math import sqrt + + +def mean(items): + """Calculate mean value from the items. + + :param items: Mean value is calculated from these items. + :type items: list + :returns: MEan value. + :rtype: float + """ + + return float(sum(items)) / len(items) + + +def stdev(items): + """Calculate stdev from the items. + + :param items: Stdev is calculated from these items. + :type items: list + :returns: Stdev. + :rtype: float + """ + + avg = mean(items) + variance = [(x - avg) ** 2 for x in items] + stddev = sqrt(mean(variance)) + return stddev + + +def relative_change(nr1, nr2): + """Compute relative change of two values. + + :param nr1: The first number. + :param nr2: The second number. + :type nr1: float + :type nr2: float + :returns: Relative change of nr1. + :rtype: float + """ + + return (nr1 - nr2) / nr2 * 100 + + +def get_files(path, extension=None, full_path=True): + """Generates the list of files to process. + + :param path: Path to files. + :param extension: Extension of files to process. If it is the empty string, + all files will be processed. + :param full_path: If True, the files with full path are generated. + :type path: str + :type extension: str + :type full_path: bool + :returns: List of files to process. + :rtype: list + """ + + file_list = list() + for root, _, files in walk(path): + for filename in files: + if extension: + if filename.endswith(extension): + if full_path: + file_list.append(join(root, filename)) + else: + file_list.append(filename) + else: + file_list.append(join(root, filename)) + + return file_list + + +def get_rst_title_char(level): + """Return character used for the given title level in rst files. + + :param level: Level of the title. + :type: int + :returns: Character used for the given title level in rst files. + :rtype: str + """ + chars = ('=', '-', '`', "'", '.', '~', '*', '+', '^') + if level < len(chars): + return chars[level] + else: + return chars[-1] -- 2.16.6