From 96c041ab6ab282ae07c10db1bd7383d63ab7def4 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Tue, 3 Mar 2020 13:42:06 +0100 Subject: [PATCH] Report: Add Soak tests Change-Id: I8e5ae26209358d676d1beb758d97a3bfe3a22675 Signed-off-by: Tibor Frank --- docs/report/index.html.template | 4 +- docs/report/introduction/report_history.rst | 2 + .../vpp_performance_tests/comparisons/index.rst | 14 +-- .../comparisons/soak_vs_ndr.rst | 19 ++- docs/report/vpp_performance_tests/index.rst | 4 +- resources/tools/presentation/specification.yaml | 131 +++++++++++++++------ 6 files changed, 113 insertions(+), 61 deletions(-) diff --git a/docs/report/index.html.template b/docs/report/index.html.template index 64312906c2..c89f7525c7 100644 --- a/docs/report/index.html.template +++ b/docs/report/index.html.template @@ -20,6 +20,7 @@ CSIT-2001 vpp_performance_tests/packet_throughput_graphs/index vpp_performance_tests/throughput_speedup_multi_core/index vpp_performance_tests/packet_latency/index + vpp_performance_tests/soak_tests/index vpp_performance_tests/reconf_tests/index vpp_performance_tests/nf_service_density/index vpp_performance_tests/hoststack_testing/index @@ -28,9 +29,6 @@ CSIT-2001 vpp_performance_tests/test_environment vpp_performance_tests/documentation/index -.. - vpp_performance_tests/soak_tests/index - .. toctree:: :maxdepth: 2 :caption: DPDK Performance diff --git a/docs/report/introduction/report_history.rst b/docs/report/introduction/report_history.rst index 0bbd832ad1..bac932a876 100644 --- a/docs/report/introduction/report_history.rst +++ b/docs/report/introduction/report_history.rst @@ -29,6 +29,8 @@ FD.io CSIT-2001 Report history and per .[ww] revision changes are listed below. | | | | | 5. Added TCP/IP tests | | | | +| | 6. Added Soak tests | +| | | +----------------+------------------------------------------------------------+ | .09 | 1. Added data: | | | | diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst index 79b103e117..b384fb0f0c 100644 --- a/docs/report/vpp_performance_tests/comparisons/index.rst +++ b/docs/report/vpp_performance_tests/comparisons/index.rst @@ -8,23 +8,11 @@ Comparisons =========== -.. - // Alternative Note for 1st Bullet when bad microcode Skx, Clx results are published - note:: - VPP throughput drop in the current release in comparison to previous - releases **on Intel Xeon 2n-skx, 3n-skx and 2n-clx testbeds**: VPP - performance test data shows lower performance and behaviour - inconsistency of these systems following the recent upgrade of - processor microcode packages (skx ucode 0x2000064, clx ucode - 0x500002c) as part of updating Ubuntu 18.04 LTS kernel version. - Tested VPP and DPDK applications (L3fwd) are affected. Skx and Clx - test data will be corrected in subsequent maintenance report - version(s) once the issue is resolved. See :ref:`vpp_known_issues`. - .. toctree:: current_vs_previous_release 2n-clx_vs_3n-hsw_testbeds + soak_vs_ndr .. 3n-skx_vs_3n-hsw_testbeds diff --git a/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst b/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst index 4e188eb61c..daf62cc0b7 100644 --- a/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst +++ b/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst @@ -1,4 +1,4 @@ -_`soak vs ndr comparison`: +.. _`soak vs ndr comparison`: Soak Tests vs. NDR Tests ------------------------ @@ -10,11 +10,22 @@ in 1-core configurations. .. note:: Test results have been generated by - `FD.io test executor vpp performance job 2n-skx`_ + `FD.io test executor vpp performance job 2n-skx`_, + `FD.io test executor vpp performance job 2n-clx`_ with RF result files csit-vpp-perf-|srelease|-\*.zip `archived here <../../_static/archive/>`_. Comparison tables in ASCII and CSV formats: - - `ASCII Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.txt>`_ - - `CSV Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.csv>`_ +.. + 2n-skx + ~~~~~~ + + - `ASCII Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.txt>`_ + - `CSV Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.csv>`_ + +2n-clx +~~~~~~ + + - `ASCII Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-clx.txt>`_ + - `CSV Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-clx.csv>`_ diff --git a/docs/report/vpp_performance_tests/index.rst b/docs/report/vpp_performance_tests/index.rst index 2ddf9c8d82..22177eaf9e 100644 --- a/docs/report/vpp_performance_tests/index.rst +++ b/docs/report/vpp_performance_tests/index.rst @@ -8,6 +8,7 @@ VPP Performance packet_throughput_graphs/index throughput_speedup_multi_core/index packet_latency/index + soak_tests/index reconf_tests/index nf_service_density/index hoststack_testing/index @@ -15,6 +16,3 @@ VPP Performance throughput_trending test_environment documentation/index - -.. - soak_tests/index diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 14a53f067b..16bb42db84 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -441,6 +441,11 @@ - 76 # Reconf - 77 # Reconf + plot-vpp-soak-2n-clx: + csit-vpp-perf-verify-2001-2n-clx: + - 93 # Soak + - 94 # Soak + plot-dpdk-throughput-latency-2n-clx: csit-dpdk-perf-verify-2001-2n-clx: - 15 # NDRPDR sel @@ -995,6 +1000,37 @@ - 83 # SOAK - 86 # SOAK + vpp-soak-vs-ndr-2n-clx-ref: + csit-vpp-perf-verify-2001-2n-clx: + - 28 # NDRPDR sel + - 29 # NDRPDR sel + - 32 # NDRPDR sel + - 40 # NDRPDR sel + - 43 # NDRPDR sel + - 44 # NDRPDR sel + - 45 # NDRPDR sel + - 47 # NDRPDR sel + - 48 # NDRPDR sel + - 49 # NDRPDR sel + vpp-soak-vs-ndr-2n-clx-cmp: + csit-vpp-perf-verify-2001-2n-clx: + - 93 # SOAK + - 94 # Soak + vpp-soak-vs-ndr-2n-clx: + csit-vpp-perf-verify-2001-2n-clx: + - 28 # NDRPDR sel + - 29 # NDRPDR sel + - 32 # NDRPDR sel + - 40 # NDRPDR sel + - 43 # NDRPDR sel + - 44 # NDRPDR sel + - 45 # NDRPDR sel + - 47 # NDRPDR sel + - 48 # NDRPDR sel + - 49 # NDRPDR sel + - 93 # SOAK + - 94 # Soak + vpp-performance-changes-3n-hsw-mrr-h1: csit-vpp-perf-verify-1904-3n-hsw: - 9 # MRR sel @@ -2325,8 +2361,6 @@ # - 113 # NFV reconf sel # - 114 # NFV reconf sel # - 115 # NFV reconf sel -# - 83 # SOAK -# - 86 # SOAK # csit-vpp-perf-verify-2001-2n-skx: # - 21 # NDRPDR sel # - 22 # NDRPDR sel @@ -2386,6 +2420,8 @@ - 89 # TCP - 90 # TCP - 91 # TCP + - 93 # Soak + - 94 # Soak csit-vpp-perf-verify-1908-3n-tsh: - 7 # NDRPDR sel @@ -2675,7 +2711,7 @@ ### T A B L E S ### ################################################################################ -## VPP Soak vs NDR +## VPP Soak vs NDR - 2n-skx #- type: "table" # title: "VPP Soak vs NDR 2n-skx" # algorithm: "table_soak_vs_ndr" @@ -2694,6 +2730,25 @@ # - "name" # - "parent" +# VPP Soak vs NDR - 2n-clx +- type: "table" + title: "VPP Soak vs NDR 2n-clx" + algorithm: "table_soak_vs_ndr" + output-file: "{DIR[STATIC,VPP]}/soak-vs-ndr-2n-clx" + reference: + title: "NDR" + data: "vpp-soak-vs-ndr-2n-clx-ref" + compare: + title: "Soak" + data: "vpp-soak-vs-ndr-2n-clx-cmp" + data: "vpp-soak-vs-ndr-2n-clx" + include-tests: "NDR" + filter: "('NDRPDR' or 'SOAK') and not 'NF_DENSITY'" + parameters: + - "throughput" + - "name" + - "parent" + ################################################################################ ## VPP Compare NICs 3n-skx 2t1c ndr @@ -6091,41 +6146,41 @@ ################################################################################ -## Soak Test (PLRsearch), boxes -#- type: "plot" -# title: "VPP Critical rate: 30 Minutes Soak Test (PLRsearch)" -# algorithm: "plot_perf_box_name" -# output-file-type: ".html" -# output-file: "{DIR[STATIC,VPP]}/soak-test-1" -# data: "plot-vpp-soak-2n-skx" -# include: -# - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Patch-Soak.64B-2t1c-avf-eth-l2patch-soak" -# - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Xcbase-Soak.64B-2t1c-avf-eth-l2xcbase-soak" -# - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-soak" -# - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Patch-Soak.64B-2t1c-eth-l2patch-soak" -# - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Xcbase-Soak.64B-2t1c-eth-l2xcbase-soak" -# - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Soak.64B-2t1c-eth-l2bdbasemaclrn-soak" -# layout: -# title: "30 Minutes Soak Test (PLRsearch)" -# layout: "plot-soak-throughput" -# -## Soak Test (PLRsearch), boxes -#- type: "plot" -# title: "VPP Critical rate: 30 Minutes Soak Test (PLRsearch)" -# algorithm: "plot_perf_box_name" -# output-file-type: ".html" -# output-file: "{DIR[STATIC,VPP]}/soak-test-2" -# data: "plot-vpp-soak-2n-skx" -# include: -# - "Tests.Vpp.Perf.Vm Vhost.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Eth-2Vhostvr1024-1Vm-Vppl2Xc-Soak.64B-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc-soak" -# - "Tests.Vpp.Perf.Vm Vhost.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Eth-2Vhostvr1024-1Vm-Soak.64B-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-soak" -# - "Tests.Vpp.Perf.Container Memif.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Eth-2Memif-1Dcr-Soak.64B-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr-soak" -# - "Tests.Vpp.Perf.Ip4.2N1L-25Ge2P1Xxv710-Avf-Ethip4-Ip4Base-Soak.64B-2t1c-avf-ethip4-ip4base-soak" -# - "Tests.Vpp.Perf.Ip4.2N1L-25Ge2P1Xxv710-Ethip4-Ip4Base-Soak.64B-2t1c-ethip4-ip4base-soak" -# - "Tests.Vpp.Perf.Ip6.2N1L-25Ge2P1Xxv710-Ethip6-Ip6Base-Soak.78B-2t1c-ethip6-ip6base-soak" -# layout: -# title: "30 Minutes Soak Test (PLRsearch)" -# layout: "plot-soak-throughput" +# Soak Test (PLRsearch), boxes - 2n-clx +- type: "plot" + title: "VPP Critical rate: 30 Minutes Soak Test (PLRsearch)" + algorithm: "plot_perf_box_name" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/soak-test-1" + data: "plot-vpp-soak-2n-clx" + include: + - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Xcbase-Soak.64B-2t1c-avf-eth-l2xcbase-soak" + - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-soak" + - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Xcbase-Soak.64B-2t1c-eth-l2xcbase-soak" + - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Soak.64B-2t1c-eth-l2bdbasemaclrn-soak" + - "Tests.Vpp.Perf.Ip4.2N1L-25Ge2P1Xxv710-Avf-Ethip4-Ip4Base-Soak.64B-2t1c-avf-ethip4-ip4base-soak" + - "Tests.Vpp.Perf.Ip4.2N1L-25Ge2P1Xxv710-Ethip4-Ip4Base-Soak.64B-2t1c-ethip4-ip4base-soak" + layout: + title: "30 Minutes Soak Test (PLRsearch)" + layout: "plot-soak-throughput" + +# Soak Test (PLRsearch), boxes - 2n-clx +- type: "plot" + title: "VPP Critical rate: 30 Minutes Soak Test (PLRsearch)" + algorithm: "plot_perf_box_name" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/soak-test-2" + data: "plot-vpp-soak-2n-clx" + include: + - "Tests.Vpp.Perf.Ip6.2N1L-25Ge2P1Xxv710-Avf-Ethip6-Ip6Base-Soak.78B-2t1c-avf-ethip6-ip6base-soak" + - "Tests.Vpp.Perf.Ip6.2N1L-25Ge2P1Xxv710-Ethip6-Ip6Base-Soak.78B-2t1c-ethip6-ip6base-soak" + - "Tests.Vpp.Perf.Container Memif.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Eth-2Memif-1Dcr-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-eth-2memif-1dcr-soak" + - "Tests.Vpp.Perf.Container Memif.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Eth-2Memif-1Dcr-Soak.64B-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr-soak" + - "Tests.Vpp.Perf.Vm Vhost.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Eth-2Vhostvr1024-1Vm-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-soak" + - "Tests.Vpp.Perf.Vm Vhost.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Eth-2Vhostvr1024-1Vm-Vppl2Xc-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc-soak" + layout: + title: "30 Minutes Soak Test (PLRsearch)" + layout: "plot-soak-throughput" ############################################################################### -- 2.16.6