From: Tibor Frank Date: Wed, 26 Feb 2020 12:34:04 +0000 (+0100) Subject: PAL: Detailed test results X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=refs%2Fchanges%2F59%2F25459%2F5 PAL: Detailed test results Change-Id: I75ebad7874602dbc136440a3b4c6f94b095728c4 Signed-off-by: Tibor Frank --- diff --git a/docs/report/detailed_test_results/vpp_performance_results/index.rst b/docs/report/detailed_test_results/vpp_performance_results/index.rst index 0459da69ab..b6cdd64a29 100644 --- a/docs/report/detailed_test_results/vpp_performance_results/index.rst +++ b/docs/report/detailed_test_results/vpp_performance_results/index.rst @@ -7,8 +7,8 @@ VPP Throughput ../vpp_performance_results_3n_tsh/index ../vpp_performance_results_2n_dnv/index ../vpp_performance_results_3n_dnv/index - ../vpp_performance_results_2n_clx/index .. + ../vpp_performance_results_2n_clx/index ../vpp_performance_results_2n_skx/index ../vpp_performance_results_3n_skx/index diff --git a/docs/report/test_configuration/vpp_performance_configuration/index.rst b/docs/report/test_configuration/vpp_performance_configuration/index.rst index 7b4b9ab460..7085c8951d 100644 --- a/docs/report/test_configuration/vpp_performance_configuration/index.rst +++ b/docs/report/test_configuration/vpp_performance_configuration/index.rst @@ -7,8 +7,8 @@ VPP Throughput ../vpp_performance_configuration_3n_tsh/index ../vpp_performance_configuration_2n_dnv/index ../vpp_performance_configuration_3n_dnv/index - ../vpp_performance_configuration_2n_clx/index .. + ../vpp_performance_configuration_2n_clx/index ../vpp_performance_configuration_2n_skx/index ../vpp_performance_configuration_3n_skx/index diff --git a/docs/report/test_operational_data/vpp_performance_operational_data/index.rst b/docs/report/test_operational_data/vpp_performance_operational_data/index.rst index 7eca0de3b6..e1c4748529 100644 --- a/docs/report/test_operational_data/vpp_performance_operational_data/index.rst +++ b/docs/report/test_operational_data/vpp_performance_operational_data/index.rst @@ -7,8 +7,8 @@ VPP Throughput ../vpp_performance_operational_data_3n_tsh/index ../vpp_performance_operational_data_2n_dnv/index ../vpp_performance_operational_data_3n_dnv/index - ../vpp_performance_operational_data_2n_clx/index .. + ../vpp_performance_operational_data_2n_clx/index ../vpp_performance_operational_data_2n_skx/index ../vpp_performance_operational_data_3n_skx/index diff --git a/docs/report/vpp_performance_tests/comparisons/2n-clx_vs_3n-hsw_testbeds.rst b/docs/report/vpp_performance_tests/comparisons/2n-clx_vs_3n-hsw_testbeds.rst new file mode 100644 index 0000000000..44f2eb91c9 --- /dev/null +++ b/docs/report/vpp_performance_tests/comparisons/2n-clx_vs_3n-hsw_testbeds.rst @@ -0,0 +1,55 @@ +2n-clx vs. 3n-Hsw Testbeds +-------------------------- + +Relative comparison of |vpp-release| packet throughput (NDR, PDR and +MRR) is calculated for the same tests executed on 3-Node Skylake (3n- +skx) and 3-Node Haswell (3n-hsw) physical testbed types, in 1-core, +2-core and 4-core configurations. + +.. note:: + + Test results have been generated by + `FD.io test executor vpp performance job 3n-hsw`_ and + `FD.io test executor vpp performance job 2n-clx`_ + with RF result + files csit-vpp-perf-|srelease|-\*.zip + `archived here <../../_static/archive/>`_. + +NDR Comparison +~~~~~~~~~~~~~~ + +Comparison tables in HTML, ASCII and CSV formats: + + - `HTML 1c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-ndr.html>`_ + - `HTML 2c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-ndr.html>`_ + - `ASCII 1c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-ndr.txt>`_ + - `ASCII 2c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-ndr.txt>`_ + - `CSV 1c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-ndr.csv>`_ + - `CSV 2c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-ndr.csv>`_ + +PDR Comparison +~~~~~~~~~~~~~~ + +Comparison tables in HTML, ASCII and CSV formats: + + - `HTML 1c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-pdr.html>`_ + - `HTML 2c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-pdr.html>`_ + - `ASCII 1c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-pdr.txt>`_ + - `ASCII 2c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-pdr.txt>`_ + - `CSV 1c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-pdr.csv>`_ + - `CSV 2c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-pdr.csv>`_ + +MRR Comparison +~~~~~~~~~~~~~~ + +Comparison tables in HTML, ASCII and CSV formats: + + - `HTML 1c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-mrr.html>`_ + - `HTML 2c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-mrr.html>`_ + - `HTML 4c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-4c-mrr.html>`_ + - `ASCII 1c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-mrr.txt>`_ + - `ASCII 2c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-mrr.txt>`_ + - `ASCII 4c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-4c-mrr.txt>`_ + - `CSV 1c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-mrr.csv>`_ + - `CSV 2c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-mrr.csv>`_ + - `CSV 4c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-4c-mrr.csv>`_ diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst index 4511a3e578..79b103e117 100644 --- a/docs/report/vpp_performance_tests/comparisons/index.rst +++ b/docs/report/vpp_performance_tests/comparisons/index.rst @@ -24,6 +24,7 @@ Comparisons .. toctree:: current_vs_previous_release + 2n-clx_vs_3n-hsw_testbeds .. 3n-skx_vs_3n-hsw_testbeds diff --git a/resources/tools/presentation/generator_files.py b/resources/tools/presentation/generator_files.py index 47683ea96b..e1ddef013d 100644 --- a/resources/tools/presentation/generator_files.py +++ b/resources/tools/presentation/generator_files.py @@ -133,6 +133,7 @@ def file_test_results(file_spec, input_data, frmt=u"rst"): if suites.empty: return suites = input_data.merge_data(suites) + suites.sort_index(inplace=True) file_name = u"" for suite_longname, suite in suites.items(): diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 49ecae7c3a..64b61fae9c 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -380,8 +380,9 @@ - 29 # NDRPDR sel - 32 # NDRPDR sel - 40 # NDRPDR sel + - 43 # NDRPDR sel - 44 # NDRPDR sel - + - 45 # NDRPDR sel plot-vpp-hdrh-lat-2n-clx: csit-vpp-perf-verify-2001-2n-clx: @@ -394,8 +395,9 @@ - 29 # NDRPDR sel - 32 # NDRPDR sel - 40 # NDRPDR sel + - 43 # NDRPDR sel - 44 # NDRPDR sel - + - 45 # NDRPDR sel plot-vpp-nfv-2n-clx-mrr: csit-vpp-perf-verify-2001-2n-clx: @@ -1210,12 +1212,39 @@ - 29 # NDRPDR sel - 32 # NDRPDR sel - 40 # NDRPDR sel + - 43 # NDRPDR sel - 44 # NDRPDR sel - + - 45 # NDRPDR sel vpp-compare-testbeds-skx-clx: - "vpp-compare-testbeds-skx-clx-ref" - "vpp-compare-testbeds-skx-clx-cmp" + vpp-compare-testbeds-hsw-clx-ref: + csit-vpp-perf-verify-2001-3n-hsw: + - 21 # NDRPDR sel + - 22 # NDRPDR sel + - 23 # NDRPDR sel + - 24 # NDRPDR sel + - 25 # NDRPDR sel + - 29 # NDRPDR sel + - 30 # NDRPDR sel + - 31 # NDRPDR sel + - 32 # NDRPDR sel + - 34 # NDRPDR sel + vpp-compare-testbeds-hsw-clx-cmp: + # TODO: Add the data + csit-vpp-perf-verify-2001-2n-clx: + - 28 # NDRPDR sel + - 29 # NDRPDR sel + - 32 # NDRPDR sel + - 40 # NDRPDR sel + - 43 # NDRPDR sel + - 44 # NDRPDR sel + - 45 # NDRPDR sel + vpp-compare-testbeds-hsw-clx: + - "vpp-compare-testbeds-hsw-clx-ref" + - "vpp-compare-testbeds-hsw-clx-cmp" + vpp-compare-testbeds-mrr-ref: csit-vpp-perf-verify-2001-3n-hsw: - 17 # MRR sel @@ -1240,6 +1269,16 @@ - "vpp-compare-testbeds-mrr-skx-clx-ref" - "vpp-compare-testbeds-mrr-skx-clx-cmp" + vpp-compare-testbeds-mrr-hsw-clx-ref: + csit-vpp-perf-verify-2001-3n-hsw: + - 17 # MRR sel + vpp-compare-testbeds-mrr-hsw-clx-cmp: + csit-vpp-perf-verify-2001-2n-clx: + - 39 # MRR + vpp-compare-testbeds-mrr-hsw-clx: + - "vpp-compare-testbeds-mrr-hsw-clx-ref" + - "vpp-compare-testbeds-mrr-hsw-clx-cmp" + dpdk-compare-testbeds-ref: csit-dpdk-perf-verify-2001-3n-hsw: - 4 # NDRPDR sel @@ -2174,7 +2213,9 @@ - 29 # NDRPDR sel - 32 # NDRPDR sel - 40 # NDRPDR sel + - 43 # NDRPDR sel - 44 # NDRPDR sel + - 45 # NDRPDR sel - 39 # MRR - 34 # NDRPDR full - 35 # NDRPDR full @@ -2463,8 +2504,8 @@ format: html: - full - pdf: - - minimal +# pdf: +# - minimal ################################################################################ ### T A B L E S ### @@ -3489,7 +3530,7 @@ # - "parent" # - "throughput" # - "tags" -# + ## VPP Comparison Across Testbeds 2n-skx to 2n-clx 2t1c pdr #- type: "table" # title: "VPP Comparison Across Testbeds 2n-skx to 2n-clx 2t1c pdr" @@ -3569,7 +3610,95 @@ # - "parent" # - "throughput" # - "tags" -# + +# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c ndr +- type: "table" + title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c ndr" + algorithm: "table_perf_comparison_nic" + output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-1c-ndr" + reference: + title: "3n-hsw xl710" + data: "vpp-compare-testbeds-hsw-clx-ref" + nic: "NIC_Intel-XL710" + compare: + title: "2n-clx xxv710" + data: "vpp-compare-testbeds-hsw-clx-cmp" + nic: "NIC_Intel-XXV710" + data: "vpp-compare-testbeds-hsw-clx" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '1C'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c ndr +- type: "table" + title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c ndr" + algorithm: "table_perf_comparison_nic" + output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-2c-ndr" + reference: + title: "3n-hsw xl710" + data: "vpp-compare-testbeds-hsw-clx-ref" + nic: "NIC_Intel-XL710" + compare: + title: "2n-clx xxv710" + data: "vpp-compare-testbeds-hsw-clx-cmp" + nic: "NIC_Intel-XXV710" + data: "vpp-compare-testbeds-hsw-clx" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '2C'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c pdr +- type: "table" + title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c pdr" + algorithm: "table_perf_comparison_nic" + output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-1c-pdr" + reference: + title: "3n-hsw xl710" + data: "vpp-compare-testbeds-hsw-clx-ref" + nic: "NIC_Intel-XL710" + compare: + title: "2n-clx xxv710" + data: "vpp-compare-testbeds-hsw-clx-cmp" + nic: "NIC_Intel-XXV710" + data: "vpp-compare-testbeds-hsw-clx" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '1C'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c pdr +- type: "table" + title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c pdr" + algorithm: "table_perf_comparison_nic" + output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-2c-pdr" + reference: + title: "3n-hsw xl710" + data: "vpp-compare-testbeds-hsw-clx-ref" + nic: "NIC_Intel-XL710" + compare: + title: "2n-clx xxv710" + data: "vpp-compare-testbeds-hsw-clx-cmp" + nic: "NIC_Intel-XXV710" + data: "vpp-compare-testbeds-hsw-clx" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '2C'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + ## VPP Comparison Across Topologies 3n-skx to 2n-skx 1c ndr #- type: "table" # title: "VPP Comparison Across Topologies 3n-skx to 2n-skx 1c ndr" @@ -4137,7 +4266,7 @@ # - "parent" # - "result" # - "tags" -# + ## VPP Comparison Across Testbeds 2n-skx to 2n-clx 2t1c mrr #- type: "table" # title: "VPP Comparison Across Testbeds 2n-skx to 2n-clx 2t1c mrr" @@ -4255,6 +4384,72 @@ # - "parent" # - "result" +# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c mrr +- type: "table" + title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c mrr" + algorithm: "table_perf_comparison_nic" + output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-1c-mrr" + reference: + title: "3n-hsw xl710" + data: "vpp-compare-testbeds-mrr-hsw-clx-ref" + nic: "NIC_Intel-XL710" + compare: + title: "2n-clx xxv710" + data: "vpp-compare-testbeds-mrr-hsw-clx-cmp" + nic: "NIC_Intel-XXV710" + data: "vpp-compare-testbeds-mrr-hsw-clx" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" + filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'MRR' and '1C'" + parameters: + - "name" + - "parent" + - "result" + - "tags" + +# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c mrr +- type: "table" + title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c mrr" + algorithm: "table_perf_comparison_nic" + output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-2c-mrr" + reference: + title: "3n-hsw xl710" + data: "vpp-compare-testbeds-mrr-hsw-clx-ref" + nic: "NIC_Intel-XL710" + compare: + title: "2n-clx xxv710" + data: "vpp-compare-testbeds-mrr-hsw-clx-cmp" + nic: "NIC_Intel-XXV710" + data: "vpp-compare-testbeds-mrr-hsw-clx" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" + filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'MRR' and '2C'" + parameters: + - "name" + - "parent" + - "result" + - "tags" + +# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 4c mrr +- type: "table" + title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 4c mrr" + algorithm: "table_perf_comparison_nic" + output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-4c-mrr" + reference: + title: "3n-hsw xl710" + data: "vpp-compare-testbeds-mrr-hsw-clx-ref" + nic: "NIC_Intel-XL710" + compare: + title: "2n-clx xxv710" + data: "vpp-compare-testbeds-mrr-hsw-clx-cmp" + nic: "NIC_Intel-XXV710" + data: "vpp-compare-testbeds-mrr-hsw-clx" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" + filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'MRR' and '4C'" + parameters: + - "name" + - "parent" + - "result" + - "tags" + # Detailed Test Results - VPP Performance Results 3n-hsw - type: "table" title: "Detailed Test Results - VPP Performance Results 3n-hsw"