From 153bf776309870aa615f8217c3b438aad199f5ed Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Mon, 19 Nov 2018 14:56:32 +0100 Subject: [PATCH] CSIT-1359: Add 3n-skx and 2n-skx comparisons to Report - 3n-Skx vs. 2n-Skx Testbeds Change-Id: I63e1d34d4589bbb771961ed3c21f0271fee728ad Signed-off-by: Tibor Frank --- .../comparisons/3n-skx_vs_2n-skx_testbeds.rst | 44 ++++ .../dpdk_performance_tests/comparisons/index.rst | 4 +- docs/report/introduction/report_history.rst | 4 + .../comparisons/3n-skx_vs_2n-skx_testbeds.rst | 43 ++++ .../vpp_performance_tests/comparisons/index.rst | 1 + resources/tools/presentation/generator_tables.py | 9 +- resources/tools/presentation/specification.yaml | 258 +++++++++++++++++++++ 7 files changed, 360 insertions(+), 3 deletions(-) create mode 100644 docs/report/dpdk_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst create mode 100644 docs/report/vpp_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst diff --git a/docs/report/dpdk_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst b/docs/report/dpdk_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst new file mode 100644 index 0000000000..bafbb38e20 --- /dev/null +++ b/docs/report/dpdk_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst @@ -0,0 +1,44 @@ + +.. _dpdk_compare_topologies_3n-Skx_vs_2n-Skx: + +3n-Skx vs. 2n-Skx Testbeds +-------------------------- + +Relative comparison of |dpdk-release| Testpmd and L3fwd packet +throughput (NDR, PDR and MRR) is calculated for the same tests executed +on 3-Node Skylake (3n-skx) and 2-Node Skylake (2n-skx) physical testbed +types, in 1-core, 2-core and 4-core configurations. + +.. note:: + + Test results have been generated by + `FD.io test executor dpdk performance job 3n-skx`_ and + `FD.io test executor dpdk performance job 2n-skx`_ + with RF result + files csit-dpdk-perf-|srelease|-\*.zip + `archived here <../../_static/archive/>`_. + +NDR Comparison +~~~~~~~~~~~~~~ + +Comparison tables in ASCII and CSV formats: + + - `ASCII NDR comparison <../../_static/dpdk/performance-compare-topologies-3n-skx-2n-skx-ndr.txt>`_ + - `CSV NDR comparison <../../_static/dpdk/performance-compare-topologies-3n-skx-2n-skx-ndr.csv>`_ + +PDR Comparison +~~~~~~~~~~~~~~ + +Comparison tables in ASCII and CSV formats: + + - `ASCII PDR comparison <../../_static/dpdk/performance-compare-topologies-3n-skx-2n-skx-pdr.txt>`_ + - `CSV PDR comparison <../../_static/dpdk/performance-compare-topologies-3n-skx-2n-skx-pdr.csv>`_ + +.. + MRR Comparison + ~~~~~~~~~~~~~~ + + Comparison tables in ASCII and CSV formats: + + - `ASCII MRR comparison <../../_static/dpdk/performance-compare-testbeds-3n-hsw-3n-skx-mrr.txt>`_ + - `CSV MRR comparison <../../_static/dpdk/performance-compare-testbeds-3n-hsw-3n-skx-mrr.csv>`_ diff --git a/docs/report/dpdk_performance_tests/comparisons/index.rst b/docs/report/dpdk_performance_tests/comparisons/index.rst index ace3baf091..7a3cb44444 100644 --- a/docs/report/dpdk_performance_tests/comparisons/index.rst +++ b/docs/report/dpdk_performance_tests/comparisons/index.rst @@ -4,10 +4,10 @@ \clearpage Comparisons -============= +=========== .. toctree:: current_vs_previous_release 3n-skx_vs_3n-hsw_testbeds - + 3n-skx_vs_2n-skx_testbeds diff --git a/docs/report/introduction/report_history.rst b/docs/report/introduction/report_history.rst index eef74febda..7c1ecae9e2 100644 --- a/docs/report/introduction/report_history.rst +++ b/docs/report/introduction/report_history.rst @@ -20,6 +20,10 @@ Document History | | :ref:`dpdk_compare_current_vs_previous_release` | | | | | | 6. Changed title of this chapter to "Document History" | +| | 7. Added comparisons between topologies: | +| | | +| | a. VPP: :ref:`vpp_compare_topologies_3n-Skx_vs_2n-Skx` | +| | b. DPDK: :ref:`dpdk_compare_topologies_3n-Skx_vs_2n-Skx` | | | | +---------+--------------------------------------------------------------------+ | .w46 | 1. dot1q KVM VMs vhost-user tests added to | diff --git a/docs/report/vpp_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst b/docs/report/vpp_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst new file mode 100644 index 0000000000..38de3b4a41 --- /dev/null +++ b/docs/report/vpp_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst @@ -0,0 +1,43 @@ + +.. _vpp_compare_topologies_3n-Skx_vs_2n-Skx: + +3n-Skx vs. 2n-Skx Testbeds +-------------------------- + +Relative comparison of |vpp-release| packet throughput (NDR, PDR and +MRR) is calculated for the same tests executed on 3-Node Skylake (3n- +skx) and 2-Node Skylake (2n-skx) physical testbed types, in 1-core, +2-core and 4-core configurations. + +.. note:: + + Test results have been generated by + `FD.io test executor vpp performance job 3n-skx`_ and + `FD.io test executor vpp performance job 2n-skx`_ + with RF result + files csit-vpp-perf-|srelease|-\*.zip + `archived here <../../_static/archive/>`_. + +NDR Comparison +~~~~~~~~~~~~~~ + +Comparison tables in ASCII and CSV formats: + + - `ASCII NDR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-ndr.txt>`_ + - `CSV NDR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-ndr.csv>`_ + +PDR Comparison +~~~~~~~~~~~~~~ + +Comparison tables in ASCII and CSV formats: + + - `ASCII PDR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-pdr.txt>`_ + - `CSV PDR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-pdr.csv>`_ + +MRR Comparison +~~~~~~~~~~~~~~ + +Comparison tables in ASCII and CSV formats: + + - `ASCII MRR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-mrr.txt>`_ + - `CSV MRR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-mrr.csv>`_ diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst index a9179c6900..52367ca423 100644 --- a/docs/report/vpp_performance_tests/comparisons/index.rst +++ b/docs/report/vpp_performance_tests/comparisons/index.rst @@ -10,4 +10,5 @@ Comparisons current_vs_previous_release 3n-skx_vs_3n-hsw_testbeds + 3n-skx_vs_2n-skx_testbeds diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index b24222c7f0..a3373db6d8 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -240,11 +240,14 @@ def table_performance_comparison(table, input_data): replace("1t1c", "1c").replace("2t1c", "1c").\ replace("2t2c", "2c").replace("4t2c", "2c").\ replace("4t4c", "4c").replace("8t4c", "4c") + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") if tbl_dict.get(tst_name_mod, None) is None: name = "{0}-{1}".format(tst_data["parent"].split("-")[0], "-".join(tst_data["name"]. split("-")[:-1])) - if "comparison across testbeds" in table["title"].lower(): + if "across testbeds" in table["title"].lower() or \ + "across topologies" in table["title"].lower(): name = name.\ replace("1t1c", "1c").replace("2t1c", "1c").\ replace("2t2c", "2c").replace("4t2c", "2c").\ @@ -286,6 +289,8 @@ def table_performance_comparison(table, input_data): replace("1t1c", "1c").replace("2t1c", "1c").\ replace("2t2c", "2c").replace("4t2c", "2c").\ replace("4t4c", "4c").replace("8t4c", "4c") + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") try: # TODO: Re-work when NDRPDRDISC tests are not used if table["include-tests"] == "MRR": @@ -323,6 +328,8 @@ def table_performance_comparison(table, input_data): replace("1t1c", "1c").replace("2t1c", "1c").\ replace("2t2c", "2c").replace("4t2c", "2c").\ replace("4t4c", "4c").replace("8t4c", "4c") + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") if tbl_dict.get(tst_name_mod, None) is None: continue if tbl_dict[tst_name_mod].get("history", None) is None: diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 816f21134b..918f28069e 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -239,6 +239,42 @@ - 12 # MRR sel - 13 # MRR sel + vpp-compare-topologies: + csit-vpp-perf-verify-1810-3n-skx: + - 10 # NDRPDR sel + - 14 # NDRPDR sel + - 15 # NDRPDR sel + - 16 # NDRPDR sel + - 17 # NDRPDR sel + - 19 # NDRPDR sel + - 20 # NDRPDR sel + - 21 # NDRPDR sel + - 22 # NDRPDR sel + - 25 # NDRPDR sel + csit-vpp-perf-verify-1810-2n-skx: + - 9 # NDRPDR sel + - 13 # NDRPDR sel + - 14 # NDRPDR sel + - 19 # NDRPDR sel + - 20 # NDRPDR sel + - 21 # NDRPDR sel + - 22 # NDRPDR sel + - 23 # NDRPDR sel + - 24 # NDRPDR sel + - 27 # NDRPDR sel + + vpp-compare-topologies-mrr: + csit-vpp-perf-verify-1810-3n-skx: + - 9 # MRR sel + - 11 # MRR sel + - 12 # MRR sel + - 13 # MRR sel + csit-vpp-perf-verify-1810-2n-skx: + - 8 # MRR sel + - 10 # MRR sel + - 11 # MRR sel + - 12 # MRR sel + vpp-performance-changes-3n-hsw-mrr: csit-vpp-perf-check-1804: - 6 # mrr - sel @@ -552,6 +588,30 @@ - 9 # NDRPDR sel - 10 # NDRPDR sel + dpdk-compare-topologies: + csit-dpdk-perf-verify-1810-3n-skx: + - 3 # NDRPDR sel + - 4 # NDRPDR sel + - 6 # NDRPDR sel + - 7 # NDRPDR sel + - 8 # NDRPDR sel + - 9 # NDRPDR sel + - 10 # NDRPDR sel + - 11 # NDRPDR sel + - 12 # NDRPDR sel + - 13 # NDRPDR sel + csit-dpdk-perf-verify-1810-2n-skx: + - 3 # NDRPDR sel + - 4 # NDRPDR sel + - 5 # NDRPDR sel + - 7 # NDRPDR sel + - 8 # NDRPDR sel + - 9 # NDRPDR sel + - 10 # NDRPDR sel + - 11 # NDRPDR sel + - 12 # NDRPDR sel + - 13 # NDRPDR sel + dpdk-perf-results-3n-hsw: csit-dpdk-perf-verify-1810-3n-hsw: - 1 @@ -1874,6 +1934,90 @@ - "parent" - "throughput" +# VPP Comparison Across Topologies 3n-skx to 2n-skx ndr +- + type: "table" + title: "VPP Comparison Across Topologies 3n-skx to 2n-skx ndr" + algorithm: "table_performance_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-compare-topologies-3n-skx-2n-skx-ndr" + reference: + title: "3-Node Skx" + data: + csit-vpp-perf-verify-1810-3n-skx: + - 10 # NDRPDR sel + - 14 # NDRPDR sel + - 15 # NDRPDR sel + - 16 # NDRPDR sel + - 17 # NDRPDR sel + - 19 # NDRPDR sel + - 20 # NDRPDR sel + - 21 # NDRPDR sel + - 22 # NDRPDR sel + - 25 # NDRPDR sel + compare: + title: "2-Node Skx" + data: + csit-vpp-perf-verify-1810-2n-skx: + - 9 # NDRPDR sel + - 13 # NDRPDR sel + - 14 # NDRPDR sel + - 19 # NDRPDR sel + - 20 # NDRPDR sel + - 21 # NDRPDR sel + - 22 # NDRPDR sel + - 23 # NDRPDR sel + - 24 # NDRPDR sel + - 27 # NDRPDR sel + data: "vpp-compare-topologies" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "throughput" + +# VPP Comparison Across Topologies 3n-skx to 2n-skx pdr +- + type: "table" + title: "VPP Comparison Across Topologies 3n-skx to 2n-skx pdr" + algorithm: "table_performance_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-compare-topologies-3n-skx-2n-skx-pdr" + reference: + title: "3-Node Skx" + data: + csit-vpp-perf-verify-1810-3n-skx: + - 10 # NDRPDR sel + - 14 # NDRPDR sel + - 15 # NDRPDR sel + - 16 # NDRPDR sel + - 17 # NDRPDR sel + - 19 # NDRPDR sel + - 20 # NDRPDR sel + - 21 # NDRPDR sel + - 22 # NDRPDR sel + - 25 # NDRPDR sel + compare: + title: "2-Node Skx" + data: + csit-vpp-perf-verify-1810-2n-skx: + - 9 # NDRPDR sel + - 13 # NDRPDR sel + - 14 # NDRPDR sel + - 19 # NDRPDR sel + - 20 # NDRPDR sel + - 21 # NDRPDR sel + - 22 # NDRPDR sel + - 23 # NDRPDR sel + - 24 # NDRPDR sel + - 27 # NDRPDR sel + data: "vpp-compare-topologies" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "throughput" + # VPP Performance Changes 3n-hsw 1t1c MRR - type: "table" @@ -2255,6 +2399,36 @@ - "parent" - "result" +# VPP Comparison Across Topologies 3n-skx to 2n-skx mrr +- + type: "table" + title: "VPP Comparison Across Topologies 3n-skx to 2n-skx mrr" + algorithm: "table_performance_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-compare-topologies-3n-skx-2n-skx-mrr" + reference: + title: "3-Node Skx" + data: + csit-vpp-perf-verify-1810-3n-skx: + - 9 # MRR sel + - 11 # MRR sel + - 12 # MRR sel + - 13 # MRR sel + compare: + title: "2-Node Skx" + data: + csit-vpp-perf-verify-1810-2n-skx: + - 8 # MRR sel + - 10 # MRR sel + - 11 # MRR sel + - 12 # MRR sel + data: "vpp-compare-topologies-mrr" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" + filter: "'MRR'" + parameters: + - "name" + - "parent" + - "result" + # Detailed Test Results - VPP Performance Results 3n-hsw - type: "table" @@ -3192,6 +3366,90 @@ - "parent" - "throughput" +# DPDK Comparison Across Topologies 3n-skx to 2n-skx ndr +- + type: "table" + title: "DPDK Comparison Across Topologies 3n-skx to 2n-skx ndr" + algorithm: "table_performance_comparison" + output-file: "{DIR[STATIC,DPDK]}/performance-compare-topologies-3n-skx-2n-skx-ndr" + reference: + title: "3-Node Skx" + data: + csit-dpdk-perf-verify-1810-3n-skx: + - 3 # NDRPDR sel + - 4 # NDRPDR sel + - 6 # NDRPDR sel + - 7 # NDRPDR sel + - 8 # NDRPDR sel + - 9 # NDRPDR sel + - 10 # NDRPDR sel + - 11 # NDRPDR sel + - 12 # NDRPDR sel + - 13 # NDRPDR sel + compare: + title: "2-Node Skx" + data: + csit-dpdk-perf-verify-1810-2n-skx: + - 3 # NDRPDR sel + - 4 # NDRPDR sel + - 5 # NDRPDR sel + - 7 # NDRPDR sel + - 8 # NDRPDR sel + - 9 # NDRPDR sel + - 10 # NDRPDR sel + - 11 # NDRPDR sel + - 12 # NDRPDR sel + - 13 # NDRPDR sel + data: "dpdk-compare-topologies" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'" + parameters: + - "name" + - "parent" + - "throughput" + +# DPDK Comparison Across Topologies 3n-skx to 2n-skx pdr +- + type: "table" + title: "DPDK Comparison Across Topologies 3n-skx to 2n-skx pdr" + algorithm: "table_performance_comparison" + output-file: "{DIR[STATIC,DPDK]}/performance-compare-topologies-3n-skx-2n-skx-pdr" + reference: + title: "3-Node Skx" + data: + csit-dpdk-perf-verify-1810-3n-skx: + - 3 # NDRPDR sel + - 4 # NDRPDR sel + - 6 # NDRPDR sel + - 7 # NDRPDR sel + - 8 # NDRPDR sel + - 9 # NDRPDR sel + - 10 # NDRPDR sel + - 11 # NDRPDR sel + - 12 # NDRPDR sel + - 13 # NDRPDR sel + compare: + title: "2-Node Skx" + data: + csit-dpdk-perf-verify-1810-2n-skx: + - 3 # NDRPDR sel + - 4 # NDRPDR sel + - 5 # NDRPDR sel + - 7 # NDRPDR sel + - 8 # NDRPDR sel + - 9 # NDRPDR sel + - 10 # NDRPDR sel + - 11 # NDRPDR sel + - 12 # NDRPDR sel + - 13 # NDRPDR sel + data: "dpdk-compare-topologies" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'" + parameters: + - "name" + - "parent" + - "throughput" + # DPDK Comparison Across Testbeds 3n-hsw to 3n-skx pdr - type: "table" -- 2.16.6