From c7b2541ae5ff737691547daef2e4b25f9d232eba Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Fri, 27 Jul 2018 13:59:21 +0200 Subject: [PATCH] CSIT-1213: Add 2n/3n-skx to Test Oper Data Change-Id: I217cc8dc6691c2a443460bd4e51ea96c4ec82806 Signed-off-by: Tibor Frank --- .../csit_test_naming.rst | 4 +- .../cot_performance_results/index.rst | 2 - .../dpdk_performance_tests/csit_release_notes.rst | 4 +- docs/report/dpdk_performance_tests/overview.rst | 8 +- .../csit_release_notes.rst | 4 +- .../report/honeycomb_functional_tests/overview.rst | 6 +- docs/report/introduction/general_notes.rst | 4 +- docs/report/introduction/methodology.rst | 10 +- docs/report/introduction/overview.rst | 18 +- .../csit_release_notes.rst | 4 +- docs/report/nsh_sfc_functional_tests/overview.rst | 4 +- .../vpp_performance_operational_data/index.rst | 10 +- .../index.rst | 4 +- .../index.rst | 12 + .../index.rst | 4 +- .../vpp_functional_tests/csit_release_notes.rst | 4 +- docs/report/vpp_functional_tests/overview.rst | 4 +- .../vpp_performance_tests/csit_release_notes.rst | 4 +- .../documentation/containers.rst | 2 +- docs/report/vpp_performance_tests/overview.rst | 8 +- resources/tools/presentation/conf.py | 4 +- resources/tools/presentation/specification.yaml | 271 +++++++++++++-------- 22 files changed, 229 insertions(+), 166 deletions(-) rename docs/report/{detailed_test_results/cot_performance_results_2n_skx => test_operational_data/vpp_performance_operational_data_2n_skx}/index.rst (80%) create mode 100644 docs/report/test_operational_data/vpp_performance_operational_data_3n_hsw/index.rst rename docs/report/{detailed_test_results/cot_performance_results_3n_skx => test_operational_data/vpp_performance_operational_data_3n_skx}/index.rst (80%) diff --git a/docs/report/csit_framework_documentation/csit_test_naming.rst b/docs/report/csit_framework_documentation/csit_test_naming.rst index 9bb18a8f8b..df1fdd74d2 100644 --- a/docs/report/csit_framework_documentation/csit_test_naming.rst +++ b/docs/report/csit_framework_documentation/csit_test_naming.rst @@ -6,8 +6,8 @@ Test Naming Background ---------- -CSIT |release| follows a common structured naming convention for all -performance and system functional tests, introduced in CSIT rls1701. +|csit-release| follows a common structured naming convention for all +performance and system functional tests, introduced in CSIT-1701. The naming should be intuitive for majority of the tests. Complete description of CSIT test naming convention is provided on diff --git a/docs/report/detailed_test_results/cot_performance_results/index.rst b/docs/report/detailed_test_results/cot_performance_results/index.rst index 656cb4259f..69ea95b593 100644 --- a/docs/report/detailed_test_results/cot_performance_results/index.rst +++ b/docs/report/detailed_test_results/cot_performance_results/index.rst @@ -4,5 +4,3 @@ VPP K8s Container Memif .. toctree:: ../cot_performance_results_3n_hsw/index - ../cot_performance_results_3n_skx/index - ../cot_performance_results_2n_skx/index diff --git a/docs/report/dpdk_performance_tests/csit_release_notes.rst b/docs/report/dpdk_performance_tests/csit_release_notes.rst index 3adbcd0d32..214eb4c4e7 100644 --- a/docs/report/dpdk_performance_tests/csit_release_notes.rst +++ b/docs/report/dpdk_performance_tests/csit_release_notes.rst @@ -1,7 +1,7 @@ Release Notes ============= -Changes in CSIT |release| +Changes in |csit-release| ------------------------- No code changes apart from bug fixes. @@ -9,7 +9,7 @@ No code changes apart from bug fixes. Known Issues ------------ -Here is the list of known issues in CSIT |release| for Testpmd performance tests: +Here is the list of known issues in |csit-release| for Testpmd performance tests: +---+---------------------------------------------------+------------+-----------------------------------------------------------------+ | # | Issue | Jira ID | Description | diff --git a/docs/report/dpdk_performance_tests/overview.rst b/docs/report/dpdk_performance_tests/overview.rst index 56d0e1e8fd..037300b3ee 100644 --- a/docs/report/dpdk_performance_tests/overview.rst +++ b/docs/report/dpdk_performance_tests/overview.rst @@ -87,7 +87,7 @@ Performance tests are split into two main categories: previously discovered throughput rate. These tests are currently done against 0.9 of reference NDR, with reference rates updated periodically. -CSIT |release| includes following performance test suites, listed per NIC type: +|csit-release| includes following performance test suites, listed per NIC type: - 2port10GE X520-DA2 Intel @@ -115,7 +115,7 @@ continuously. Performance Tests Naming ------------------------ -CSIT |release| follows a common structured naming convention for all performance +|csit-release| follows a common structured naming convention for all performance and system functional tests, introduced in CSIT rls1701. The naming should be intuitive for majority of the tests. Complete description @@ -124,7 +124,7 @@ of CSIT test naming convention is provided on :ref:`csit_test_naming`. Methodology: Multi-Core and Multi-Threading ------------------------------------------- -**Intel Hyper-Threading** - CSIT |release| performance tests are executed with +**Intel Hyper-Threading** - |csit-release| performance tests are executed with SUT servers' Intel XEON processors configured in Intel Hyper-Threading Disabled mode (BIOS setting). This is the simplest configuration used to establish baseline single-thread single-core application packet processing and forwarding @@ -132,7 +132,7 @@ performance. Subsequent releases of CSIT will add performance tests with Intel Hyper-Threading Enabled (requires BIOS settings change and hard reboot of server). -**Multi-core Tests** - CSIT |release| multi-core tests are executed in the +**Multi-core Tests** - |csit-release| multi-core tests are executed in the following VPP thread and core configurations: #. 1t1c - 1 pmd worker thread on 1 CPU physical core. diff --git a/docs/report/honeycomb_functional_tests/csit_release_notes.rst b/docs/report/honeycomb_functional_tests/csit_release_notes.rst index 3ac979f72f..21e2c6ac4a 100644 --- a/docs/report/honeycomb_functional_tests/csit_release_notes.rst +++ b/docs/report/honeycomb_functional_tests/csit_release_notes.rst @@ -1,7 +1,7 @@ Release Notes ============= -Changes in CSIT |release| +Changes in |csit-release| ------------------------- No changes. @@ -9,7 +9,7 @@ No changes. Known Issues ------------ -Here is the list of known issues in CSIT |release| for Honeycomb functional +Here is the list of known issues in |csit-release| for Honeycomb functional tests in VIRL: +---+--------------------------------------------+---------------+-------------------------------------------------------------------------+ diff --git a/docs/report/honeycomb_functional_tests/overview.rst b/docs/report/honeycomb_functional_tests/overview.rst index 741c2dd5e7..9100ed6e2c 100644 --- a/docs/report/honeycomb_functional_tests/overview.rst +++ b/docs/report/honeycomb_functional_tests/overview.rst @@ -46,7 +46,7 @@ for clarity). Functional Tests Coverage ------------------------- -The following Honeycomb functional test areas are included in the CSIT |release| +The following Honeycomb functional test areas are included in the |csit-release| with results listed in this report: - **Basic interface management** - CRUD for interface state, @@ -161,7 +161,7 @@ with results listed in this report: - Netconf/Restconf northbound over IPv6 - Test case count: 12 -Total 219 Honeycomb functional tests in the CSIT |release|. +Total 219 Honeycomb functional tests in the |csit-release|. Operational data in Honeycomb should mirror configuration data at all times. Because of this, test cases follow this general pattern: @@ -179,7 +179,7 @@ the DUT node. Functional Tests Naming ----------------------- -CSIT |release| introduced a common structured naming convention for all +|csit-release| introduced a common structured naming convention for all performance and functional tests. This change was driven by substantially growing number and type of CSIT test cases. Firstly, the original practice did not always follow any strict naming convention. Secondly test names did not diff --git a/docs/report/introduction/general_notes.rst b/docs/report/introduction/general_notes.rst index b94a82e2dc..25f08ab8e8 100644 --- a/docs/report/introduction/general_notes.rst +++ b/docs/report/introduction/general_notes.rst @@ -62,14 +62,14 @@ is listed separately, as follows: virtual FD.io testbeds focusing on NSH_SFC of VPP. Tests cover a range of CRUD operations executed against VPP. -In addition to above, CSIT |release| report does also include VPP unit test +In addition to above, |csit-release| report does also include VPP unit test results. VPP unit tests are developed within the FD.io VPP project and as they complement CSIT system functional tests, they are provided mainly as a reference and to provide a more complete view of automated testing executed against |vpp-release|. FD.io CSIT system is developed using two main coding platforms :abbr:`RF (Robot -Framework)` and Python2.7. CSIT |release| source code for the executed test +Framework)` and Python2.7. |csit-release| source code for the executed test suites is available in CSIT branch |release| in the directory :file:`./tests/`. A local copy of CSIT source code can be obtained by cloning CSIT git repository - :command:`git clone diff --git a/docs/report/introduction/methodology.rst b/docs/report/introduction/methodology.rst index 0b1182b9b9..b5b737ec69 100644 --- a/docs/report/introduction/methodology.rst +++ b/docs/report/introduction/methodology.rst @@ -117,7 +117,7 @@ Enabled mode (two logical cores per each physical core). HT setting is applied in BIOS and requires server SUT reload for it to take effect, making it impractical for continuous changes of HT mode of operation. -CSIT |release| performance tests are executed with server SUTs' Intel +|csit-release| performance tests are executed with server SUTs' Intel XEON processors configured with Intel Hyper-Threading Disabled for all Xeon Haswell testbeds (3n-hsw) and with Intel Hyper-Threading Enabled for all Xeon Skylake testbeds. @@ -128,7 +128,7 @@ More information about physical testbeds is provided in Multi-core Tests ~~~~~~~~~~~~~~~~ -CSIT |release| multi-core tests are executed in the following VPP worker +|csit-release| multi-core tests are executed in the following VPP worker thread and physical core configurations: #. Intel Xeon Haswell testbeds (3n-hsw) with Intel HT disabled @@ -239,7 +239,7 @@ following environment settings: LXC/DRC Container Memif ----------------------- -CSIT |release| includes tests taking advantage of VPP memif virtual +|csit-release| includes tests taking advantage of VPP memif virtual interface (shared memory interface) to interconnect VPP running in Containers. VPP vswitch instance runs in bare-metal user-mode handling NIC interfaces and connecting over memif (Slave side) to VPPs running in @@ -261,7 +261,7 @@ in :ref:`container_orchestration_in_csit`. K8s Container Memif ------------------- -CSIT |release| includes tests of VPP topologies running in K8s +|csit-release| includes tests of VPP topologies running in K8s orchestrated Pods/Containers and connected over memif virtual interfaces. In order to provide simple topology coding flexibility and extensibility container orchestration is done with `Kubernetes @@ -290,7 +290,7 @@ combination with HW cryptodev devices - Intel QAT 8950 50G - present in LF FD.io physical testbeds. DPDK cryptodev can be used for all IPSec data plane functions supported by VPP. -Currently CSIT |release| implements following IPSec test cases: +Currently |csit-release| implements following IPSec test cases: - AES-GCM, CBC-SHA1 ciphers, in combination with IPv4 routed-forwarding with Intel xl710 NIC. diff --git a/docs/report/introduction/overview.rst b/docs/report/introduction/overview.rst index 2d5a1e0671..e16a6e175f 100644 --- a/docs/report/introduction/overview.rst +++ b/docs/report/introduction/overview.rst @@ -2,23 +2,23 @@ Overview ======== This is the **F**\ast **D**\ata **I**/**O** Project (**FD.io**) **C**\ontinuous -**S**\ystem **I**\ntegration and **T**\esting (**CSIT**) project report for CSIT -|release| system testing of |vpp-release|. +**S**\ystem **I**\ntegration and **T**\esting (**CSIT**) project report for +|csit-release| system testing of |vpp-release|. This is the full html version, there is also a reduced `PDF version of this report`_. The report describes CSIT functional and performance tests and their -continuous execution delivered in CSIT |release|. A high-level overview is +continuous execution delivered in |csit-release|. A high-level overview is provided for each CSIT test environment running in :abbr:`LF (Linux Foundation)` FD.io Continuous Performance Labs. This is followed by summary of all executed tests against the |vpp-release| and associated FD.io projects and sub-systems -(Honeycomb, DPDK, NSH_SFC), CSIT |release| release notes, result highlights and +(Honeycomb, DPDK, NSH_SFC), |csit-release| release notes, result highlights and known issues discovered in CSIT. More detailed description of each environment, pointers to CSIT test code documentation and detailed test resuls with links to the source data files are also provided. -CSIT |release| report contains following main sections and sub-sections: +|csit-release| report contains following main sections and sub-sections: #. **Introduction** - general introduction to CSIT project; *Overview* - this section; *CSIT Test Naming* - CSIT general naming convention for test @@ -29,7 +29,7 @@ CSIT |release| report contains following main sections and sub-sections: #. **VPP Performance Tests** - VPP performance tests executed in physical FD.io testbeds; *Overview* - tested topologies, test coverage and naming specifics, methodology for multi-core, packet throughput and latency, and - KVM VM vhost tests; *CSIT Release Notes* - changes in CSIT |release|, added + KVM VM vhost tests; *CSIT Release Notes* - changes in |csit-release|, added tests, performance changes, environment or methodology changes, known CSIT issues; *Packet Throughput Graphs* and *Packet Latency Graphs* - plotted NDR, PDR throughput and latency results from multiple @@ -41,7 +41,7 @@ CSIT |release| report contains following main sections and sub-sections: #. **DPDK Performance Tests** - DPDK performance tests executed in physical FD.io testbeds; *Overview* - tested topologies, test coverage; - *CSIT Release Notes* - changes in CSIT |release|, any known CSIT issues; + *CSIT Release Notes* - changes in |csit-release|, any known CSIT issues; *Packet Throughput Graphs* and *Packet Latency Graphs* - plotted NDR, PDR throughput and latency results from multiple test job executions; *Test Environment* - environment description; *Documentation* - @@ -49,7 +49,7 @@ CSIT |release| report contains following main sections and sub-sections: #. **VPP Functional Tests** - VPP functional tests executed in virtual FD.io testbeds; *Overview* - tested virtual topologies, test coverage and - naming specifics; *CSIT Release Notes* - changes in CSIT |release|, added + naming specifics; *CSIT Release Notes* - changes in |csit-release|, added tests, environment or methodology changes, known CSIT issues, tests to be added; *Test Environment* - environment description ; *Documentation* - source code documentation for VPP functional tests. @@ -84,4 +84,4 @@ CSIT |release| report contains following main sections and sub-sections: #. **CSIT Framework Documentation** - description of the overall CSIT framework design hierarchy, CSIT test naming convention, followed by description of Presentation and Analytics Layer (PAL) introduced in - CSIT |release|. \ No newline at end of file + |csit-release|. \ No newline at end of file diff --git a/docs/report/nsh_sfc_functional_tests/csit_release_notes.rst b/docs/report/nsh_sfc_functional_tests/csit_release_notes.rst index 120f797be5..fb0f9b93c0 100644 --- a/docs/report/nsh_sfc_functional_tests/csit_release_notes.rst +++ b/docs/report/nsh_sfc_functional_tests/csit_release_notes.rst @@ -1,7 +1,7 @@ Release Notes ============= -Changes in CSIT |release| +Changes in |csit-release| ------------------------- #. Added NSH SFC functional tests @@ -11,7 +11,7 @@ Changes in CSIT |release| Known Issues ------------ -Here is the list of known issues in CSIT |release| for NSH_SFC functional tests in VIRL: +Here is the list of known issues in |csit-release| for NSH_SFC functional tests in VIRL: +---+-------------------------------------------------+----------+------------------------------------------------------+ | # | Issue | Jira ID | Description | diff --git a/docs/report/nsh_sfc_functional_tests/overview.rst b/docs/report/nsh_sfc_functional_tests/overview.rst index 754fd9242d..a3010f0ae6 100644 --- a/docs/report/nsh_sfc_functional_tests/overview.rst +++ b/docs/report/nsh_sfc_functional_tests/overview.rst @@ -93,7 +93,7 @@ is shown in the figure below including the applicable packet flow thru the VM NSH_SFC Functional Tests Coverage --------------------------------- -Following NSH_SFC functional test areas are covered in the CSIT |release| with +Following NSH_SFC functional test areas are covered in the |csit-release| with results listed in this report: - **NSH SFC Classifier** - TG sends some TCP packets to test NSH SFC @@ -114,5 +114,5 @@ results listed in this report: VXLAN-GPE-NSH encapsulated packtes to the TG from other NIC. - Test case count: 6 -Total 25 NSH SFC functional tests in the CSIT |release|. +Total 25 NSH SFC functional tests in the |csit-release|. diff --git a/docs/report/test_operational_data/vpp_performance_operational_data/index.rst b/docs/report/test_operational_data/vpp_performance_operational_data/index.rst index a2abf4d04a..29eb62ee26 100644 --- a/docs/report/test_operational_data/vpp_performance_operational_data/index.rst +++ b/docs/report/test_operational_data/vpp_performance_operational_data/index.rst @@ -1,12 +1,8 @@ VPP Performance NDR/PDR ======================= -.. note:: - - Data sources for reported test results: i) `FD.io test executor vpp - performance jobs`_, ii) archived FD.io jobs test result `output files - <../../_static/archive/>`_. - .. toctree:: - vpp_performance_operational_data + ../vpp_performance_operational_data_3n_hsw/index + ../vpp_performance_operational_data_3n_skx/index + ../vpp_performance_operational_data_2n_skx/index diff --git a/docs/report/detailed_test_results/cot_performance_results_2n_skx/index.rst b/docs/report/test_operational_data/vpp_performance_operational_data_2n_skx/index.rst similarity index 80% rename from docs/report/detailed_test_results/cot_performance_results_2n_skx/index.rst rename to docs/report/test_operational_data/vpp_performance_operational_data_2n_skx/index.rst index 503caa70c1..3b3692c812 100644 --- a/docs/report/detailed_test_results/cot_performance_results_2n_skx/index.rst +++ b/docs/report/test_operational_data/vpp_performance_operational_data_2n_skx/index.rst @@ -3,10 +3,10 @@ .. note:: - Data sources for reported test results: i) `FD.io test executor ligato + Data sources for reported test results: i) `FD.io test executor vpp performance jobs`_, ii) archived FD.io jobs test result `output files <../../_static/archive/>`_. .. toctree:: - cot_performance_results_2n_skx + vpp_performance_operational_data_2n_skx diff --git a/docs/report/test_operational_data/vpp_performance_operational_data_3n_hsw/index.rst b/docs/report/test_operational_data/vpp_performance_operational_data_3n_hsw/index.rst new file mode 100644 index 0000000000..33ad7e4426 --- /dev/null +++ b/docs/report/test_operational_data/vpp_performance_operational_data_3n_hsw/index.rst @@ -0,0 +1,12 @@ +2n-hsw +------ + +.. note:: + + Data sources for reported test results: i) `FD.io test executor vpp + performance jobs`_, ii) archived FD.io jobs test result `output files + <../../_static/archive/>`_. + +.. toctree:: + + vpp_performance_operational_data_3n_hsw diff --git a/docs/report/detailed_test_results/cot_performance_results_3n_skx/index.rst b/docs/report/test_operational_data/vpp_performance_operational_data_3n_skx/index.rst similarity index 80% rename from docs/report/detailed_test_results/cot_performance_results_3n_skx/index.rst rename to docs/report/test_operational_data/vpp_performance_operational_data_3n_skx/index.rst index 1de4f9cf95..743436e659 100644 --- a/docs/report/detailed_test_results/cot_performance_results_3n_skx/index.rst +++ b/docs/report/test_operational_data/vpp_performance_operational_data_3n_skx/index.rst @@ -3,10 +3,10 @@ .. note:: - Data sources for reported test results: i) `FD.io test executor ligato + Data sources for reported test results: i) `FD.io test executor vpp performance jobs`_, ii) archived FD.io jobs test result `output files <../../_static/archive/>`_. .. toctree:: - cot_performance_results_3n_skx + vpp_performance_operational_data_3n_skx diff --git a/docs/report/vpp_functional_tests/csit_release_notes.rst b/docs/report/vpp_functional_tests/csit_release_notes.rst index dcb317f8ce..f597ce0c9f 100644 --- a/docs/report/vpp_functional_tests/csit_release_notes.rst +++ b/docs/report/vpp_functional_tests/csit_release_notes.rst @@ -1,7 +1,7 @@ Release Notes ============= -Changes in CSIT |release| +Changes in |csit-release| ------------------------- #. VPP functional test framework changes: @@ -19,7 +19,7 @@ Changes in CSIT |release| Known Issues ------------ -List of known issues in CSIT |release| for VPP functional tests in VIRL: +List of known issues in |csit-release| for VPP functional tests in VIRL: +---+-------------------------------------------------+----------+------------------------------------------------------+ | # | .. centered:: Issue | Jira ID | .. centered:: Description | diff --git a/docs/report/vpp_functional_tests/overview.rst b/docs/report/vpp_functional_tests/overview.rst index 32d70f102b..90afd4ec83 100644 --- a/docs/report/vpp_functional_tests/overview.rst +++ b/docs/report/vpp_functional_tests/overview.rst @@ -93,7 +93,7 @@ is shown in the figure below including the applicable packet flow thru the VM Functional Tests Coverage ------------------------- -Following VPP functional test areas are covered in the CSIT |release| with +Following VPP functional test areas are covered in the |csit-release| with results listed in this report: - **DHCP - Client and Proxy** - Dynamic Host Control Protocol Client and Proxy @@ -127,7 +127,7 @@ results listed in this report: Functional Tests Naming ----------------------- -CSIT |release| follows a common structured naming convention for all performance +|csit-release| follows a common structured naming convention for all performance and system functional tests, introduced in CSIT rls1701. The naming should be intuitive for majority of the tests. Complete description diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst index 1954ad1a49..4fdee9738c 100644 --- a/docs/report/vpp_performance_tests/csit_release_notes.rst +++ b/docs/report/vpp_performance_tests/csit_release_notes.rst @@ -1,7 +1,7 @@ Release Notes ============= -Changes in CSIT |release| +Changes in |csit-release| ------------------------- #. **VPP performance tests** @@ -137,7 +137,7 @@ provides continuous performance trending for VPP master branch: Known Issues ------------ -List of known issues in CSIT |release| for VPP performance tests: +List of known issues in |csit-release| for VPP performance tests: +---+-------------------------------------------------+------------+-----------------------------------------------------------------+ | # | Issue | Jira ID | Description | diff --git a/docs/report/vpp_performance_tests/documentation/containers.rst b/docs/report/vpp_performance_tests/documentation/containers.rst index c6428d2e2a..5ec7d9c112 100644 --- a/docs/report/vpp_performance_tests/documentation/containers.rst +++ b/docs/report/vpp_performance_tests/documentation/containers.rst @@ -504,7 +504,7 @@ Listed CSIT container networking test topologies are defined with DUT containerized VPP switch forwarding packets between NF containers. Each NF container runs their own instance of VPP in L2XC configuration. -Following container networking topologies are tested in CSIT |release|: +Following container networking topologies are tested in |csit-release|: - LXC topologies: diff --git a/docs/report/vpp_performance_tests/overview.rst b/docs/report/vpp_performance_tests/overview.rst index 73441d12db..86116bf7c5 100644 --- a/docs/report/vpp_performance_tests/overview.rst +++ b/docs/report/vpp_performance_tests/overview.rst @@ -87,7 +87,7 @@ VM Service Switching VM service switching topology test cases require VPP DUT to communicate with Virtual Machines (VMs) over vhost-user virtual interfaces. -Two types of VM service topologies are tested in CSIT |release|: +Two types of VM service topologies are tested in |csit-release|: #. "Parallel" topology with packets flowing within SUT from NIC(s) via VPP DUT to VM, back to VPP DUT, then out thru NIC(s). @@ -161,7 +161,7 @@ Container Service Switching Container service switching topology test cases require VPP DUT to communicate with Containers (Ctrs) over memif virtual interfaces. -Three types of VM service topologies are tested in CSIT |release|: +Three types of VM service topologies are tested in |csit-release|: #. "Parallel" topology with packets flowing within SUT from NIC(s) via VPP DUT to Container, back to VPP DUT, then out thru NIC(s). @@ -259,7 +259,7 @@ topologies and configurations: regardless of packet loss. Maximum load for specified Ethernet frame size is set to the bi-directional link rate. -CSIT |release| includes following performance test areas covered across +|csit-release| includes following performance test areas covered across a range of NIC drivers and NIC models: +-----------------------+----------------------------------------------+ @@ -349,7 +349,7 @@ NIC models has been limited to few baseline tests. Performance Tests Naming ------------------------ -FD.io CSIT |release| follows a common structured naming convention for +FD.io |csit-release| follows a common structured naming convention for all performance and system functional tests, introduced in CSIT rls1701. The naming should be intuitive for majority of the tests. Complete diff --git a/resources/tools/presentation/conf.py b/resources/tools/presentation/conf.py index 5544c29f80..25d5b714a4 100644 --- a/resources/tools/presentation/conf.py +++ b/resources/tools/presentation/conf.py @@ -45,7 +45,7 @@ source_suffix = ['.rst', '.md'] master_doc = 'index' # General information about the project. -project = u'FD.io CSIT rls1807 Report' +project = u'FD.io CSIT-18.07' copyright = u'2018, FD.io' author = u'FD.io CSIT' @@ -61,6 +61,7 @@ author = u'FD.io CSIT' rst_epilog = """ .. |release-1| replace:: {prev_release} .. |srelease| replace:: {srelease} +.. |csit-release| replace:: CSIT-{csitrelease} .. |vpp-release| replace:: VPP-{vpprelease} release .. |vpp-release-1| replace:: VPP-{vpp_prev_release} release .. |dpdk-release| replace:: DPDK {dpdkrelease} @@ -97,6 +98,7 @@ rst_epilog = """ """.format(release='rls1807', prev_release='rls1804', srelease='1807', + csitrelease='18.07', vpprelease='18.07', vpp_prev_release='18.04', dpdkrelease='18.05', diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 35490e27b9..1f9ea61816 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -37,8 +37,6 @@ DIR[DTR,MRR,VPP,3N,SKX]: "{DIR[DTR]}/vpp_mrr_results_3n_skx" DIR[DTR,MRR,VPP,2N,SKX]: "{DIR[DTR]}/vpp_mrr_results_2n_skx" DIR[DTR,PERF,COT,3N,HSW]: "{DIR[DTR]}/cot_performance_results_3n_hsw" - DIR[DTR,PERF,COT,3N,SKX]: "{DIR[DTR]}/cot_performance_results_3n_skx" - DIR[DTR,PERF,COT,2N,SKX]: "{DIR[DTR]}/cot_performance_results_2n_skx" DIR[DTR,FUNC,VPP,UBUNTU]: "{DIR[DTR]}/vpp_functional_results_ubuntu" DIR[DTR,FUNC,VPP,CENTOS]: "{DIR[DTR]}/vpp_functional_results_centos" DIR[DTR,FUNC,HC]: "{DIR[DTR]}/honeycomb_functional_results" @@ -58,7 +56,9 @@ # Detailed tests operational data DIR[DTO]: "{DIR[WORKING,SRC]}/test_operational_data" - DIR[DTO,PERF,VPP]: "{DIR[DTO]}/vpp_performance_operational_data" + DIR[DTO,PERF,VPP,3N,HSW]: "{DIR[DTO]}/vpp_performance_operational_data_3n_hsw" + DIR[DTO,PERF,VPP,3N,SKX]: "{DIR[DTO]}/vpp_performance_operational_data_3n_skx" + DIR[DTO,PERF,VPP,2N,SKX]: "{DIR[DTO]}/vpp_performance_operational_data_2n_skx" # .css patch file to fix tables generated by Sphinx DIR[CSS_PATCH_FILE]: "{DIR[STATIC]}/theme_overrides.css" @@ -1286,10 +1286,10 @@ - type: "table" - title: "Test Operational Data - VPP Performance Operational Data" + title: "Test Operational Data - VPP Performance Operational Data 3n-hsw" algorithm: "table_merged_details" output-file-ext: ".csv" - output-file: "{DIR[DTO,PERF,VPP]}/vpp_test_operational" + output-file: "{DIR[DTO,PERF,VPP,3N,HSW]}/vpp_test_operational_3n_hsw" columns: - title: "Name" @@ -1298,8 +1298,49 @@ title: "VPP Operational Data - Outputs of 'show runtime' at NDR packet rate" data: "data show-run" rows: "generated" - data: - "vpp-perf-results-3n-hsw" + data: "vpp-perf-results-3n-hsw" + filter: "not 'NDRCHK' and not 'PDRCHK'" + parameters: + - "parent" + - "name" + - "show-run" + +- + type: "table" + title: "Test Operational Data - VPP Performance Operational Data 3n-skx" + algorithm: "table_merged_details" + output-file-ext: ".csv" + output-file: "{DIR[DTO,PERF,VPP,3N,SKX]}/vpp_test_operational_3n_skx" + columns: + - + title: "Name" + data: "data name" + - + title: "VPP Operational Data - Outputs of 'show runtime' at NDR packet rate" + data: "data show-run" + rows: "generated" + data: "vpp-perf-results-3n-skx" + filter: "not 'NDRCHK' and not 'PDRCHK'" + parameters: + - "parent" + - "name" + - "show-run" + +- + type: "table" + title: "Test Operational Data - VPP Performance Operational Data 2n-skx" + algorithm: "table_merged_details" + output-file-ext: ".csv" + output-file: "{DIR[DTO,PERF,VPP,2N,SKX]}/vpp_test_operational_2n_skx" + columns: + - + title: "Name" + data: "data name" + - + title: "VPP Operational Data - Outputs of 'show runtime' at NDR packet rate" + data: "data show-run" + rows: "generated" + data: "vpp-perf-results-2n-skx" filter: "not 'NDRCHK' and not 'PDRCHK'" parameters: - "parent" @@ -1562,56 +1603,6 @@ - "doc" - "msg" -- - type: "table" - title: "Detailed Test Results - Container Orchestrated Topologies Performance Results 3n-skx" - algorithm: "table_details" - output-file-ext: ".csv" - output-file: "{DIR[DTR,PERF,COT,3N,SKX]}/cot_performance_results_3n_skx" - columns: - - - title: "Name" - data: "data name" - - - title: "Documentation" - data: "data doc" - - - title: "Status" - data: "data msg" - rows: "generated" - data: "ligato-perf-results-3n-skx" - filter: "all" - parameters: - - "name" - - "parent" - - "doc" - - "msg" - -- - type: "table" - title: "Detailed Test Results - Container Orchestrated Topologies Performance Results 2n-skx" - algorithm: "table_details" - output-file-ext: ".csv" - output-file: "{DIR[DTR,PERF,COT,2N,SKX]}/cot_performance_results_2n_skx" - columns: - - - title: "Name" - data: "data name" - - - title: "Documentation" - data: "data doc" - - - title: "Status" - data: "data msg" - rows: "generated" - data: "ligato-perf-results-2n-skx" - filter: "all" - parameters: - - "name" - - "parent" - - "doc" - - "msg" - - type: "table" title: "Detailed Test Results - DPDK Performance Results 3n-hsw" @@ -1784,14 +1775,13 @@ - type: "file" - title: "VPP Performance Configuration" + title: "VPP Performance Configuration 3n-hsw" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTC,PERF,VPP]}/vpp_performance_configuration" + output-file: "{DIR[DTC,PERF,VPP,3N,HSW]}/vpp_performance_configuration_3n_hsw" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTC,PERF,VPP]}" - data: - "vpp-perf-results-3n-hsw" + dir-tables: "{DIR[DTC,PERF,VPP,3N,HSW]}" + data: "vpp-perf-results-3n-hsw" filter: "not 'NDRCHK' and not 'PDRCHK'" parameters: - "name" @@ -1802,14 +1792,81 @@ - type: "file" - title: "VPP Performance Operational Data" + title: "VPP Performance Configuration 3n-skx" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTO,PERF,VPP]}/vpp_performance_operational_data" + output-file: "{DIR[DTC,PERF,VPP,3N,SKX]}/vpp_performance_configuration_3n_skx" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTO,PERF,VPP]}" - data: - "vpp-perf-results-3n-hsw" + dir-tables: "{DIR[DTC,PERF,VPP,3N,SKX]}" + data: "vpp-perf-results-3n-skx" + filter: "not 'NDRCHK' and not 'PDRCHK'" + parameters: + - "name" + - "doc" + - "level" + - "parent" + data-start-level: 3 + +- + type: "file" + title: "VPP Performance Configuration 2n-skx" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTC,PERF,VPP,2N,SKX]}/vpp_performance_configuration_2n_skx" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTC,PERF,VPP,2N,SKX]}" + data: "vpp-perf-results-2n-skx" + filter: "not 'NDRCHK' and not 'PDRCHK'" + parameters: + - "name" + - "doc" + - "level" + - "parent" + data-start-level: 3 + +- + type: "file" + title: "VPP Performance Operational Data 3n-hsw" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTO,PERF,VPP,3N,HSW]}/vpp_performance_operational_data_3n_hsw" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTO,PERF,VPP,3N,HSW]}" + data: "vpp-perf-results-3n-hsw" + filter: "not 'NDRCHK' and not 'PDRCHK'" + parameters: + - "name" + - "doc" + - "level" + - "parent" + data-start-level: 3 + +- + type: "file" + title: "VPP Performance Operational Data 3n-skx" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTO,PERF,VPP,3N,SKX]}/vpp_performance_operational_data_3n_skx" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTO,PERF,VPP,3N,SKX]}" + data: "vpp-perf-results-3n-skx" + filter: "not 'NDRCHK' and not 'PDRCHK'" + parameters: + - "name" + - "doc" + - "level" + - "parent" + data-start-level: 3 + +- + type: "file" + title: "VPP Performance Operational Data 2n-skx" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTO,PERF,VPP,2N,SKX]}/vpp_performance_operational_data_2n_skx" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTO,PERF,VPP,2N,SKX]}" + data: "vpp-perf-results-2n-skx" filter: "not 'NDRCHK' and not 'PDRCHK'" parameters: - "name" @@ -1871,14 +1928,13 @@ - type: "file" - title: "VPP MRR Configuration" + title: "VPP MRR Configuration 3n-hsw" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTC,MRR,VPP]}/vpp_mrr_configuration" + output-file: "{DIR[DTC,MRR,VPP,3N,HSW]}/vpp_mrr_configuration_3n_hsw" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTC,MRR,VPP]}" - data: - "vpp-mrr-results-3n-hsw" + dir-tables: "{DIR[DTC,MRR,VPP,3N,HSW]}" + data: "vpp-mrr-results-3n-hsw" filter: "'MRR'" parameters: - "name" @@ -1889,14 +1945,14 @@ - type: "file" - title: "VPP Functional Results - Ubuntu" + title: "VPP MRR Configuration 3n-skx" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTR,FUNC,VPP,UBUNTU]}/vpp_functional_results_ubuntu" + output-file: "{DIR[DTC,MRR,VPP,3N,SKX]}/vpp_mrr_configuration_3n_skx" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTR,FUNC,VPP,UBUNTU]}" - data: "vpp-func-results-ubuntu" - filter: "all" + dir-tables: "{DIR[DTC,MRR,VPP,3N,SKX]}" + data: "vpp-mrr-results-3n-skx" + filter: "'MRR'" parameters: - "name" - "doc" @@ -1906,15 +1962,14 @@ - type: "file" - title: "VPP Functional Results - CentOS" + title: "VPP MRR Configuration 2n-skx" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTR,FUNC,VPP,CENTOS]}/vpp_functional_results_centos" + output-file: "{DIR[DTC,MRR,VPP,2N,SKX]}/vpp_mrr_configuration_2n_skx" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTR,FUNC,VPP,CENTOS]}" - data: - "vpp-func-results-centos" - filter: "all" + dir-tables: "{DIR[DTC,MRR,VPP,2N,SKX]}" + data: "vpp-mrr-results-2n-skx" + filter: "'MRR'" parameters: - "name" - "doc" @@ -1924,30 +1979,29 @@ - type: "file" - title: "VPP Functional Configuration - Ubuntu" + title: "VPP Functional Results - Ubuntu" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTC,FUNC,VPP]}/vpp_functional_configuration" + output-file: "{DIR[DTR,FUNC,VPP,UBUNTU]}/vpp_functional_results_ubuntu" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTC,FUNC,VPP]}" - data: - "vpp-func-results-ubuntu" + dir-tables: "{DIR[DTR,FUNC,VPP,UBUNTU]}" + data: "vpp-func-results-ubuntu" filter: "all" parameters: - "name" - "doc" - "level" - "parent" - data-start-level: 3 + data-start-level: 2 - type: "file" - title: "VPP Functional Configuration - CentOS" + title: "VPP Functional Results - CentOS" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTC,FUNC,VPP,CENTOS]}/vpp_functional_configuration_centos" + output-file: "{DIR[DTR,FUNC,VPP,CENTOS]}/vpp_functional_results_centos" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTC,FUNC,VPP,CENTOS]}" + dir-tables: "{DIR[DTR,FUNC,VPP,CENTOS]}" data: "vpp-func-results-centos" filter: "all" @@ -1956,58 +2010,59 @@ - "doc" - "level" - "parent" - data-start-level: 3 + data-start-level: 2 - type: "file" - title: "Container Orchestrated Performance Results 3n-hsw" + title: "VPP Functional Configuration - Ubuntu" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTR,PERF,COT,3N,HSW]}/cot_performance_results_3n_hsw" + output-file: "{DIR[DTC,FUNC,VPP,UBUNTU]}/vpp_functional_configuration_ubuntu" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTR,PERF,COT,3N,HSW]}" - data: "ligato-perf-results-3n-hsw" + dir-tables: "{DIR[DTC,FUNC,VPP,UBUNTU]}" + data: "vpp-func-results-ubuntu" filter: "all" parameters: - "name" - "doc" - "level" - "parent" - data-start-level: 3 + data-start-level: 2 - type: "file" - title: "Container Orchestrated Performance Results 3n-skx" + title: "VPP Functional Configuration - CentOS" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTR,PERF,COT,3N,SKX]}/cot_performance_results_3n_skx" + output-file: "{DIR[DTC,FUNC,VPP,CENTOS]}/vpp_functional_configuration_centos" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTR,PERF,COT,3N,SKX]}" - data: "ligato-perf-results-3n-skx" + dir-tables: "{DIR[DTC,FUNC,VPP,CENTOS]}" + data: + "vpp-func-results-centos" filter: "all" parameters: - "name" - "doc" - "level" - "parent" - data-start-level: 3 + data-start-level: 2 - type: "file" - title: "Container Orchestrated Performance Results 2n-skx" + title: "Container Orchestrated Performance Results 3n-hsw" algorithm: "file_test_results" output-file-ext: ".rst" - output-file: "{DIR[DTR,PERF,COT,2N,SKX]}/cot_performance_results_2n_skx" + output-file: "{DIR[DTR,PERF,COT,3N,HSW]}/cot_performance_results_3n_hsw" file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" - dir-tables: "{DIR[DTR,PERF,COT,2N,SKX]}" - data: "ligato-perf-results-2n-skx" + dir-tables: "{DIR[DTR,PERF,COT,3N,HSW]}" + data: "ligato-perf-results-3n-hsw" filter: "all" parameters: - "name" - "doc" - "level" - "parent" - data-start-level: 2 + data-start-level: 3 - type: "file" -- 2.16.6