From 468da48d3f3ced298ff8dccc76885157dff5cfe7 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Tue, 7 Aug 2018 13:05:06 +0200 Subject: [PATCH] CSIT-1229: Add DMM to report Change-Id: Ib3ea8c13371a29ea2b4661a8506190dcb15c8ced Signed-off-by: Tibor Frank Signed-off-by: sharath --- .../dmm_functional_results/index.rst | 12 ++++ .../dmm_functional_tests/csit_release_notes.rst | 22 ++++++++ docs/report/dmm_functional_tests/documentation.rst | 6 ++ docs/report/dmm_functional_tests/index.rst | 9 +++ docs/report/dmm_functional_tests/overview.rst | 66 ++++++++++++++++++++++ .../dmm_functional_tests/test_environment.rst | 1 + docs/report/index.html.template | 10 ++++ docs/report/index.pdf.template | 1 + docs/report/introduction/overview.rst | 7 +++ resources/tools/presentation/conf.py | 2 + resources/tools/presentation/specification.yaml | 56 ++++++++++++++++++ 11 files changed, 192 insertions(+) create mode 100644 docs/report/detailed_test_results/dmm_functional_results/index.rst create mode 100644 docs/report/dmm_functional_tests/csit_release_notes.rst create mode 100644 docs/report/dmm_functional_tests/documentation.rst create mode 100644 docs/report/dmm_functional_tests/index.rst create mode 100644 docs/report/dmm_functional_tests/overview.rst create mode 100644 docs/report/dmm_functional_tests/test_environment.rst diff --git a/docs/report/detailed_test_results/dmm_functional_results/index.rst b/docs/report/detailed_test_results/dmm_functional_results/index.rst new file mode 100644 index 0000000000..2acd888914 --- /dev/null +++ b/docs/report/detailed_test_results/dmm_functional_results/index.rst @@ -0,0 +1,12 @@ +DMM Functional +============== + +.. note:: + + Data sources for reported test results: i) `FD.io test executor DMM + functional jobs`_, ii) archived FD.io jobs test result `output files + <../../_static/archive/>`_. + +.. toctree:: + + dmm_functional_results diff --git a/docs/report/dmm_functional_tests/csit_release_notes.rst b/docs/report/dmm_functional_tests/csit_release_notes.rst new file mode 100644 index 0000000000..04b41f3c60 --- /dev/null +++ b/docs/report/dmm_functional_tests/csit_release_notes.rst @@ -0,0 +1,22 @@ +Release Notes +============= + +Changes in |csit-release| +------------------------- + +#. Added DMM functional tests + + - DMM base vs epoll test case + +Known Issues +------------ + +Here is the list of known issues in |csit-release| for DMM functional tests in +VIRL: + ++---+-------------------------------------------------+----------+------------------------------------------------------+ +| # | Issue | Jira ID | Description | ++---+-------------------------------------------------+----------+------------------------------------------------------+ +| 1 | DMM CentOs job is disabled | DMM-17 | DPDK version and relative kernel version issue, | +| | | | CentOS job is temporarily disabled. | ++---+-------------------------------------------------+----------+------------------------------------------------------+ diff --git a/docs/report/dmm_functional_tests/documentation.rst b/docs/report/dmm_functional_tests/documentation.rst new file mode 100644 index 0000000000..56fb357a60 --- /dev/null +++ b/docs/report/dmm_functional_tests/documentation.rst @@ -0,0 +1,6 @@ +Documentation +============= + +`CSIT DMM Functional Tests Documentation`_ contains detailed +functional description and input parameters for each test case. + diff --git a/docs/report/dmm_functional_tests/index.rst b/docs/report/dmm_functional_tests/index.rst new file mode 100644 index 0000000000..defb9ddbc8 --- /dev/null +++ b/docs/report/dmm_functional_tests/index.rst @@ -0,0 +1,9 @@ +DMM Functional +============== + +.. toctree:: + + overview + csit_release_notes + test_environment + documentation diff --git a/docs/report/dmm_functional_tests/overview.rst b/docs/report/dmm_functional_tests/overview.rst new file mode 100644 index 0000000000..d30398a282 --- /dev/null +++ b/docs/report/dmm_functional_tests/overview.rst @@ -0,0 +1,66 @@ +Overview +======== + +Virtual Topologies +------------------ + +CSIT DMM functional tests are executed on virtualized topologies created using +:abbr:`VIRL (Virtual Internet Routing Lab)` simulation platform contributed by +Cisco. VIRL runs on physical baremetal servers hosted by LF FD.io project. +Based on the packet path through server SUTs, one logical topology type +is used for DMM DUT data plane testing: + +#. NIC-to-NIC switching topologies. + +NIC-to-NIC Switching +~~~~~~~~~~~~~~~~~~~~ + +The simplest logical topology for software data plane application like +DMM is NIC-to-NIC switching. Tested topologies for the 3-Node +testbed is shown in the figure below. + +.. only:: latex + + .. raw:: latex + + \begin{figure}[H] + \centering + \graphicspath{{../_tmp/src/vpp_performance_tests/}} + \includegraphics[width=0.90\textwidth]{logical-3n-nic2nic} + \label{fig:logical-3n-nic2nic} + \end{figure} + +.. only:: html + + .. figure:: ../vpp_performance_tests/logical-3n-nic2nic.svg + :alt: logical-3n-nic2nic + :align: center + +SUT1 and SUT2 are two VMs (Ubuntu), TG +is a Traffic Generator (TG, another Ubuntu VM). SUTs run sample server-client +application using the DMM libs in Linux user-mode as a Device Under Test (DUT) +within the VM. Currently TG node is not being used in DMM-CSIT. Logical +connectivity between SUTs is provided using virtual NICs using VMs' virtio +driver. + +Virtual testbeds are created on-demand whenever a verification job is started +(e.g. triggered by the gerrit patch submission) and destroyed upon completion +of all functional tests. Each node is a Virtual Machine and each connection +that is drawn on the diagram is available for use in any test case. During the +test execution, all nodes are reachable through the Management network connected +to every node via dedicated virtual NICs and virtual links (not shown above +for clarity). + +DMM Functional Tests Coverage +----------------------------- + +Following DMM functional test areas are covered in the |csit-release| with +results listed in this report: + +- **DMM basic testcase** - DMM has only one test case right now. + The testcase demonstrates single server[DUT1] and single client[DUT2] scenario + using DMM framework and kernel tcp/ip stack. + + - Test case count: 1 + +Total 1 DMM functional test in the |csit-release|. diff --git a/docs/report/dmm_functional_tests/test_environment.rst b/docs/report/dmm_functional_tests/test_environment.rst new file mode 100644 index 0000000000..3b35fd432f --- /dev/null +++ b/docs/report/dmm_functional_tests/test_environment.rst @@ -0,0 +1 @@ +.. include:: ../vpp_functional_tests/test_environment.rst diff --git a/docs/report/index.html.template b/docs/report/index.html.template index b5b55c0dd2..f262ea36e5 100644 --- a/docs/report/index.html.template +++ b/docs/report/index.html.template @@ -61,6 +61,15 @@ CSIT 18.07 nsh_sfc_functional_tests/test_environment nsh_sfc_functional_tests/documentation +.. toctree:: + :maxdepth: 2 + :caption: DMM Functional + + dmm_functional_tests/overview + dmm_functional_tests/csit_release_notes + dmm_functional_tests/test_environment + dmm_functional_tests/documentation + .. toctree:: :maxdepth: 2 :caption: Detailed Results @@ -72,6 +81,7 @@ CSIT 18.07 detailed_test_results/vpp_functional_results/index detailed_test_results/honeycomb_functional_results/index detailed_test_results/nshsfc_functional_results/index + detailed_test_results/dmm_functional_results/index .. toctree:: :maxdepth: 2 diff --git a/docs/report/index.pdf.template b/docs/report/index.pdf.template index 0a0ba48593..5db11eb388 100644 --- a/docs/report/index.pdf.template +++ b/docs/report/index.pdf.template @@ -10,4 +10,5 @@ CSIT 18.07 vpp_functional_tests/index honeycomb_functional_tests/index nsh_sfc_functional_tests/index + dmm_functional_tests/index csit_framework_documentation/index diff --git a/docs/report/introduction/overview.rst b/docs/report/introduction/overview.rst index ea96c85b76..c3537a9f66 100644 --- a/docs/report/introduction/overview.rst +++ b/docs/report/introduction/overview.rst @@ -67,6 +67,13 @@ the source data files are also provided. *Test Environment* - environment description; *Documentation* - source code documentation for NSH_SFC functional tests. +#. **DMM Functional** - DMM functional tests executed in + virtual FD.io testbeds; *Overview* - tested virtual topologies, test + coverage and naming specifics; *CSIT Release Notes* - changes in CSIT + |release|, added tests, environment or methodology changes, known CSIT issues; + *Test Environment* - environment description; + *Documentation* - source code documentation for DMM functional tests. + #. **Detailed Results** - auto-generated results from CSIT jobs executions using CSIT Robot Framework output files as source data; *VPP Performance Results*, *DPDK Performance Results*, *VPP Functional diff --git a/resources/tools/presentation/conf.py b/resources/tools/presentation/conf.py index 8236cab605..fb6311061a 100644 --- a/resources/tools/presentation/conf.py +++ b/resources/tools/presentation/conf.py @@ -84,6 +84,7 @@ rst_epilog = """ .. _CSIT VPP Functional Tests Documentation: https://docs.fd.io/csit/{release}/doc/tests.vpp.func.html .. _CSIT VPP Performance Tests Documentation: https://docs.fd.io/csit/{release}/doc/tests.vpp.perf.html .. _CSIT NSH_SFC Functional Tests Documentation: https://docs.fd.io/csit/{release}/doc/tests.nsh_sfc.func.html +.. _CSIT DMM Functional Tests Documentation: https://docs.fd.io/csit/{release}/doc/tests.dmm.func.html .. _VPP test framework documentation: https://docs.fd.io/vpp/{vpprelease}/vpp_make_test/html/ .. _FD.io test executor dpdk performance job 3n-hsw: https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-verify-{srelease}-3n-hsw .. _FD.io test executor dpdk performance job 3n-skx: https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-verify-{srelease}-3n-skx @@ -96,6 +97,7 @@ rst_epilog = """ .. _FD.io test executor vpp functional jobs using CentOs: https://jenkins.fd.io/view/csit/job/csit-vpp-functional-{srelease}-centos7-virl .. _FD.io test executor Honeycomb functional jobs: https://jenkins.fd.io/view/csit/job/hc2vpp-csit-integration-{srelease}-ubuntu1604 .. _FD.io test executor NSH_SFC functional jobs: https://jenkins.fd.io/view/csit/job/csit-nsh_sfc-verify-func-{srelease}-ubuntu1604-virl +.. _FD.io test executor DMM functional jobs: https://jenkins.fd.io/view/csit/job/csit-dmm-functional-{srelease}-ubuntu1604-virl .. _FD.io VPP compile job: https://jenkins.fd.io/view/vpp/job/vpp-merge-{srelease}-ubuntu1604/ .. _FD.io DPDK compile job: https://jenkins.fd.io/view/deb-dpdk/job/deb_dpdk-merge-{sdpdkrelease}-ubuntu1604/ .. _CSIT Testbed Setup: https://git.fd.io/csit/tree/resources/tools/testbed-setup/README.md?h={release} diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index f186b43868..67c43bdbc5 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -41,6 +41,7 @@ DIR[DTR,FUNC,VPP,CENTOS]: "{DIR[DTR]}/vpp_functional_results_centos" DIR[DTR,FUNC,HC]: "{DIR[DTR]}/honeycomb_functional_results" DIR[DTR,FUNC,NSHSFC]: "{DIR[DTR]}/nshsfc_functional_results" + DIR[DTR,FUNC,DMM]: "{DIR[DTR]}/dmm_functional_results" DIR[DTR,PERF,VPP,IMPRV]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_improvements" # Detailed test configurations @@ -448,6 +449,12 @@ csit-nsh_sfc-verify-func-1807-ubuntu1604-virl: - 2 + # DMM Functional + dmm-func-results: + # TODO: Replace with 1807 job: + csit-dmm-functional-master-ubuntu1604-virl: + - 95 + plot-layouts: plot-cps: @@ -855,6 +862,9 @@ - "lastSuccessfulBuild" csit-nsh_sfc-verify-func-1807-ubuntu1604-virl: - 2 + # TODO: Replace with 1807 job: + csit-dmm-functional-master-ubuntu1604-virl: + - 95 csit-hc2vpp-verify-func-1807-ubuntu1604: # TODO: Use the build number - "lastSuccessfulBuild" @@ -2194,6 +2204,34 @@ - "doc" - "status" +################################################################################ +# DMM +- + type: "table" + title: "Detailed Test Results - DMM Functional Results" + algorithm: "table_details" + output-file-ext: ".csv" + output-file: "{DIR[DTR,FUNC,DMM]}/dmm_functional_results" + columns: + - + title: "Name" + data: "data name" + - + title: "Documentation" + data: "data doc" + - + title: "Status" + data: "data status" + rows: "generated" + data: + "dmm-func-results" + filter: "all" + parameters: + - "name" + - "parent" + - "doc" + - "status" + ################################################################################ ### F I L E S ### ################################################################################ @@ -2627,6 +2665,24 @@ - "parent" data-start-level: 3 +- + type: "file" + title: "DMM Functional Results" + algorithm: "file_test_results" + output-file-ext: ".rst" + output-file: "{DIR[DTR,FUNC,DMM]}/dmm_functional_results" + file-header: "\n.. |br| raw:: html\n\n
\n\n\n.. |prein| raw:: html\n\n 
\n\n\n.. |preout| raw:: html\n\n
\n\n" + dir-tables: "{DIR[DTR,FUNC,DMM]}" + data: + "dmm-func-results" + filter: "all" + parameters: + - "name" + - "doc" + - "level" + - "parent" + data-start-level: 3 + ################################################################################ ### P L O T S ### ################################################################################ -- 2.16.6