========
This is the **F**\ast **D**\ata **I**/**O** Project (**FD.io**) **C**\ontinuous
-**S**\ystem **I**\ntegration and **T**\esting (**CSIT**) project report for CSIT
-|release| system testing of |vpp-release|.
+**S**\ystem **I**\ntegration and **T**\esting (**CSIT**) project report for
+|csit-release| system testing of |vpp-release|.
This is the full html version, there is also a reduced
`PDF version of this report`_.
The report describes CSIT functional and performance tests and their
-continuous execution delivered in CSIT |release|. A high-level overview is
+continuous execution delivered in |csit-release|. A high-level overview is
provided for each CSIT test environment running in :abbr:`LF (Linux Foundation)`
FD.io Continuous Performance Labs. This is followed by summary of all executed
tests against the |vpp-release| and associated FD.io projects and sub-systems
-(Honeycomb, DPDK, NSH_SFC), CSIT |release| release notes, result highlights and
+(Honeycomb, DPDK, NSH_SFC), |csit-release| release notes, result highlights and
known issues discovered in CSIT. More detailed description of each environment,
pointers to CSIT test code documentation and detailed test resuls with links to
the source data files are also provided.
-CSIT |release| report contains following main sections and sub-sections:
+|csit-release| report contains following main sections and sub-sections:
#. **Introduction** - general introduction to CSIT project; *Overview* -
this section; *CSIT Test Naming* - CSIT general naming convention for test
#. **VPP Performance Tests** - VPP performance tests executed in physical
FD.io testbeds; *Overview* - tested topologies, test coverage and naming
specifics, methodology for multi-core, packet throughput and latency, and
- KVM VM vhost tests; *CSIT Release Notes* - changes in CSIT |release|, added
+ KVM VM vhost tests; *CSIT Release Notes* - changes in |csit-release|, added
tests, performance changes, environment or methodology changes, known CSIT
issues; *Packet Throughput Graphs* and *Packet Latency
Graphs* - plotted NDR, PDR throughput and latency results from multiple
#. **DPDK Performance Tests** - DPDK performance tests executed in
physical FD.io testbeds; *Overview* - tested topologies, test coverage;
- *CSIT Release Notes* - changes in CSIT |release|, any known CSIT issues;
+ *CSIT Release Notes* - changes in |csit-release|, any known CSIT issues;
*Packet Throughput Graphs* and *Packet Latency Graphs*
- plotted NDR, PDR throughput and latency results from multiple test job
executions; *Test Environment* - environment description; *Documentation* -
#. **VPP Functional Tests** - VPP functional tests executed in virtual
FD.io testbeds; *Overview* - tested virtual topologies, test coverage and
- naming specifics; *CSIT Release Notes* - changes in CSIT |release|, added
+ naming specifics; *CSIT Release Notes* - changes in |csit-release|, added
tests, environment or methodology changes, known CSIT issues, tests to be
added; *Test Environment* - environment description ; *Documentation* -
source code documentation for VPP functional tests.
#. **CSIT Framework Documentation** - description of the overall CSIT
framework design hierarchy, CSIT test naming convention, followed by
description of Presentation and Analytics Layer (PAL) introduced in
- CSIT |release|.
\ No newline at end of file
+ |csit-release|.
\ No newline at end of file
DIR[DTR,MRR,VPP,3N,SKX]: "{DIR[DTR]}/vpp_mrr_results_3n_skx"
DIR[DTR,MRR,VPP,2N,SKX]: "{DIR[DTR]}/vpp_mrr_results_2n_skx"
DIR[DTR,PERF,COT,3N,HSW]: "{DIR[DTR]}/cot_performance_results_3n_hsw"
- DIR[DTR,PERF,COT,3N,SKX]: "{DIR[DTR]}/cot_performance_results_3n_skx"
- DIR[DTR,PERF,COT,2N,SKX]: "{DIR[DTR]}/cot_performance_results_2n_skx"
DIR[DTR,FUNC,VPP,UBUNTU]: "{DIR[DTR]}/vpp_functional_results_ubuntu"
DIR[DTR,FUNC,VPP,CENTOS]: "{DIR[DTR]}/vpp_functional_results_centos"
DIR[DTR,FUNC,HC]: "{DIR[DTR]}/honeycomb_functional_results"
# Detailed tests operational data
DIR[DTO]: "{DIR[WORKING,SRC]}/test_operational_data"
- DIR[DTO,PERF,VPP]: "{DIR[DTO]}/vpp_performance_operational_data"
+ DIR[DTO,PERF,VPP,3N,HSW]: "{DIR[DTO]}/vpp_performance_operational_data_3n_hsw"
+ DIR[DTO,PERF,VPP,3N,SKX]: "{DIR[DTO]}/vpp_performance_operational_data_3n_skx"
+ DIR[DTO,PERF,VPP,2N,SKX]: "{DIR[DTO]}/vpp_performance_operational_data_2n_skx"
# .css patch file to fix tables generated by Sphinx
DIR[CSS_PATCH_FILE]: "{DIR[STATIC]}/theme_overrides.css"
-
type: "table"
- title: "Test Operational Data - VPP Performance Operational Data"
+ title: "Test Operational Data - VPP Performance Operational Data 3n-hsw"
algorithm: "table_merged_details"
output-file-ext: ".csv"
- output-file: "{DIR[DTO,PERF,VPP]}/vpp_test_operational"
+ output-file: "{DIR[DTO,PERF,VPP,3N,HSW]}/vpp_test_operational_3n_hsw"
columns:
-
title: "Name"
title: "VPP Operational Data - Outputs of 'show runtime' at NDR packet rate"
data: "data show-run"
rows: "generated"
- data:
- "vpp-perf-results-3n-hsw"
+ data: "vpp-perf-results-3n-hsw"
+ filter: "not 'NDRCHK' and not 'PDRCHK'"
+ parameters:
+ - "parent"
+ - "name"
+ - "show-run"
+
+-
+ type: "table"
+ title: "Test Operational Data - VPP Performance Operational Data 3n-skx"
+ algorithm: "table_merged_details"
+ output-file-ext: ".csv"
+ output-file: "{DIR[DTO,PERF,VPP,3N,SKX]}/vpp_test_operational_3n_skx"
+ columns:
+ -
+ title: "Name"
+ data: "data name"
+ -
+ title: "VPP Operational Data - Outputs of 'show runtime' at NDR packet rate"
+ data: "data show-run"
+ rows: "generated"
+ data: "vpp-perf-results-3n-skx"
+ filter: "not 'NDRCHK' and not 'PDRCHK'"
+ parameters:
+ - "parent"
+ - "name"
+ - "show-run"
+
+-
+ type: "table"
+ title: "Test Operational Data - VPP Performance Operational Data 2n-skx"
+ algorithm: "table_merged_details"
+ output-file-ext: ".csv"
+ output-file: "{DIR[DTO,PERF,VPP,2N,SKX]}/vpp_test_operational_2n_skx"
+ columns:
+ -
+ title: "Name"
+ data: "data name"
+ -
+ title: "VPP Operational Data - Outputs of 'show runtime' at NDR packet rate"
+ data: "data show-run"
+ rows: "generated"
+ data: "vpp-perf-results-2n-skx"
filter: "not 'NDRCHK' and not 'PDRCHK'"
parameters:
- "parent"
- "doc"
- "msg"
--
- type: "table"
- title: "Detailed Test Results - Container Orchestrated Topologies Performance Results 3n-skx"
- algorithm: "table_details"
- output-file-ext: ".csv"
- output-file: "{DIR[DTR,PERF,COT,3N,SKX]}/cot_performance_results_3n_skx"
- columns:
- -
- title: "Name"
- data: "data name"
- -
- title: "Documentation"
- data: "data doc"
- -
- title: "Status"
- data: "data msg"
- rows: "generated"
- data: "ligato-perf-results-3n-skx"
- filter: "all"
- parameters:
- - "name"
- - "parent"
- - "doc"
- - "msg"
-
--
- type: "table"
- title: "Detailed Test Results - Container Orchestrated Topologies Performance Results 2n-skx"
- algorithm: "table_details"
- output-file-ext: ".csv"
- output-file: "{DIR[DTR,PERF,COT,2N,SKX]}/cot_performance_results_2n_skx"
- columns:
- -
- title: "Name"
- data: "data name"
- -
- title: "Documentation"
- data: "data doc"
- -
- title: "Status"
- data: "data msg"
- rows: "generated"
- data: "ligato-perf-results-2n-skx"
- filter: "all"
- parameters:
- - "name"
- - "parent"
- - "doc"
- - "msg"
-
-
type: "table"
title: "Detailed Test Results - DPDK Performance Results 3n-hsw"
-
type: "file"
- title: "VPP Performance Configuration"
+ title: "VPP Performance Configuration 3n-hsw"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTC,PERF,VPP]}/vpp_performance_configuration"
+ output-file: "{DIR[DTC,PERF,VPP,3N,HSW]}/vpp_performance_configuration_3n_hsw"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTC,PERF,VPP]}"
- data:
- "vpp-perf-results-3n-hsw"
+ dir-tables: "{DIR[DTC,PERF,VPP,3N,HSW]}"
+ data: "vpp-perf-results-3n-hsw"
filter: "not 'NDRCHK' and not 'PDRCHK'"
parameters:
- "name"
-
type: "file"
- title: "VPP Performance Operational Data"
+ title: "VPP Performance Configuration 3n-skx"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTO,PERF,VPP]}/vpp_performance_operational_data"
+ output-file: "{DIR[DTC,PERF,VPP,3N,SKX]}/vpp_performance_configuration_3n_skx"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTO,PERF,VPP]}"
- data:
- "vpp-perf-results-3n-hsw"
+ dir-tables: "{DIR[DTC,PERF,VPP,3N,SKX]}"
+ data: "vpp-perf-results-3n-skx"
+ filter: "not 'NDRCHK' and not 'PDRCHK'"
+ parameters:
+ - "name"
+ - "doc"
+ - "level"
+ - "parent"
+ data-start-level: 3
+
+-
+ type: "file"
+ title: "VPP Performance Configuration 2n-skx"
+ algorithm: "file_test_results"
+ output-file-ext: ".rst"
+ output-file: "{DIR[DTC,PERF,VPP,2N,SKX]}/vpp_performance_configuration_2n_skx"
+ file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
+ dir-tables: "{DIR[DTC,PERF,VPP,2N,SKX]}"
+ data: "vpp-perf-results-2n-skx"
+ filter: "not 'NDRCHK' and not 'PDRCHK'"
+ parameters:
+ - "name"
+ - "doc"
+ - "level"
+ - "parent"
+ data-start-level: 3
+
+-
+ type: "file"
+ title: "VPP Performance Operational Data 3n-hsw"
+ algorithm: "file_test_results"
+ output-file-ext: ".rst"
+ output-file: "{DIR[DTO,PERF,VPP,3N,HSW]}/vpp_performance_operational_data_3n_hsw"
+ file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
+ dir-tables: "{DIR[DTO,PERF,VPP,3N,HSW]}"
+ data: "vpp-perf-results-3n-hsw"
+ filter: "not 'NDRCHK' and not 'PDRCHK'"
+ parameters:
+ - "name"
+ - "doc"
+ - "level"
+ - "parent"
+ data-start-level: 3
+
+-
+ type: "file"
+ title: "VPP Performance Operational Data 3n-skx"
+ algorithm: "file_test_results"
+ output-file-ext: ".rst"
+ output-file: "{DIR[DTO,PERF,VPP,3N,SKX]}/vpp_performance_operational_data_3n_skx"
+ file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
+ dir-tables: "{DIR[DTO,PERF,VPP,3N,SKX]}"
+ data: "vpp-perf-results-3n-skx"
+ filter: "not 'NDRCHK' and not 'PDRCHK'"
+ parameters:
+ - "name"
+ - "doc"
+ - "level"
+ - "parent"
+ data-start-level: 3
+
+-
+ type: "file"
+ title: "VPP Performance Operational Data 2n-skx"
+ algorithm: "file_test_results"
+ output-file-ext: ".rst"
+ output-file: "{DIR[DTO,PERF,VPP,2N,SKX]}/vpp_performance_operational_data_2n_skx"
+ file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
+ dir-tables: "{DIR[DTO,PERF,VPP,2N,SKX]}"
+ data: "vpp-perf-results-2n-skx"
filter: "not 'NDRCHK' and not 'PDRCHK'"
parameters:
- "name"
-
type: "file"
- title: "VPP MRR Configuration"
+ title: "VPP MRR Configuration 3n-hsw"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTC,MRR,VPP]}/vpp_mrr_configuration"
+ output-file: "{DIR[DTC,MRR,VPP,3N,HSW]}/vpp_mrr_configuration_3n_hsw"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTC,MRR,VPP]}"
- data:
- "vpp-mrr-results-3n-hsw"
+ dir-tables: "{DIR[DTC,MRR,VPP,3N,HSW]}"
+ data: "vpp-mrr-results-3n-hsw"
filter: "'MRR'"
parameters:
- "name"
-
type: "file"
- title: "VPP Functional Results - Ubuntu"
+ title: "VPP MRR Configuration 3n-skx"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTR,FUNC,VPP,UBUNTU]}/vpp_functional_results_ubuntu"
+ output-file: "{DIR[DTC,MRR,VPP,3N,SKX]}/vpp_mrr_configuration_3n_skx"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTR,FUNC,VPP,UBUNTU]}"
- data: "vpp-func-results-ubuntu"
- filter: "all"
+ dir-tables: "{DIR[DTC,MRR,VPP,3N,SKX]}"
+ data: "vpp-mrr-results-3n-skx"
+ filter: "'MRR'"
parameters:
- "name"
- "doc"
-
type: "file"
- title: "VPP Functional Results - CentOS"
+ title: "VPP MRR Configuration 2n-skx"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTR,FUNC,VPP,CENTOS]}/vpp_functional_results_centos"
+ output-file: "{DIR[DTC,MRR,VPP,2N,SKX]}/vpp_mrr_configuration_2n_skx"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTR,FUNC,VPP,CENTOS]}"
- data:
- "vpp-func-results-centos"
- filter: "all"
+ dir-tables: "{DIR[DTC,MRR,VPP,2N,SKX]}"
+ data: "vpp-mrr-results-2n-skx"
+ filter: "'MRR'"
parameters:
- "name"
- "doc"
-
type: "file"
- title: "VPP Functional Configuration - Ubuntu"
+ title: "VPP Functional Results - Ubuntu"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTC,FUNC,VPP]}/vpp_functional_configuration"
+ output-file: "{DIR[DTR,FUNC,VPP,UBUNTU]}/vpp_functional_results_ubuntu"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTC,FUNC,VPP]}"
- data:
- "vpp-func-results-ubuntu"
+ dir-tables: "{DIR[DTR,FUNC,VPP,UBUNTU]}"
+ data: "vpp-func-results-ubuntu"
filter: "all"
parameters:
- "name"
- "doc"
- "level"
- "parent"
- data-start-level: 3
+ data-start-level: 2
-
type: "file"
- title: "VPP Functional Configuration - CentOS"
+ title: "VPP Functional Results - CentOS"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTC,FUNC,VPP,CENTOS]}/vpp_functional_configuration_centos"
+ output-file: "{DIR[DTR,FUNC,VPP,CENTOS]}/vpp_functional_results_centos"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTC,FUNC,VPP,CENTOS]}"
+ dir-tables: "{DIR[DTR,FUNC,VPP,CENTOS]}"
data:
"vpp-func-results-centos"
filter: "all"
- "doc"
- "level"
- "parent"
- data-start-level: 3
+ data-start-level: 2
-
type: "file"
- title: "Container Orchestrated Performance Results 3n-hsw"
+ title: "VPP Functional Configuration - Ubuntu"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTR,PERF,COT,3N,HSW]}/cot_performance_results_3n_hsw"
+ output-file: "{DIR[DTC,FUNC,VPP,UBUNTU]}/vpp_functional_configuration_ubuntu"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTR,PERF,COT,3N,HSW]}"
- data: "ligato-perf-results-3n-hsw"
+ dir-tables: "{DIR[DTC,FUNC,VPP,UBUNTU]}"
+ data: "vpp-func-results-ubuntu"
filter: "all"
parameters:
- "name"
- "doc"
- "level"
- "parent"
- data-start-level: 3
+ data-start-level: 2
-
type: "file"
- title: "Container Orchestrated Performance Results 3n-skx"
+ title: "VPP Functional Configuration - CentOS"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTR,PERF,COT,3N,SKX]}/cot_performance_results_3n_skx"
+ output-file: "{DIR[DTC,FUNC,VPP,CENTOS]}/vpp_functional_configuration_centos"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTR,PERF,COT,3N,SKX]}"
- data: "ligato-perf-results-3n-skx"
+ dir-tables: "{DIR[DTC,FUNC,VPP,CENTOS]}"
+ data:
+ "vpp-func-results-centos"
filter: "all"
parameters:
- "name"
- "doc"
- "level"
- "parent"
- data-start-level: 3
+ data-start-level: 2
-
type: "file"
- title: "Container Orchestrated Performance Results 2n-skx"
+ title: "Container Orchestrated Performance Results 3n-hsw"
algorithm: "file_test_results"
output-file-ext: ".rst"
- output-file: "{DIR[DTR,PERF,COT,2N,SKX]}/cot_performance_results_2n_skx"
+ output-file: "{DIR[DTR,PERF,COT,3N,HSW]}/cot_performance_results_3n_hsw"
file-header: "\n.. |br| raw:: html\n\n <br />\n\n\n.. |prein| raw:: html\n\n <pre>\n\n\n.. |preout| raw:: html\n\n </pre>\n\n"
- dir-tables: "{DIR[DTR,PERF,COT,2N,SKX]}"
- data: "ligato-perf-results-2n-skx"
+ dir-tables: "{DIR[DTR,PERF,COT,3N,HSW]}"
+ data: "ligato-perf-results-3n-hsw"
filter: "all"
parameters:
- "name"
- "doc"
- "level"
- "parent"
- data-start-level: 2
+ data-start-level: 3
-
type: "file"
Code Review / csit.git / commitdiff