From 3cb49d761fa32c84ea7b6b9ba9d6c874dff2921a Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Fri, 16 Feb 2018 13:13:51 +0100 Subject: [PATCH] CSIT-932: Remove outliers form statistical calculations Change-Id: Ibd90931bdc3dd8615fe2c975935c8a54fe85e6c3 Signed-off-by: Tibor Frank --- .../vpp_performance_tests/csit_release_notes.rst | 160 +++------------------ resources/tools/presentation/generator_tables.py | 12 +- resources/tools/presentation/specification.yaml | 9 +- 3 files changed, 32 insertions(+), 149 deletions(-) diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst index 6731de92d9..754abc0d13 100644 --- a/docs/report/vpp_performance_tests/csit_release_notes.rst +++ b/docs/report/vpp_performance_tests/csit_release_notes.rst @@ -60,155 +60,31 @@ comparison is calculated between the mean values based on collected and archived test results' samples for involved VPP releases. Standard deviation has been also listed for CSIT |release|. -NDR Throughput: Best 20 Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +NDR Throughput Changes +~~~~~~~~~~~~~~~~~~~~~~ -.. only:: html +NDR small packet throughput changes between releases are available in a CSV and +pretty ASCII formats: - .. csv-table:: - :align: center - :file: ../../../_build/_static/vpp/performance-changes-ndr-1t1c-top.csv + - `csv format for 1t1c <../_static/vpp/performance-changes-ndr-1t1c-full.csv>`_, + - `csv format for 2t2c <../_static/vpp/performance-changes-ndr-2t2c-full.csv>`_, + - `pretty ASCII format for 1t1c <../_static/vpp/performance-changes-ndr-1t1c-full.txt>`_, + - `pretty ASCII format for 2t2c <../_static/vpp/performance-changes-ndr-2t2c-full.txt>`_. -.. only:: latex +PDR Throughput Changes +~~~~~~~~~~~~~~~~~~~~~~ - .. raw:: latex +NDR small packet throughput changes between releases are available in a CSV and +pretty ASCII formats: - \makeatletter - \csvset{ - perfimprovements column width/.style={after head=\csv@pretable\begin{longtable}{m{7cm} m{#1} m{#1} m{#1} m{#1} m{#1}}\csv@tablehead}, - } - \makeatother - - {\tiny - \csvautobooklongtable[separator=comma, - respect all, - no check column count, - perfimprovements column width=1cm, - late after line={\\\hline}, - late after last line={\end{longtable}} - ]{../_build/_static/vpp/performance-changes-ndr-1t1c-top.csv} - } - -NDR Throughput: Worst 20 Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. only:: html - - .. csv-table:: - :align: center - :file: ../../../_build/_static/vpp/performance-changes-ndr-1t1c-bottom.csv - -.. only:: latex - - .. raw:: latex - - \makeatletter - \csvset{ - perfimprovements column width/.style={after head=\csv@pretable\begin{longtable}{m{7cm} m{#1} m{#1} m{#1} m{#1} m{#1}}\csv@tablehead}, - } - \makeatother - - {\tiny - \csvautobooklongtable[separator=comma, - respect all, - no check column count, - perfimprovements column width=1cm, - late after line={\\\hline}, - late after last line={\end{longtable}} - ]{../_build/_static/vpp/performance-changes-ndr-1t1c-bottom.csv} - } - -.. only:: html - - NDR Throughput: All Changes - ~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Complete results for all NDR tests are available in a CSV and pretty - ASCII formats: - - - `csv format for 1t1c <../_static/vpp/performance-changes-ndr-1t1c-full.csv>`_, - - `csv format for 2t2c <../_static/vpp/performance-changes-ndr-2t2c-full.csv>`_, - - `csv format for 4t4c <../_static/vpp/performance-changes-ndr-4t4c-full.csv>`_, - - `pretty ASCII format for 1t1c <../_static/vpp/performance-changes-ndr-1t1c-full.txt>`_, - - `pretty ASCII format for 2t2c <../_static/vpp/performance-changes-ndr-2t2c-full.txt>`_, - - `pretty ASCII format for 4t4c <../_static/vpp/performance-changes-ndr-4t4c-full.txt>`_. - -PDR Throughput: Best 20 Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. only:: html - - .. csv-table:: - :align: center - :file: ../../../_build/_static/vpp/performance-changes-pdr-1t1c-top.csv - -.. only:: latex - - .. raw:: latex - - \makeatletter - \csvset{ - perfimprovements column width/.style={after head=\csv@pretable\begin{longtable}{m{7cm} m{#1} m{#1} m{#1} m{#1} m{#1}}\csv@tablehead}, - } - \makeatother - - {\tiny - \csvautobooklongtable[separator=comma, - respect all, - no check column count, - perfimprovements column width=1cm, - late after line={\\\hline}, - late after last line={\end{longtable}} - ]{../_build/_static/vpp/performance-changes-pdr-1t1c-top.csv} - } - -PDR Throughput: Worst 20 Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. only:: html - - .. csv-table:: - :align: center - :file: ../../../_build/_static/vpp/performance-changes-pdr-1t1c-bottom.csv - -.. only:: latex - - .. raw:: latex - - \makeatletter - \csvset{ - perfimprovements column width/.style={after head=\csv@pretable\begin{longtable}{m{7cm} m{#1} m{#1} m{#1} m{#1} m{#1}}\csv@tablehead}, - } - \makeatother - - {\tiny - \csvautobooklongtable[separator=comma, - respect all, - no check column count, - perfimprovements column width=1cm, - late after line={\\\hline}, - late after last line={\end{longtable}} - ]{../_build/_static/vpp/performance-changes-pdr-1t1c-bottom.csv} - } - -.. only:: html - - PDR Throughput: All Changes - ~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Complete results for all PDR tests are available in a CSV and pretty - ASCII formats: - - - `csv format for 1t1c <../_static/vpp/performance-changes-pdr-1t1c-full.csv>`_, - - `csv format for 2t2c <../_static/vpp/performance-changes-pdr-2t2c-full.csv>`_, - - `csv format for 4t4c <../_static/vpp/performance-changes-pdr-4t4c-full.csv>`_, - - `pretty ASCII format for 1t1c <../_static/vpp/performance-changes-pdr-1t1c-full.txt>`_, - - `pretty ASCII format for 2t2c <../_static/vpp/performance-changes-pdr-2t2c-full.txt>`_, - - `pretty ASCII format for 4t4c <../_static/vpp/performance-changes-pdr-4t4c-full.txt>`_. + - `csv format for 1t1c <../_static/vpp/performance-changes-pdr-1t1c-full.csv>`_, + - `csv format for 2t2c <../_static/vpp/performance-changes-pdr-2t2c-full.csv>`_, + - `pretty ASCII format for 1t1c <../_static/vpp/performance-changes-pdr-1t1c-full.txt>`_, + - `pretty ASCII format for 2t2c <../_static/vpp/performance-changes-pdr-2t2c-full.txt>`_. Measured improvements are in line with VPP code optimizations listed in -`VPP-17.10 release notes -`_. +`VPP-18.01 release notes +`_. Known Issues ------------ diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index f55c41ecf7..76254c86dd 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -402,16 +402,20 @@ def table_performance_comparison(table, input_data): item = [tbl_dict[tst_name]["name"], ] if tbl_dict[tst_name]["ref-data"]: item.append(round(mean(remove_outliers( - tbl_dict[tst_name]["ref-data"], 2)) / 1000000, 2)) + tbl_dict[tst_name]["ref-data"], + table["outlier-const"])) / 1000000, 2)) item.append(round(stdev(remove_outliers( - tbl_dict[tst_name]["ref-data"], 2)) / 1000000, 2)) + tbl_dict[tst_name]["ref-data"], + table["outlier-const"])) / 1000000, 2)) else: item.extend([None, None]) if tbl_dict[tst_name]["cmp-data"]: item.append(round(mean(remove_outliers( - tbl_dict[tst_name]["cmp-data"], 2)) / 1000000, 2)) + tbl_dict[tst_name]["cmp-data"], + table["outlier-const"])) / 1000000, 2)) item.append(round(stdev(remove_outliers( - tbl_dict[tst_name]["cmp-data"], 2)) / 1000000, 2)) + tbl_dict[tst_name]["cmp-data"], + table["outlier-const"])) / 1000000, 2)) else: item.extend([None, None]) if item[1] is not None and item[3] is not None: diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 447d558ae8..11c7e0a6c2 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -669,9 +669,11 @@ # TODO: specify data sources data: csit-vpp-perf-1801-all: - - 124 - - 127 - - 128 + - 124 # sel + - 127 # sel + - 128 # sel + - 141 # sel + - 142 # sel data: "vpp-performance-changes" filter: "all" parameters: @@ -681,6 +683,7 @@ # Number of the best and the worst tests presented in the table. Use 0 (zero) # to present all tests. nr-of-tests-shown: 20 + outlier-const: 1.5 #- # type: "table" -- 2.16.6