CSIT-932: Remove outliers form statistical calculations

Change-Id: Ibd90931bdc3dd8615fe2c975935c8a54fe85e6c3
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst
index 6731de9..754abc0 100644 (file)
--- a/docs/report/vpp_performance_tests/csit_release_notes.rst
+++ b/docs/report/vpp_performance_tests/csit_release_notes.rst
@@ -60,155 +60,31 @@ comparison is calculated between the mean values based on collected and
 archived test results' samples for involved VPP releases. Standard deviation\r
 has been also listed for CSIT |release|.\r
 \r
-NDR Throughput: Best 20 Changes\r
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\r
+NDR Throughput Changes\r
+~~~~~~~~~~~~~~~~~~~~~~\r
 \r
-.. only:: html\r
+NDR small packet throughput changes between releases are available in a CSV and\r
+pretty ASCII formats:\r
 \r
-   .. csv-table::\r
-      :align: center\r
-      :file: ../../../_build/_static/vpp/performance-changes-ndr-1t1c-top.csv\r
+  - `csv format for 1t1c <../_static/vpp/performance-changes-ndr-1t1c-full.csv>`_,\r
+  - `csv format for 2t2c <../_static/vpp/performance-changes-ndr-2t2c-full.csv>`_,\r
+  - `pretty ASCII format for 1t1c <../_static/vpp/performance-changes-ndr-1t1c-full.txt>`_,\r
+  - `pretty ASCII format for 2t2c <../_static/vpp/performance-changes-ndr-2t2c-full.txt>`_.\r
 \r
-.. only:: latex\r
+PDR Throughput Changes\r
+~~~~~~~~~~~~~~~~~~~~~~\r
 \r
-   .. raw:: latex\r
+NDR small packet throughput changes between releases are available in a CSV and\r
+pretty ASCII formats:\r
 \r
-      \makeatletter\r
-      \csvset{\r
-        perfimprovements column width/.style={after head=\csv@pretable\begin{longtable}{m{7cm} m{#1} m{#1} m{#1} m{#1} m{#1}}\csv@tablehead},\r
-      }\r
-      \makeatother\r
-\r
-      {\tiny\r
-      \csvautobooklongtable[separator=comma,\r
-        respect all,\r
-        no check column count,\r
-        perfimprovements column width=1cm,\r
-        late after line={\\\hline},\r
-        late after last line={\end{longtable}}\r
-        ]{../_build/_static/vpp/performance-changes-ndr-1t1c-top.csv}\r
-      }\r
-\r
-NDR Throughput: Worst 20 Changes\r
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\r
-\r
-.. only:: html\r
-\r
-   .. csv-table::\r
-      :align: center\r
-      :file: ../../../_build/_static/vpp/performance-changes-ndr-1t1c-bottom.csv\r
-\r
-.. only:: latex\r
-\r
-   .. raw:: latex\r
-\r
-      \makeatletter\r
-      \csvset{\r
-        perfimprovements column width/.style={after head=\csv@pretable\begin{longtable}{m{7cm} m{#1} m{#1} m{#1} m{#1} m{#1}}\csv@tablehead},\r
-      }\r
-      \makeatother\r
-\r
-      {\tiny\r
-      \csvautobooklongtable[separator=comma,\r
-        respect all,\r
-        no check column count,\r
-        perfimprovements column width=1cm,\r
-        late after line={\\\hline},\r
-        late after last line={\end{longtable}}\r
-        ]{../_build/_static/vpp/performance-changes-ndr-1t1c-bottom.csv}\r
-      }\r
-\r
-.. only:: html\r
-\r
-      NDR Throughput: All Changes\r
-      ~~~~~~~~~~~~~~~~~~~~~~~~~~~\r
-\r
-      Complete results for all NDR tests are available in a CSV and pretty\r
-      ASCII formats:\r
-\r
-        - `csv format for 1t1c <../_static/vpp/performance-changes-ndr-1t1c-full.csv>`_,\r
-        - `csv format for 2t2c <../_static/vpp/performance-changes-ndr-2t2c-full.csv>`_,\r
-        - `csv format for 4t4c <../_static/vpp/performance-changes-ndr-4t4c-full.csv>`_,\r
-        - `pretty ASCII format for 1t1c <../_static/vpp/performance-changes-ndr-1t1c-full.txt>`_,\r
-        - `pretty ASCII format for 2t2c <../_static/vpp/performance-changes-ndr-2t2c-full.txt>`_,\r
-        - `pretty ASCII format for 4t4c <../_static/vpp/performance-changes-ndr-4t4c-full.txt>`_.\r
-\r
-PDR Throughput: Best 20 Changes\r
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\r
-\r
-.. only:: html\r
-\r
-   .. csv-table::\r
-      :align: center\r
-      :file: ../../../_build/_static/vpp/performance-changes-pdr-1t1c-top.csv\r
-\r
-.. only:: latex\r
-\r
-   .. raw:: latex\r
-\r
-      \makeatletter\r
-      \csvset{\r
-        perfimprovements column width/.style={after head=\csv@pretable\begin{longtable}{m{7cm} m{#1} m{#1} m{#1} m{#1} m{#1}}\csv@tablehead},\r
-      }\r
-      \makeatother\r
-\r
-      {\tiny\r
-      \csvautobooklongtable[separator=comma,\r
-        respect all,\r
-        no check column count,\r
-        perfimprovements column width=1cm,\r
-        late after line={\\\hline},\r
-        late after last line={\end{longtable}}\r
-        ]{../_build/_static/vpp/performance-changes-pdr-1t1c-top.csv}\r
-      }\r
-\r
-PDR Throughput: Worst 20 Changes\r
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\r
-\r
-.. only:: html\r
-\r
-   .. csv-table::\r
-      :align: center\r
-      :file: ../../../_build/_static/vpp/performance-changes-pdr-1t1c-bottom.csv\r
-\r
-.. only:: latex\r
-\r
-   .. raw:: latex\r
-\r
-      \makeatletter\r
-      \csvset{\r
-        perfimprovements column width/.style={after head=\csv@pretable\begin{longtable}{m{7cm} m{#1} m{#1} m{#1} m{#1} m{#1}}\csv@tablehead},\r
-      }\r
-      \makeatother\r
-\r
-      {\tiny\r
-      \csvautobooklongtable[separator=comma,\r
-        respect all,\r
-        no check column count,\r
-        perfimprovements column width=1cm,\r
-        late after line={\\\hline},\r
-        late after last line={\end{longtable}}\r
-        ]{../_build/_static/vpp/performance-changes-pdr-1t1c-bottom.csv}\r
-      }\r
-\r
-.. only:: html\r
-\r
-      PDR Throughput: All Changes\r
-      ~~~~~~~~~~~~~~~~~~~~~~~~~~~\r
-\r
-            Complete results for all PDR tests are available in a CSV and pretty\r
-            ASCII formats:\r
-\r
-              - `csv format for 1t1c <../_static/vpp/performance-changes-pdr-1t1c-full.csv>`_,\r
-              - `csv format for 2t2c <../_static/vpp/performance-changes-pdr-2t2c-full.csv>`_,\r
-              - `csv format for 4t4c <../_static/vpp/performance-changes-pdr-4t4c-full.csv>`_,\r
-              - `pretty ASCII format for 1t1c <../_static/vpp/performance-changes-pdr-1t1c-full.txt>`_,\r
-              - `pretty ASCII format for 2t2c <../_static/vpp/performance-changes-pdr-2t2c-full.txt>`_,\r
-              - `pretty ASCII format for 4t4c <../_static/vpp/performance-changes-pdr-4t4c-full.txt>`_.\r
+  - `csv format for 1t1c <../_static/vpp/performance-changes-pdr-1t1c-full.csv>`_,\r
+  - `csv format for 2t2c <../_static/vpp/performance-changes-pdr-2t2c-full.csv>`_,\r
+  - `pretty ASCII format for 1t1c <../_static/vpp/performance-changes-pdr-1t1c-full.txt>`_,\r
+  - `pretty ASCII format for 2t2c <../_static/vpp/performance-changes-pdr-2t2c-full.txt>`_.\r
 \r
 Measured improvements are in line with VPP code optimizations listed in\r
-`VPP-17.10 release notes\r
-<https://docs.fd.io/vpp/17.10/release_notes_1710.html>`_.\r
+`VPP-18.01 release notes\r
+<https://docs.fd.io/vpp/18.01/release_notes_1801.html>`_.\r
 \r
 Known Issues\r
 ------------\r

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index f55c41e..76254c8 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -402,16 +402,20 @@ def table_performance_comparison(table, input_data):
         item = [tbl_dict[tst_name]["name"], ]
         if tbl_dict[tst_name]["ref-data"]:
             item.append(round(mean(remove_outliers(
-                tbl_dict[tst_name]["ref-data"], 2)) / 1000000, 2))
+                tbl_dict[tst_name]["ref-data"],
+                table["outlier-const"])) / 1000000, 2))
             item.append(round(stdev(remove_outliers(
-                tbl_dict[tst_name]["ref-data"], 2)) / 1000000, 2))
+                tbl_dict[tst_name]["ref-data"],
+                table["outlier-const"])) / 1000000, 2))
         else:
             item.extend([None, None])
         if tbl_dict[tst_name]["cmp-data"]:
             item.append(round(mean(remove_outliers(
-                tbl_dict[tst_name]["cmp-data"], 2)) / 1000000, 2))
+                tbl_dict[tst_name]["cmp-data"],
+                table["outlier-const"])) / 1000000, 2))
             item.append(round(stdev(remove_outliers(
-                tbl_dict[tst_name]["cmp-data"], 2)) / 1000000, 2))
+                tbl_dict[tst_name]["cmp-data"],
+                table["outlier-const"])) / 1000000, 2))
         else:
             item.extend([None, None])
         if item[1] is not None and item[3] is not None:

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 447d558..11c7e0a 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -669,9 +669,11 @@
 # TODO: specify data sources
     data:
       csit-vpp-perf-1801-all:
-      - 124
-      - 127
-      - 128
+      - 124  # sel
+      - 127  # sel
+      - 128  # sel
+      - 141  # sel
+      - 142  # sel
   data: "vpp-performance-changes"
   filter: "all"
   parameters:
@@ -681,6 +683,7 @@
   # Number of the best and the worst tests presented in the table. Use 0 (zero)
   # to present all tests.
   nr-of-tests-shown: 20
+  outlier-const: 1.5
 
 #-
 #  type: "table"