Code Review / csit.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | review | tree
raw | inline | side by side
CSIT-1229: Add DMM to report
[csit.git] / resources / tools / presentation / specification.yaml
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 286ccd0..67c43bd 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -41,6 +41,7 @@
     DIR[DTR,FUNC,VPP,CENTOS]: "{DIR[DTR]}/vpp_functional_results_centos"
     DIR[DTR,FUNC,HC]: "{DIR[DTR]}/honeycomb_functional_results"
     DIR[DTR,FUNC,NSHSFC]: "{DIR[DTR]}/nshsfc_functional_results"
+    DIR[DTR,FUNC,DMM]: "{DIR[DTR]}/dmm_functional_results"
     DIR[DTR,PERF,VPP,IMPRV]: "{DIR[WORKING,SRC]}/vpp_performance_tests/performance_improvements"
 
     # Detailed test configurations
@@ -129,6 +130,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
 
     vpp-compare-testbeds:
       # TODO: Add Data sources
@@ -136,6 +139,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
       csit-vpp-perf-verify-1807-3n-skx:
       - 33  # NDRPDR sel
       - 35  # NDRPDR sel
@@ -152,6 +157,8 @@
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 9   # MRR sel
+      - 21  # MRR sel
+      - 31  # MRR sel
       csit-vpp-perf-verify-1807-3n-skx:
       - 28  # MRR sel
       - 32  # MRR sel
@@ -190,11 +197,14 @@
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 9   # MRR sel
+      - 21  # MRR sel
+      - 31  # MRR sel
 
     vpp-perf-results-3n-hsw:
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 11  # NDRPDR full 1
+      - 22  # NDRPDR full 2
 
     vpp-perf-results-3n-skx:
       csit-vpp-perf-verify-1807-3n-skx:
@@ -224,6 +234,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
 
     plot-throughput-speedup-analysis-3n-skx:
       csit-vpp-perf-verify-1807-3n-skx:
@@ -257,6 +269,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
 
     plot-vpp-throughput-latency-3n-skx:
       csit-vpp-perf-verify-1807-3n-skx:
@@ -285,20 +299,27 @@
       - 42  # NDRPDR sel
 
     plot-vpp-http-server-performance:
-      # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 24  # WRK
       - 25  # WRK
+      - 23  # WRK
+      - 26  # WRK
+      - 27  # WRK
+      - 28  # WRK
+      - 29  # WRK
+      - 30  # WRK
+      - 32  # WRK
+      - 35  # WRK
 
     # VPP Functional
     vpp-func-results-ubuntu:
       csit-vpp-functional-1807-ubuntu1604-virl:
-      # TODO: Add Data sources
+      # TODO: Use the build number
       - "lastSuccessfulBuild"
 
     vpp-func-results-centos:
       csit-vpp-functional-1807-centos7-virl:
-      # TODO: Add Data sources
+      # TODO: Use the build number
       - "lastSuccessfulBuild"
 
     # VPP Ligato
@@ -420,7 +441,7 @@
     # HoneyComb Functional
     hc-func-results:
       csit-hc2vpp-verify-func-1807-ubuntu1604:
-      # TODO: Add Data sources
+      # TODO: Use the build number
       - "lastSuccessfulBuild"
 
     # NSH SFC Functional
@@ -428,6 +449,12 @@
       csit-nsh_sfc-verify-func-1807-ubuntu1604-virl:
       - 2
 
+    # DMM Functional
+    dmm-func-results:
+      # TODO: Replace with 1807 job:
+      csit-dmm-functional-master-ubuntu1604-virl:
+      - 95
+
   plot-layouts:
 
     plot-cps:
@@ -527,6 +554,7 @@
       yaxis:
         gridcolor: "rgb(238, 238, 238)"
         hoverformat: ".4s"
+        tickformat: ".3s"
         linecolor: "rgb(238, 238, 238)"
         linewidth: 1
         showgrid: True
@@ -677,12 +705,25 @@
     csit-vpp-perf-verify-1807-3n-hsw:
     - 8   # MRR full
     - 9   # MRR sel
+    - 21  # MRR sel
+    - 31  # MRR sel
     - 11  # NDRPDR full 1
+    - 22  # NDRPDR full 2
     - 15  # NDRPDR sel
     - 16  # NDRPDR sel
     - 19  # NDRPDR sel
+    - 18  # NDRPDR sel
+    - 20  # NDRPDR sel
     - 24  # WRK
     - 25  # WRK
+    - 23  # WRK
+    - 26  # WRK
+    - 27  # WRK
+    - 28  # WRK
+    - 29  # WRK
+    - 30  # WRK
+    - 32  # WRK
+    - 35  # WRK
     csit-vpp-perf-verify-1807-3n-skx:
     - 31  # MRR full
     - 28  # MRR sel
@@ -814,15 +855,18 @@
     - 9
     - 10
     csit-vpp-functional-1807-ubuntu1604-virl:
-    # TODO: Add Data sources
+    # TODO: Use the build number
     - "lastSuccessfulBuild"
     csit-vpp-functional-1807-centos7-virl:
-    # TODO: Add Data sources
+    # TODO: Use the build number
     - "lastSuccessfulBuild"
     csit-nsh_sfc-verify-func-1807-ubuntu1604-virl:
     - 2
+    # TODO: Replace with 1807 job:
+    csit-dmm-functional-master-ubuntu1604-virl:
+    - 95
     csit-hc2vpp-verify-func-1807-ubuntu1604:
-    # TODO: Add Data sources
+    # TODO: Use the build number
     - "lastSuccessfulBuild"
 
 -
@@ -884,6 +928,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
   data: "vpp-performance-changes"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'1T1C' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -936,6 +982,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
   data: "vpp-performance-changes"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T2C' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -988,6 +1036,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
   data: "vpp-performance-changes"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'1T1C' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -1040,6 +1090,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
   data: "vpp-performance-changes"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T2C' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -1061,6 +1113,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
   compare:
     title: "3-Node Skx"
     data:
@@ -1096,6 +1150,8 @@
       - 15  # NDRPDR sel
       - 16  # NDRPDR sel
       - 19  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 20  # NDRPDR sel
   compare:
     title: "3-Node Skx"
     data:
@@ -1158,6 +1214,8 @@
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 9   # MRR sel
+      - 21  # MRR sel
+      - 31  # MRR sel
   data: "vpp-performance-changes-mrr"
   include-tests: "MRR" # "PDR" | "NDR" | "MRR"
   filter: "'1T1C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -1206,6 +1264,8 @@
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 9   # MRR sel
+      - 21  # MRR sel
+      - 31  # MRR sel
   data: "vpp-performance-changes-mrr"
   include-tests: "MRR" # "PDR" | "NDR" | "MRR"
   filter: "'2T2C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -1254,6 +1314,8 @@
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 9   # MRR sel
+      - 21  # MRR sel
+      - 31  # MRR sel
   data: "vpp-performance-changes-mrr"
   include-tests: "MRR" # "PDR" | "NDR" | "MRR"
   filter: "'4T4C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
@@ -1273,6 +1335,8 @@
       # TODO: Add Data sources
       csit-vpp-perf-verify-1807-3n-hsw:
       - 9   # MRR sel
+      - 21  # MRR sel
+      - 31  # MRR sel
   compare:
     title: "3-Node Skx"
     data:
@@ -1305,9 +1369,6 @@
   -
     title: "Name"
     data: "data name"
-  -
-    title: "Documentation"
-    data: "data doc"
   -
     title: "Status"
     data: "data msg"
@@ -1330,9 +1391,6 @@
   -
     title: "Name"
     data: "data name"
-  -
-    title: "Documentation"
-    data: "data doc"
   -
     title: "Status"
     data: "data msg"
@@ -1355,9 +1413,6 @@
   -
     title: "Name"
     data: "data name"
-  -
-    title: "Documentation"
-    data: "data doc"
   -
     title: "Status"
     data: "data msg"
@@ -1506,9 +1561,6 @@
   -
     title: "Name"
     data: "data name"
-  -
-    title: "Documentation"
-    data: "data doc"
   -
     title: "Status"
     data: "data msg"
@@ -1531,9 +1583,6 @@
   -
     title: "Name"
     data: "data name"
-  -
-    title: "Documentation"
-    data: "data doc"
   -
     title: "Status"
     data: "data msg"
@@ -1556,9 +1605,6 @@
   -
     title: "Name"
     data: "data name"
-  -
-    title: "Documentation"
-    data: "data doc"
   -
     title: "Status"
     data: "data msg"
@@ -1737,9 +1783,6 @@
   -
     title: "Name"
     data: "data name"
-  -
-    title: "Documentation"
-    data: "data doc"
   -
     title: "Status"
     data: "data msg"
@@ -2161,6 +2204,34 @@
   - "doc"
   - "status"
 
+################################################################################
+# DMM
+-
+  type: "table"
+  title: "Detailed Test Results - DMM Functional Results"
+  algorithm: "table_details"
+  output-file-ext: ".csv"
+  output-file: "{DIR[DTR,FUNC,DMM]}/dmm_functional_results"
+  columns:
+  -
+    title: "Name"
+    data: "data name"
+  -
+    title: "Documentation"
+    data: "data doc"
+  -
+    title: "Status"
+    data: "data status"
+  rows: "generated"
+  data:
+    "dmm-func-results"
+  filter: "all"
+  parameters:
+  - "name"
+  - "parent"
+  - "doc"
+  - "status"
+
 ################################################################################
 ###                                F I L E S                                 ###
 ################################################################################
@@ -2594,6 +2665,24 @@
   - "parent"
   data-start-level: 3
 
+-
+  type: "file"
+  title: "DMM Functional Results"
+  algorithm: "file_test_results"
+  output-file-ext: ".rst"
+  output-file: "{DIR[DTR,FUNC,DMM]}/dmm_functional_results"
+  file-header: "\n.. |br| raw:: html\n\n    <br />\n\n\n.. |prein| raw:: html\n\n    <pre>\n\n\n.. |preout| raw:: html\n\n    </pre>\n\n"
+  dir-tables: "{DIR[DTR,FUNC,DMM]}"
+  data:
+    "dmm-func-results"
+  filter: "all"
+  parameters:
+  - "name"
+  - "doc"
+  - "level"
+  - "parent"
+  data-start-level: 3
+
 ################################################################################
 ###                                P L O T S                                 ###
 ################################################################################
@@ -9922,7 +10011,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   traces:
     boxmean: False
@@ -9946,7 +10035,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   traces:
     boxmean: False
@@ -9971,7 +10060,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   traces:
     boxmean: False
@@ -9995,7 +10084,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   traces:
     boxmean: False
@@ -10020,7 +10109,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   traces:
     boxmean: False
@@ -10044,7 +10133,7 @@
            not 'VHOST' and
            not 'IPSECHW'"
   parameters:
-  - "throughput"
+  - "latency"
   - "parent"
   traces:
     boxmean: False
@@ -10731,7 +10820,7 @@
   title: "VPP Latency: vhost-ip4-3n-hsw-xl710-64b-2t2c-base_and_scale-ndr-lat"
   algorithm: "plot_latency_box"
   output-file-type: ".html"
-  output-file: "{DIR[STATIC,VPP]}/ip4-l2sw-3n-hsw-xl710-64b-2t2c-base_and_scale-ndr-lat"
+  output-file: "{DIR[STATIC,VPP]}/vhost-ip4-3n-hsw-xl710-64b-2t2c-base_and_scale-ndr-lat"
   data: "plot-vpp-throughput-latency-3n-hsw"
   filter: "'NIC_Intel-XL710' and
            '64B' and
@@ -12314,53 +12403,6 @@
     title: ""  # No title
     layout: "plot-throughput-speedup-analysis"
 
-# Speedup - VPP IPv4 Tunnels 3n-hsw-xl710
--
-  type: "plot"
-  title: "Speedup: ip4tun-3n-hsw-xl710-64b-base_and_scale-ndr-tsa"
-  algorithm: "plot_throughput_speedup_analysis"
-  output-file-type: ".html"
-  output-file: "{DIR[STATIC,VPP]}/ip4tun-3n-hsw-xl710-64b-base_and_scale-ndr-tsa"
-  data: "plot-throughput-speedup-analysis-3n-hsw"
-  filter: "'NIC_Intel-XL710' and
-           '64B' and
-           'ENCAP' and
-           ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and
-           'NDRPDR' and
-           'IP4FWD' and
-           not 'IPSEC' and
-           not 'VHOST'"
-  parameters:
-  - "throughput"
-  - "parent"
-  - "tags"
-  layout:
-    title: ""  # No title
-    layout: "plot-throughput-speedup-analysis"
-
--
-  type: "plot"
-  title: "Speedup: ip4tun-3n-hsw-xl710-64b-base_and_scale-pdr-tsa"
-  algorithm: "plot_throughput_speedup_analysis"
-  output-file-type: ".html"
-  output-file: "{DIR[STATIC,VPP]}/ip4tun-3n-hsw-xl710-64b-base_and_scale-pdr-tsa"
-  data: "plot-throughput-speedup-analysis-3n-hsw"
-  filter: "'NIC_Intel-XL710' and
-           '64B' and
-           'ENCAP' and
-           ('VXLAN' or 'VXLANGPE' or 'LISP' or 'LISPGPE' or 'GRE') and
-           'NDRPDR' and
-           'IP4FWD' and
-           not 'IPSEC' and
-           not 'VHOST'"
-  parameters:
-  - "throughput"
-  - "parent"
-  - "tags"
-  layout:
-    title: ""  # No title
-    layout: "plot-throughput-speedup-analysis"
-
 # Speedup - VPP IPv4 Tunnels 3n-skx-x710
 -
   type: "plot"
Integration tests