From e244fff23cc6d0ff85ab50632316b7bf9db2ea62 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Fri, 15 May 2020 07:29:15 +0200 Subject: [PATCH] Trending: Add alerts for NDRPDR weekly tests Change-Id: I35a806f8dd61e2c6d6dca2cdcfa4f7131e061cea Signed-off-by: Tibor Frank --- docs/cpta/introduction/failures.rst | 48 ++- .../container_memif-3n-hsw-xl710.rst | 2 +- docs/cpta/ndrpdr_trending/ip4-3n-hsw-x710.rst | 2 +- docs/cpta/ndrpdr_trending/ip6-3n-hsw-xl710.rst | 2 +- resources/tools/presentation/generator_tables.py | 21 +- .../tools/presentation/specification_CPTA.yaml | 333 +++++++++++++++------ 6 files changed, 295 insertions(+), 113 deletions(-) diff --git a/docs/cpta/introduction/failures.rst b/docs/cpta/introduction/failures.rst index f49a7fc26a..214b87c455 100644 --- a/docs/cpta/introduction/failures.rst +++ b/docs/cpta/introduction/failures.rst @@ -13,37 +13,57 @@ Legend to the tables: - **Last Failure [VPP-Build-Id]**: VPP build as of last failure. - **Last Failure [CSIT-Job-Build-Id]**: CSIT build as of last failure. -2n-skx ------- +2n-skx-mrr +---------- .. include:: ../../../_build/_static/vpp/failed-tests-2n-skx.rst -3n-skx ------- +2n-skx-ndrpdr +------------- + +.. include:: ../../../_build/_static/vpp/failed-tests-2n-skx-ndrpdr.rst + +3n-skx-mrr +---------- .. include:: ../../../_build/_static/vpp/failed-tests-3n-skx.rst -3n-hsw ------- +3n-skx-ndrpdr +------------- + +.. include:: ../../../_build/_static/vpp/failed-tests-3n-skx-ndrpdr.rst + +3n-hsw-mrr +---------- .. include:: ../../../_build/_static/vpp/failed-tests-3n-hsw.rst -3n-tsh ------- +3n-hsw-ndrpdr +------------- + +.. include:: ../../../_build/_static/vpp/failed-tests-3n-hsw-ndrpdr.rst + +3n-tsh-mrr +---------- .. include:: ../../../_build/_static/vpp/failed-tests-3n-tsh.rst -2n-dnv ------- +2n-dnv-mrr +---------- .. include:: ../../../_build/_static/vpp/failed-tests-2n-dnv.rst -3n-dnv ------- +3n-dnv-mrr +---------- .. include:: ../../../_build/_static/vpp/failed-tests-3n-dnv.rst -2n-clx ------- +2n-clx-mrr +---------- .. include:: ../../../_build/_static/vpp/failed-tests-2n-clx.rst + +2n-clx-ndrpdr +------------- + +.. include:: ../../../_build/_static/vpp/failed-tests-2n-clx-ndrpdr.rst diff --git a/docs/cpta/ndrpdr_trending/container_memif-3n-hsw-xl710.rst b/docs/cpta/ndrpdr_trending/container_memif-3n-hsw-xl710.rst index 8361822312..a94056c361 100644 --- a/docs/cpta/ndrpdr_trending/container_memif-3n-hsw-xl710.rst +++ b/docs/cpta/ndrpdr_trending/container_memif-3n-hsw-xl710.rst @@ -1,5 +1,5 @@ 3n-hsw-xl710 ------------ +------------ 64b-memif-base-dpdk ``````````````````` diff --git a/docs/cpta/ndrpdr_trending/ip4-3n-hsw-x710.rst b/docs/cpta/ndrpdr_trending/ip4-3n-hsw-x710.rst index a07eec9c89..25e29f9056 100644 --- a/docs/cpta/ndrpdr_trending/ip4-3n-hsw-x710.rst +++ b/docs/cpta/ndrpdr_trending/ip4-3n-hsw-x710.rst @@ -66,7 +66,7 @@ 64b-ip4routing-features-udp-dpdk -````````````````````````````````` +```````````````````````````````` 1t1c-ndr :::::::: diff --git a/docs/cpta/ndrpdr_trending/ip6-3n-hsw-xl710.rst b/docs/cpta/ndrpdr_trending/ip6-3n-hsw-xl710.rst index 4f2d0aceb8..754ee7d37e 100644 --- a/docs/cpta/ndrpdr_trending/ip6-3n-hsw-xl710.rst +++ b/docs/cpta/ndrpdr_trending/ip6-3n-hsw-xl710.rst @@ -1,5 +1,5 @@ 3n-hsw-xl710 ------------ +------------ 78b-ip6routing-base-dpdk ```````````````````````` diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index c745be81cd..2644eb0348 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1260,6 +1260,10 @@ def table_failed_tests(table, input_data): ) data = input_data.filter_data(table, continue_on_error=True) + test_type = u"MRR" + if u"NDRPDR" in table.get(u"filter", list()): + test_type = u"NDRPDR" + # Prepare the header of the tables header = [ u"Test Case", @@ -1323,15 +1327,14 @@ def table_failed_tests(table, input_data): fails_last_csit = val[3] if fails_nr: max_fails = fails_nr if fails_nr > max_fails else max_fails - tbl_lst.append( - [ - tst_data[u"name"], - fails_nr, - fails_last_date, - fails_last_vpp, - f"mrr-daily-build-{fails_last_csit}" - ] - ) + tbl_lst.append([ + tst_data[u"name"], + fails_nr, + fails_last_date, + fails_last_vpp, + f"{u'mrr-daily' if test_type == u'MRR' else u'ndrpdr-weekly'}" + f"-build-{fails_last_csit}" + ]) tbl_lst.sort(key=lambda rel: rel[2], reverse=True) tbl_sorted = list() diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index f3f8ccff23..5684f8f544 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -133,6 +133,10 @@ - "last-failed-tests-3n-tsh" - "last-failed-tests-2n-dnv" - "last-failed-tests-3n-dnv" + - "last-failed-tests-2n-skx-ndrpdr" + - "last-failed-tests-3n-skx-ndrpdr" + - "last-failed-tests-2n-clx-ndrpdr" + - "last-failed-tests-3n-hsw-ndrpdr" urls: - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-skx" - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-3n-skx" @@ -141,6 +145,10 @@ - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-3n-tsh" - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-dnv" - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-3n-dnv" + - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-ndrpdr-weekly-master-2n-skx" + - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-ndrpdr-weekly-master-3n-skx" + - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-ndrpdr-weekly-master-2n-clx" + - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-ndrpdr-weekly-master-3n-hsw" url-details: "https://docs.fd.io/csit/master/trending/introduction/failures.html" configurations: @@ -177,9 +185,6 @@ table-last-failed-tests-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - "lastCompletedBuild" - table-last-failed-tests-nf-2n-skx: - csit-vpp-perf-mrr-weekly-master-2n-skx: - - "lastCompletedBuild" table-last-failed-tests-2n-clx: csit-vpp-perf-mrr-daily-master-2n-clx: - "lastCompletedBuild" @@ -192,6 +197,18 @@ table-last-failed-tests-2n-dnv: csit-vpp-perf-mrr-daily-master-2n-dnv: - "lastCompletedBuild" + table-last-failed-tests-3n-hsw-ndrpdr: + csit-vpp-perf-ndrpdr-weekly-master-3n-hsw: + - "lastCompletedBuild" + table-last-failed-tests-3n-skx-ndrpdr: + csit-vpp-perf-ndrpdr-weekly-master-3n-skx: + - "lastCompletedBuild" + table-last-failed-tests-2n-skx-ndrpdr: + csit-vpp-perf-ndrpdr-weekly-master-2n-skx: + - "lastCompletedBuild" + table-last-failed-tests-2n-clx-ndrpdr: + csit-vpp-perf-ndrpdr-weekly-master-2n-clx: + - "lastCompletedBuild" # 3n-hsw plot-performance-trending-all-3n-hsw: @@ -276,16 +293,19 @@ start: 144 end: "lastCompletedBuild" + # 3n-dnv plot-performance-trending-vpp-3n-dnv: csit-vpp-perf-mrr-daily-master-3n-dnv: start: 329 end: "lastCompletedBuild" + # 2n-dnv plot-performance-trending-vpp-2n-dnv: csit-vpp-perf-mrr-daily-master-2n-dnv: start: 335 end: "lastCompletedBuild" + # NDRPDR plot-ndrpdr-comparison-2n-skx: csit-vpp-perf-ndrpdr-weekly-master-2n-skx: start: 76 @@ -831,6 +851,58 @@ - "parent" - "status" +- + type: "table" + title: "Last failed tests (last builds) 3n-hsw-ndrpdr" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-3n-hsw-ndrpdr" + data: "table-last-failed-tests-3n-hsw-ndrpdr" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "status" + +- + type: "table" + title: "Last failed tests (last builds) 3n-skx-ndrpdr" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-3n-skx-ndrpdr" + data: "table-last-failed-tests-3n-skx-ndrpdr" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "status" + +- + type: "table" + title: "Last failed tests (last builds) 2n-skx-ndrpdr" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-2n-skx-ndrpdr" + data: "table-last-failed-tests-2n-skx-ndrpdr" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "status" + +- + type: "table" + title: "Last failed tests (last builds) 2n-clx-ndrpdr" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-2n-clx-ndrpdr" + data: "table-last-failed-tests-2n-clx-ndrpdr" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "status" + # 3n-hsw - type: "table" @@ -841,9 +913,9 @@ data: "plot-performance-trending-all-3n-hsw" filter: "'MRR' and '1T1C' and not 'NIC_Intel-XXV710'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -856,9 +928,9 @@ data: "plot-performance-trending-all-3n-hsw" filter: "'MRR' and '2T2C' and not 'NIC_Intel-XXV710'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -871,9 +943,9 @@ data: "plot-performance-trending-all-3n-hsw" filter: "'MRR' and '4T4C' and not 'NIC_Intel-XXV710'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1105,7 +1177,7 @@ testbed: "2n-clx" add-links: False -- +#- type: "table" title: "Failed MRR tests 3n-hsw" algorithm: "table_failed_tests" @@ -1114,9 +1186,9 @@ data: "plot-performance-trending-all-3n-hsw" filter: "'MRR' and not 'NIC_Intel-XXV710'" parameters: - - "name" - - "parent" - - "status" + - "name" + - "parent" + - "status" window: 7 # days - @@ -1127,6 +1199,28 @@ output-file: "{DIR[STATIC,VPP]}/failed-tests-3n-hsw.rst" testbed: "3n-hsw" +- + type: "table" + title: "Failed NDRPDR tests 3n-hsw" + algorithm: "table_failed_tests" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-3n-hsw-ndrpdr" + data: "plot-ndrpdr-comparison-3n-hsw" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "status" + window: 30 # days + +- + type: "table" + title: "HTML Failed NDRPDR tests 3n-hsw" + algorithm: "table_failed_tests_html" + input-file: "{DIR[STATIC,VPP]}/failed-tests-3n-hsw-ndrpdr.csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-3n-hsw-ndrpdr.rst" + testbed: "3n-hsw" + ################################################################################ # 3n-skx - @@ -1138,9 +1232,9 @@ data: "plot-performance-trending-all-3n-skx" filter: "'MRR' and '2T1C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1153,9 +1247,9 @@ data: "plot-performance-trending-all-3n-skx" filter: "'MRR' and '4T2C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1168,9 +1262,9 @@ data: "plot-performance-trending-all-3n-skx" filter: "'MRR' and '8T4C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1207,9 +1301,9 @@ data: "plot-performance-trending-all-3n-skx" filter: "'MRR'" parameters: - - "name" - - "parent" - - "status" + - "name" + - "parent" + - "status" window: 7 # days - @@ -1220,6 +1314,28 @@ output-file: "{DIR[STATIC,VPP]}/failed-tests-3n-skx.rst" testbed: "3n-skx" +- + type: "table" + title: "Failed NDRPDR tests 3n-skx" + algorithm: "table_failed_tests" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-3n-skx-ndrpdr" + data: "plot-ndrpdr-comparison-3n-skx" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "status" + window: 30 # days + +- + type: "table" + title: "HTML Failed NDRPDR tests 3n-skx" + algorithm: "table_failed_tests_html" + input-file: "{DIR[STATIC,VPP]}/failed-tests-3n-skx-ndrpdr.csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-3n-skx-ndrpdr.rst" + testbed: "3n-skx" + ################################################################################ # 2n-skx - @@ -1231,9 +1347,9 @@ data: "plot-performance-trending-all-2n-skx" filter: "'MRR' and '2T1C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1246,9 +1362,9 @@ data: "plot-performance-trending-all-2n-skx" filter: "'MRR' and '4T2C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1261,9 +1377,9 @@ data: "plot-performance-trending-all-2n-skx" filter: "'MRR' and '8T4C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1300,9 +1416,9 @@ data: "plot-performance-trending-all-2n-skx" filter: "'MRR'" parameters: - - "name" - - "parent" - - "status" + - "name" + - "parent" + - "status" window: 7 # days - @@ -1313,6 +1429,28 @@ output-file: "{DIR[STATIC,VPP]}/failed-tests-2n-skx.rst" testbed: "2n-skx" +- + type: "table" + title: "Failed NDRPDR tests 2n-skx" + algorithm: "table_failed_tests" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-2n-skx-ndrpdr" + data: "plot-ndrpdr-comparison-2n-skx" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "status" + window: 30 # days + +- + type: "table" + title: "HTML Failed NDRPDR tests 2n-skx" + algorithm: "table_failed_tests_html" + input-file: "{DIR[STATIC,VPP]}/failed-tests-2n-skx-ndrpdr.csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-2n-skx-ndrpdr.rst" + testbed: "2n-skx" + ################################################################################ # 2n-clx - @@ -1324,9 +1462,9 @@ data: "plot-performance-trending-vpp-2n-clx" filter: "'MRR' and '2T1C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1339,9 +1477,9 @@ data: "plot-performance-trending-vpp-2n-clx" filter: "'MRR' and '4T2C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1354,9 +1492,9 @@ data: "plot-performance-trending-vpp-2n-clx" filter: "'MRR' and '8T4C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1393,9 +1531,9 @@ data: "plot-performance-trending-vpp-2n-clx" filter: "'MRR'" parameters: - - "name" - - "parent" - - "status" + - "name" + - "parent" + - "status" window: 7 # days - @@ -1406,9 +1544,30 @@ output-file: "{DIR[STATIC,VPP]}/failed-tests-2n-clx.rst" testbed: "2n-clx" +- + type: "table" + title: "Failed NDRPDR tests 2n-clx" + algorithm: "table_failed_tests" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-2n-clx-ndrpdr" + data: "plot-ndrpdr-comparison-2n-clx" + filter: "'NDRPDR'" + parameters: + - "name" + - "parent" + - "status" + window: 30 # days + +- + type: "table" + title: "HTML Failed NDRPDR tests 2n-clx" + algorithm: "table_failed_tests_html" + input-file: "{DIR[STATIC,VPP]}/failed-tests-2n-clx-ndrpdr.csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-2n-clx-ndrpdr.rst" + testbed: "2n-clx" -################################################################################ -# 3n-tsh +################################################################################# +## 3n-tsh - type: "table" title: "Performance trending dashboard 1t1c-3n-tsh" @@ -1418,9 +1577,9 @@ data: "plot-performance-trending-all-3n-tsh" filter: "'MRR' and '1T1C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1448,9 +1607,9 @@ data: "plot-performance-trending-all-3n-tsh" filter: "'MRR' and '4T4C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1487,9 +1646,9 @@ data: "plot-performance-trending-all-3n-tsh" filter: "'MRR'" parameters: - - "name" - - "parent" - - "status" + - "name" + - "parent" + - "status" window: 7 # days - @@ -1511,9 +1670,9 @@ data: "plot-performance-trending-vpp-3n-dnv" filter: "'MRR' and '1T1C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1526,9 +1685,9 @@ data: "plot-performance-trending-vpp-3n-dnv" filter: "'MRR' and '2T2C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1541,9 +1700,9 @@ data: "plot-performance-trending-vpp-3n-dnv" filter: "'MRR' and '4T4C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1580,9 +1739,9 @@ data: "plot-performance-trending-vpp-3n-dnv" filter: "'MRR'" parameters: - - "name" - - "parent" - - "status" + - "name" + - "parent" + - "status" window: 7 # days - @@ -1604,9 +1763,9 @@ data: "plot-performance-trending-vpp-2n-dnv" filter: "'MRR' and '1T1C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1619,9 +1778,9 @@ data: "plot-performance-trending-vpp-2n-dnv" filter: "'MRR' and '2T2C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1634,9 +1793,9 @@ data: "plot-performance-trending-vpp-2n-dnv" filter: "'MRR' and '4T4C'" parameters: - - "name" - - "parent" - - "result" + - "name" + - "parent" + - "result" window: 14 long-trend-window: 180 @@ -1673,9 +1832,9 @@ data: "plot-performance-trending-vpp-2n-dnv" filter: "'MRR'" parameters: - - "name" - - "parent" - - "status" + - "name" + - "parent" + - "status" window: 7 # days - -- 2.16.6