From c98f749024b1f42d0065a16ac1ee904a4c9ca704 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Mon, 27 Sep 2021 15:14:32 +0200 Subject: [PATCH] Trending: Add aws test beds Change-Id: I0536d346277431e988155ca17e5891b9068cc074 Signed-off-by: Tibor Frank --- docs/cpta/data/index.rst | 8 + docs/cpta/introduction/dashboard.rst | 26 +++ docs/cpta/stats/vpp_mrr.rst | 24 +++ docs/cpta/trending/ip4-2n-aws-nitro50g.rst | 68 +++++++ docs/cpta/trending/ip4-3n-aws-nitro50g.rst | 68 +++++++ docs/cpta/trending/ip4.rst | 2 + resources/tools/presentation/generator_tables.py | 13 +- .../specifications/trending/data_sets.yaml | 20 ++ .../specifications/trending/elements.yaml | 211 ++++++++++++++++++++- 9 files changed, 436 insertions(+), 4 deletions(-) create mode 100644 docs/cpta/trending/ip4-2n-aws-nitro50g.rst create mode 100644 docs/cpta/trending/ip4-3n-aws-nitro50g.rst diff --git a/docs/cpta/data/index.rst b/docs/cpta/data/index.rst index 4c47f0fde6..839b598e2d 100644 --- a/docs/cpta/data/index.rst +++ b/docs/cpta/data/index.rst @@ -38,6 +38,14 @@ in the CSV formats: - `CSV format <../_static/vpp/csit-vpp-perf-mrr-daily-master-2n-dnv-trending.csv>`_ + - 3n-aws + + - `CSV format <../_static/vpp/csit-vpp-perf-mrr-daily-master-3n-aws-trending.csv>`_ + + - 2n-aws + + - `CSV format <../_static/vpp/csit-vpp-perf-mrr-daily-master-2n-aws-trending.csv>`_ + **VPP NDRPDR** - 3n-skx diff --git a/docs/cpta/introduction/dashboard.rst b/docs/cpta/introduction/dashboard.rst index 58480ba3e3..3a924fcfdb 100644 --- a/docs/cpta/introduction/dashboard.rst +++ b/docs/cpta/introduction/dashboard.rst @@ -141,6 +141,32 @@ VPP worker on 8t4c .. include:: ../../../_build/_static/vpp/performance-trending-dashboard-8t4c-2n-clx.rst +2n-aws +------ + +VPP worker on 2t1c +`````````````````` + +.. include:: ../../../_build/_static/vpp/performance-trending-dashboard-2t1c-2n-aws.rst + +VPP worker on 4t2c +`````````````````` + +.. include:: ../../../_build/_static/vpp/performance-trending-dashboard-4t2c-2n-aws.rst + +3n-aws +------ + +VPP worker on 2t1c +`````````````````` + +.. include:: ../../../_build/_static/vpp/performance-trending-dashboard-2t1c-3n-aws.rst + +VPP worker on 4t2c +`````````````````` + +.. include:: ../../../_build/_static/vpp/performance-trending-dashboard-4t2c-3n-aws.rst + NDR Weekly VPP worker on 2t1c ````````````````````````````` diff --git a/docs/cpta/stats/vpp_mrr.rst b/docs/cpta/stats/vpp_mrr.rst index b4700c8975..d7eaf3b4b4 100644 --- a/docs/cpta/stats/vpp_mrr.rst +++ b/docs/cpta/stats/vpp_mrr.rst @@ -96,3 +96,27 @@ VPP MRR


+ +2n-aws +------ + +.. raw:: html + +
+ Links to builds: + csit-ref + +


+
+ +3n-aws +------ + +.. raw:: html + +
+ Links to builds: + csit-ref + +


+
diff --git a/docs/cpta/trending/ip4-2n-aws-nitro50g.rst b/docs/cpta/trending/ip4-2n-aws-nitro50g.rst new file mode 100644 index 0000000000..3f6cd5553e --- /dev/null +++ b/docs/cpta/trending/ip4-2n-aws-nitro50g.rst @@ -0,0 +1,68 @@ +2n-aws-nitro50g +--------------- + +64b-ip4routing-base-scale-ena +````````````````````````````` + +2t1c +:::: + +.. raw:: html + + + +
+ Links to builds: + vpp-ref, + csit-ref + +


+
+ +4t2c +:::: + +.. raw:: html + + + +
+ Links to builds: + vpp-ref, + csit-ref + +


+
+ +1518b-ip4routing-base-scale-ena +``````````````````````````````` + +2t1c +:::: + +.. raw:: html + + + +
+ Links to builds: + vpp-ref, + csit-ref + +


+
+ +4t2c +:::: + +.. raw:: html + + + +
+ Links to builds: + vpp-ref, + csit-ref + +


+
diff --git a/docs/cpta/trending/ip4-3n-aws-nitro50g.rst b/docs/cpta/trending/ip4-3n-aws-nitro50g.rst new file mode 100644 index 0000000000..cca2b3a0ee --- /dev/null +++ b/docs/cpta/trending/ip4-3n-aws-nitro50g.rst @@ -0,0 +1,68 @@ +3n-aws-nitro50g +--------------- + +64b-ip4routing-base-scale-ena +````````````````````````````` + +2t1c +:::: + +.. raw:: html + + + +
+ Links to builds: + vpp-ref, + csit-ref + +


+
+ +4t2c +:::: + +.. raw:: html + + + +
+ Links to builds: + vpp-ref, + csit-ref + +


+
+ +1518b-ip4routing-base-scale-ena +``````````````````````````````` + +2t1c +:::: + +.. raw:: html + + + +
+ Links to builds: + vpp-ref, + csit-ref + +


+
+ +4t2c +:::: + +.. raw:: html + + + +
+ Links to builds: + vpp-ref, + csit-ref + +


+
diff --git a/docs/cpta/trending/ip4.rst b/docs/cpta/trending/ip4.rst index 1c9c4a1988..7f60e0fd2c 100644 --- a/docs/cpta/trending/ip4.rst +++ b/docs/cpta/trending/ip4.rst @@ -18,3 +18,5 @@ IPv4 Routing ip4-2n-zn2-xxv710 ip4-2n-zn2-x710 ip4-2n-zn2-cx556a + ip4-2n-aws-nitro50g + ip4-3n-aws-nitro50g diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 1e1307b5bc..a995711bcb 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1111,6 +1111,8 @@ def _generate_url(testbed, test_name): nic = u"x553" elif u"cx556" in test_name or u"cx556a" in test_name: nic = u"cx556a" + elif u"ena" in test_name: + nic = u"nitro50g" else: nic = u"" @@ -1143,15 +1145,18 @@ def _generate_url(testbed, test_name): cores = u"4t4c" elif u"2t1c" in test_name or \ (u"-1c-" in test_name and - testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")): + testbed in + (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")): cores = u"2t1c" elif u"4t2c" in test_name or \ (u"-2c-" in test_name and - testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")): + testbed in + (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")): cores = u"4t2c" elif u"8t4c" in test_name or \ (u"-4c-" in test_name and - testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")): + testbed in + (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")): cores = u"8t4c" else: cores = u"" @@ -1166,6 +1171,8 @@ def _generate_url(testbed, test_name): driver = u"rdma" elif u"dnv" in testbed or u"tsh" in testbed: driver = u"ixgbe" + elif u"ena" in test_name: + driver = u"ena" else: driver = u"dpdk" diff --git a/resources/tools/presentation/specifications/trending/data_sets.yaml b/resources/tools/presentation/specifications/trending/data_sets.yaml index 3afc97e6a4..2d61c3c042 100644 --- a/resources/tools/presentation/specifications/trending/data_sets.yaml +++ b/resources/tools/presentation/specifications/trending/data_sets.yaml @@ -29,6 +29,12 @@ table-last-failed-tests-2n-dnv: csit-vpp-perf-mrr-daily-master-2n-dnv: - "lastCompletedBuild" + table-last-failed-tests-3n-aws: + csit-vpp-perf-mrr-weekly-master-3n-aws: + - "lastCompletedBuild" + table-last-failed-tests-2n-aws: + csit-vpp-perf-mrr-weekly-master-2n-aws: + - "lastCompletedBuild" table-last-failed-tests-3n-skx-ndrpdr: csit-vpp-perf-ndrpdr-weekly-master-3n-skx: - "lastCompletedBuild" @@ -184,6 +190,20 @@ end: "lastCompletedBuild" max-builds: 180 + # 3n-aws + plot-performance-trending-vpp-3n-aws: + csit-vpp-perf-mrr-weekly-master-3n-aws: + start: 5 + end: "lastCompletedBuild" + max-builds: 26 + + # 2n-aws + plot-performance-trending-vpp-2n-aws: + csit-vpp-perf-mrr-weekly-master-2n-aws: + start: 4 + end: "lastCompletedBuild" + max-builds: 26 + # NDRPDR plot-ndrpdr-comparison-2n-skx: csit-vpp-perf-ndrpdr-weekly-master-2n-skx: diff --git a/resources/tools/presentation/specifications/trending/elements.yaml b/resources/tools/presentation/specifications/trending/elements.yaml index ca1ba6bba9..21e5d5cb25 100644 --- a/resources/tools/presentation/specifications/trending/elements.yaml +++ b/resources/tools/presentation/specifications/trending/elements.yaml @@ -138,6 +138,22 @@ layout: layout: "plot-statistics" +- type: "plot" + title: "Statistics: 3n-aws mrr" + algorithm: "plot_statistics" + output-file: "{DIR[STATIC,VPP]}/stats-3n-aws-mrr" + data: "plot-performance-trending-vpp-3n-aws" + layout: + layout: "plot-statistics" + +- type: "plot" + title: "Statistics: 2n-aws mrr" + algorithm: "plot_statistics" + output-file: "{DIR[STATIC,VPP]}/stats-2n-aws-mrr" + data: "plot-performance-trending-vpp-2n-aws" + layout: + layout: "plot-statistics" + ################################################################################ ### T A B L E S ### ################################################################################ @@ -336,6 +352,32 @@ - "status" - "msg" +- type: "table" + title: "Last failed tests (last builds) VPP 3n-aws" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-vpp-3n-aws-mrr" + data: "table-last-failed-tests-3n-aws" + filter: "'MRR'" + parameters: + - "name" + - "parent" + - "status" + - "msg" + +- type: "table" + title: "Last failed tests (last builds) VPP 2n-aws" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-vpp-2n-aws-mrr" + data: "table-last-failed-tests-2n-aws" + filter: "'MRR'" + parameters: + - "name" + - "parent" + - "status" + - "msg" + - type: "table" title: "Last failed tests (last builds) VPP 3n-skx-ndrpdr" algorithm: "table_last_failed_tests" @@ -1341,6 +1383,137 @@ output-file: "{DIR[STATIC,VPP]}/failed-tests-2n-dnv.rst" testbed: "2n-dnv" + +################################################################################ +# 3n-aws +- type: "table" + title: "Performance trending dashboard 2t1c-3n-aws" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-3n-aws" + data: "plot-performance-trending-vpp-3n-aws" + filter: "'MRR' and '2T1C'" + parameters: + - "name" + - "parent" + - "result" + window: 5 + long-trend-window: 26 + +- type: "table" + title: "Performance trending dashboard 4t2c-3n-aws" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-4t2c-3n-aws" + data: "plot-performance-trending-vpp-3n-aws" + filter: "'MRR' and '4T2C'" + parameters: + - "name" + - "parent" + - "result" + window: 5 + long-trend-window: 26 + +- type: "table" + title: "HTML performance trending dashboard 2t1c-3n-aws" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-3n-aws.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-3n-aws.rst" + testbed: "3n-aws" + +- type: "table" + title: "HTML performance trending dashboard 4t2c-3n-aws" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-4t2c-3n-aws.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-4t2c-3n-aws.rst" + testbed: "3n-aws" + +- type: "table" + title: "Failed MRR tests 3n-aws" + algorithm: "table_failed_tests" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-3n-aws" + data: "plot-performance-trending-vpp-3n-aws" + filter: "'MRR'" + parameters: + - "name" + - "parent" + - "status" + - "msg" + window: 7 # days + +- type: "table" + title: "HTML Failed MRR tests 3n-aws" + algorithm: "table_failed_tests_html" + input-file: "{DIR[STATIC,VPP]}/failed-tests-3n-aws.csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-3n-aws.rst" + testbed: "3n-aws" + +################################################################################ +# 2n-aws +- type: "table" + title: "Performance trending dashboard 2t1c-2n-aws" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-aws" + data: "plot-performance-trending-vpp-2n-aws" + filter: "'MRR' and '2T1C'" + parameters: + - "name" + - "parent" + - "result" + window: 5 + long-trend-window: 26 + +- type: "table" + title: "Performance trending dashboard 4t2c-2n-aws" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-4t2c-2n-aws" + data: "plot-performance-trending-vpp-2n-aws" + filter: "'MRR' and '4T2C'" + parameters: + - "name" + - "parent" + - "result" + window: 5 + long-trend-window: 26 + +- type: "table" + title: "HTML performance trending dashboard 2t1c-2n-aws" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-aws.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-aws.rst" + testbed: "2n-aws" + +- type: "table" + title: "HTML performance trending dashboard 4t2c-2n-aws" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-4t2c-2n-aws.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-4t2c-2n-aws.rst" + testbed: "2n-aws" + +- type: "table" + title: "Failed MRR tests 2n-aws" + algorithm: "table_failed_tests" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-2n-aws" + data: "plot-performance-trending-vpp-2n-aws" + filter: "'MRR'" + parameters: + - "name" + - "parent" + - "status" + - "msg" + window: 7 # days + +- type: "table" + title: "HTML Failed MRR tests 2n-aws" + algorithm: "table_failed_tests_html" + input-file: "{DIR[STATIC,VPP]}/failed-tests-2n-aws.csv" + output-file: "{DIR[STATIC,VPP]}/failed-tests-2n-aws.rst" + testbed: "2n-aws" + ################################################################################ ### C P T A ### ################################################################################ @@ -1871,7 +2044,7 @@ - title: "2n-dnv-x553-64b-{core}-ip4routing-base-scale-ixgbe" output-file-name: "2n-dnv-x553-64b-{core}-ip4-ixgbe" data: "plot-performance-trending-vpp-2n-dnv" - core: ["2t1c", "4t2c", "8t4c"] + core: ["1t1c", "2t2c", "4t4c"] include: - "Tests.Vpp.Perf.Ip4.2N1L-10Ge2P1X553-Ethip4-Ip4Base-mrr.64B-{core}-ethip4-ip4base-mrr" - "Tests.Vpp.Perf.Ip4.2N1L-10Ge2P1X553-Ethip4-Ip4Scale20K-mrr.64B-{core}-ethip4-ip4scale20k-mrr" @@ -1880,6 +2053,24 @@ - "Tests.Vpp.Perf.Ip4.2N1L-10Ge2P1X553-Ethip4-Ip4Scale20K-rnd-mrr.64B-{core}-ethip4-ip4scale20k-rnd-mrr" layout: "plot-cpta" + - title: "2n-aws-nitro50g-64b-{core}-ip4routing-base-scale-ena" + output-file-name: "2n-aws-nitro50g-64b-{core}-ip4-ena" + data: "plot-performance-trending-vpp-2n-aws" + core: ["2t1c", "4t2c"] + include: + - "Tests.Vpp.Perf.Ip4.2n1l-50ge1p1ENA-ethip4-ip4base-mrr.64B-1c-ethip4-ip4base-mrr" + - "Tests.Vpp.Perf.Ip4.2n1l-50ge1p1ENA-ethip4-ip4scale2m-mrr.64B-1c-ethip4-ip4scale2m-mrr" + layout: "plot-cpta" + + - title: "2n-aws-nitro50g-1518b-{core}-ip4routing-base-scale-ena" + output-file-name: "2n-aws-nitro50g-1518b-{core}-ip4-ena" + data: "plot-performance-trending-vpp-2n-aws" + core: ["2t1c", "4t2c"] + include: + - "Tests.Vpp.Perf.Ip4.2n1l-50ge1p1ENA-ethip4-ip4base-mrr.1518B-1c-ethip4-ip4base-mrr" + - "Tests.Vpp.Perf.Ip4.2n1l-50ge1p1ENA-ethip4-ip4scale2m-mrr.1518B-1c-ethip4-ip4scale2m-mrr" + layout: "plot-cpta" + - title: "2n-skx-x710-64b-{core}-ip4routing-base-scale-avf-dpdk" output-file-name: "2n-skx-x710-64b-{core}-ip4-avf-dpdk" data: "plot-performance-trending-vpp-2n-skx" @@ -2105,6 +2296,24 @@ - "Tests.Vpp.Perf.Ip4.10Ge2P1X553-Ethip4-Ip4Scale20K-rnd-mrr.64B-{core}-ethip4-ip4scale20k-rnd-mrr" layout: "plot-cpta" + - title: "3n-aws-nitro50g-64b-{core}-ip4routing-base-scale-ena" + output-file-name: "3n-aws-nitro50g-64b-{core}-ip4-ena" + data: "plot-performance-trending-vpp-3n-aws" + core: ["2t1c", "4t2c"] + include: + - "Tests.Vpp.Perf.Ip4.50ge1p1ENA-ethip4-ip4base-mrr.64B-1c-ethip4-ip4base-mrr" + - "Tests.Vpp.Perf.Ip4.50ge1p1ENA-ethip4-ip4scale2m-mrr.64B-1c-ethip4-ip4scale2m-mrr" + layout: "plot-cpta" + + - title: "3n-aws-nitro50g-1518b-{core}-ip4routing-base-scale-ena" + output-file-name: "3n-aws-nitro50g-1518b-{core}-ip4-ena" + data: "plot-performance-trending-vpp-3n-aws" + core: ["2t1c", "4t2c"] + include: + - "Tests.Vpp.Perf.Ip4.50ge1p1ENA-ethip4-ip4base-mrr.1518B-1c-ethip4-ip4base-mrr" + - "Tests.Vpp.Perf.Ip4.50ge1p1ENA-ethip4-ip4scale2m-mrr.1518B-1c-ethip4-ip4scale2m-mrr" + layout: "plot-cpta" + - title: "3n-skx-x710-64b-{core}-ip4routing-base-scale-avf" output-file-name: "3n-skx-x710-64b-{core}-ip4-base-scale-avf" data: "plot-performance-trending-vpp-3n-skx" -- 2.16.6