From db2fcb13bab0085abaaa704e87e345d43734ac86 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Mon, 13 May 2019 13:56:39 +0200 Subject: [PATCH] CSIT-1500: Add comparison table for SOAK vs NDRPDR Change-Id: I890b28b7a8c1f60cce4c985ccd94fcdfd3c47d7d Signed-off-by: Tibor Frank --- .../vpp_performance_tests/comparisons/index.rst | 2 +- .../comparisons/soak_vs_ndr.rst | 18 ++++ resources/tools/presentation/generator_tables.py | 109 +++++++++++++++++++++ resources/tools/presentation/pal.py | 84 ++++++++-------- resources/tools/presentation/specification.yaml | 68 +++++++++++++ 5 files changed, 238 insertions(+), 43 deletions(-) create mode 100644 docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst index 8c502c49d7..1ce9427d99 100644 --- a/docs/report/vpp_performance_tests/comparisons/index.rst +++ b/docs/report/vpp_performance_tests/comparisons/index.rst @@ -14,4 +14,4 @@ Comparisons 3n-skx_vs_3n-hsw_testbeds 3n-skx_vs_2n-skx_testbeds nic_comparison - + soak_vs_ndr diff --git a/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst b/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst new file mode 100644 index 0000000000..e84d902149 --- /dev/null +++ b/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst @@ -0,0 +1,18 @@ +SOAK Tests vs. NDR Tests +------------------------ + +Relative comparison of |vpp-release| SOAK PLRSearch vs. NDR packet throughput +is calculated for the tests executed on 2-Node Skylake physical testbed types, +in 1-core configurations. + +.. note:: + + Test results have been generated by + `FD.io test executor vpp performance job 2n-skx`_ + with RF result files csit-vpp-perf-|srelease|-\*.zip + `archived here <../../_static/archive/>`_. + +Comparison tables in ASCII and CSV formats: + + - `ASCII SOAK vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.txt>`_ + - `CSV SOAK vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.csv>`_ diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 0646db3ab8..1a15605618 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -525,6 +525,115 @@ def table_nics_comparison(table, input_data): convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) +def table_soak_vs_ndr(table, input_data): + """Generate the table(s) with algorithm: table_soak_vs_ndr + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) + + # Prepare the header of the table + try: + header = [ + "Test case", + "{0} Throughput [Mpps]".format(table["reference"]["title"]), + "{0} Stdev [Mpps]".format(table["reference"]["title"]), + "{0} Throughput [Mpps]".format(table["compare"]["title"]), + "{0} Stdev [Mpps]".format(table["compare"]["title"]), + "Delta [%]"] + header_str = ",".join(header) + "\n" + except (AttributeError, KeyError) as err: + logging.error("The model is invalid, missing parameter: {0}". + format(err)) + return + + # Create a list of available SOAK test results: + tbl_dict = dict() + for job, builds in table["compare"]["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + if tst_data["type"] == "SOAK": + tst_name_mod = tst_name.replace("-soak", "") + if tbl_dict.get(tst_name_mod, None) is None: + tbl_dict[tst_name_mod] = { + "name": tst_name_mod, + "ref-data": list(), + "cmp-data": list() + } + try: + tbl_dict[tst_name_mod]["cmp-data"].append( + tst_data["throughput"]["LOWER"]) + except (KeyError, TypeError): + pass + tests_lst = tbl_dict.keys() + + # Add corresponding NDR test results: + for job, builds in table["reference"]["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + tst_name_mod = tst_name.replace("-ndrpdr", "").\ + replace("-mrr", "") + if tst_name_mod in tests_lst: + try: + if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"): + if table["include-tests"] == "MRR": + result = tst_data["result"]["receive-rate"].avg + elif table["include-tests"] == "PDR": + result = tst_data["throughput"]["PDR"]["LOWER"] + elif table["include-tests"] == "NDR": + result = tst_data["throughput"]["NDR"]["LOWER"] + else: + result = None + if result is not None: + tbl_dict[tst_name_mod]["ref-data"].append( + result) + except (KeyError, TypeError): + continue + + tbl_lst = list() + for tst_name in tbl_dict.keys(): + item = [tbl_dict[tst_name]["name"], ] + data_t = tbl_dict[tst_name]["ref-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + data_t = tbl_dict[tst_name]["cmp-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + if item[-4] is not None and item[-2] is not None and item[-4] != 0: + item.append(int(relative_change(float(item[-4]), float(item[-2])))) + if len(item) == len(header): + tbl_lst.append(item) + + # Sort the table according to the relative change + tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) + + # Generate csv tables: + csv_file = "{0}.csv".format(table["output-file"]) + with open(csv_file, "w") as file_handler: + file_handler.write(header_str) + for test in tbl_lst: + file_handler.write(",".join([str(item) for item in test]) + "\n") + + convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) + + def table_performance_trending_dashboard(table, input_data): """Generate the table(s) with algorithm: table_performance_trending_dashboard diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py index dfed6b3e62..238145b70f 100644 --- a/resources/tools/presentation/pal.py +++ b/resources/tools/presentation/pal.py @@ -94,48 +94,48 @@ def main(): return 1 ret_code = 1 - try: - env = Environment(spec.environment, args.force) - env.set_environment() - - prepare_static_content(spec) - - data = InputData(spec) - data.download_and_parse_data(repeat=1) - - generate_tables(spec, data) - generate_plots(spec, data) - generate_files(spec, data) - - if spec.output["output"] == "report": - generate_report(args.release, spec, args.week) - logging.info("Successfully finished.") - elif spec.output["output"] == "CPTA": - sys.stdout.write(generate_cpta(spec, data)) - try: - alert = Alerting(spec) - alert.generate_alerts() - except AlertingError as err: - logging.warning(repr(err)) - logging.info("Successfully finished.") - ret_code = 0 - - except AlertingError as err: - logging.critical("Finished with an alerting error.") - logging.critical(repr(err)) - except PresentationError as err: - logging.critical("Finished with an PAL error.") - logging.critical(repr(err)) - except (KeyError, ValueError) as err: - logging.critical("Finished with an error.") - logging.critical(repr(err)) - except Exception as err: - logging.critical("Finished with an unexpected error.") - logging.critical(repr(err)) - finally: - if spec is not None: - clean_environment(spec.environment) - return ret_code + # try: + env = Environment(spec.environment, args.force) + env.set_environment() + + prepare_static_content(spec) + + data = InputData(spec) + data.download_and_parse_data(repeat=1) + + generate_tables(spec, data) + generate_plots(spec, data) + generate_files(spec, data) + + if spec.output["output"] == "report": + generate_report(args.release, spec, args.week) + logging.info("Successfully finished.") + elif spec.output["output"] == "CPTA": + sys.stdout.write(generate_cpta(spec, data)) + try: + alert = Alerting(spec) + alert.generate_alerts() + except AlertingError as err: + logging.warning(repr(err)) + logging.info("Successfully finished.") + ret_code = 0 + + # except AlertingError as err: + # logging.critical("Finished with an alerting error.") + # logging.critical(repr(err)) + # except PresentationError as err: + # logging.critical("Finished with an PAL error.") + # logging.critical(repr(err)) + # except (KeyError, ValueError) as err: + # logging.critical("Finished with an error.") + # logging.critical(repr(err)) + # except Exception as err: + # logging.critical("Finished with an unexpected error.") + # logging.critical(repr(err)) + # finally: + # if spec is not None: + # clean_environment(spec.environment) + # return ret_code if __name__ == '__main__': diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 9abdf08a1f..0875d9a045 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -587,6 +587,26 @@ - "vpp-performance-changes-2n-skx-ref" - "vpp-performance-changes-2n-skx-cmp" + vpp-soak-vs-ndr-2n-skx-ref: + csit-vpp-perf-verify-1904-2n-skx: + - 12 # NDRPDR sel + - 14 # NDRPDR sel + - 16 # NDRPDR sel + - 17 # NDRPDR sel + - 18 # NDRPDR sel + - 21 # NDRPDR sel + - 27 # NDRPDR sel + - 30 # NDRPDR sel + - 31 # NDRPDR sel + - 32 # NDRPDR sel + vpp-soak-vs-ndr-2n-skx-cmp: + csit-vpp-perf-verify-1904-2n-skx: + - 70 # SOAK + - 71 # SOAK + vpp-soak-vs-ndr-2n-skx: + - "vpp-soak-vs-ndr-2n-skx-ref" + - "vpp-soak-vs-ndr-2n-skx-cmp" + vpp-performance-changes-3n-hsw-mrr-h1: csit-vpp-perf-verify-1810-3n-hsw: - 9 # MRR sel @@ -1625,6 +1645,54 @@ ################################################################################ ### T A B L E S ### ################################################################################ + +# VPP SOAK vs NDR +- type: "table" + title: "VPP SOAK vs NDR 2n-skx" + algorithm: "table_soak_vs_ndr" + output-file: "{DIR[STATIC,VPP]}/soak-vs-ndr-2n-skx" + reference: + title: "NDR" + data: "vpp-soak-vs-ndr-2n-skx-ref" + compare: + title: "SOAK" + data: "vpp-soak-vs-ndr-2n-skx-cmp" + data: "vpp-soak-vs-ndr-2n-skx" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "('NDRPDR' or 'SOAK')" + parameters: + - "throughput" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # VPP Compare NICs 3n-hsw 1t1c ndr - type: "table" -- 2.16.6