From 0634490c0557d94856f6061c323cf6a2592f294b Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Thu, 17 Aug 2023 14:17:33 +0200 Subject: [PATCH] Add ability to parse more kinds of test results Previously, only BMRR results were recongnized. Now also lower bounds for PDR (optionally NDR) and soak are recongnized. This code expects all tests are of the same type, e.g. when both MRR and NDRPDR tests are run, only MRR result will get parsed. If test or parsing fails, generate fake data based on overall pass or fail, so at least passrate of unknown tests can be compared in theory. Currently affects only per-patch job (vpp-csit-verify-perf-*), but is useful mainly for the upcoming bisect job. + Do not force MRR test type in vpp-csit jobs. - Some test results are still not recognized (e.g. hoststack). + Do not exit per-patch job early on robot failure. + Only changes that introduce a failure (not present in parent) get -1. + The same is true also for introducing unrecognized test results. - The fake values from passrate can be misleading. + Add default nic tag only if NIC tag is missing. + In all jobs, not only in vpp-csit ones. + Do not add NIC tags for device jobs. - No job supports NDR parsing yet. + Can be enabled in future from ci-management side. Change-Id: Iee904116d1ffed69aec7e31821c67d8447f49ebe Signed-off-by: Vratko Polak --- resources/libraries/bash/entry/per_patch_perf.sh | 2 - resources/libraries/bash/function/common.sh | 14 +- resources/libraries/bash/function/per_patch.sh | 219 +++++++++++++++++++++-- 3 files changed, 209 insertions(+), 26 deletions(-) diff --git a/resources/libraries/bash/entry/per_patch_perf.sh b/resources/libraries/bash/entry/per_patch_perf.sh index 4a756d2c25..b1854992e2 100644 --- a/resources/libraries/bash/entry/per_patch_perf.sh +++ b/resources/libraries/bash/entry/per_patch_perf.sh @@ -74,14 +74,12 @@ for ((iter=0; iter "${out_file}" || { - die "Some parsing grep command has failed." - } + in_file="${rel_dir}/output.xml" || die + out_file="${rel_dir}/results.txt" || die + echo "Parsing ${in_file} putting results into ${out_file}" || die + # Frst attempt: (B)MRR. + if parse_results_mrr "${in_file}" "${out_file}"; then + return 0 + fi + # BMRR parsing failed. Attempt PDR/NDR. + if parse_results_ndrpdr "${in_file}" "${out_file}"; then + return 0 + fi + # PDR/NDR parsing failed. Attempt soak. + if parse_results_soak "${in_file}" "${out_file}"; then + return 0 + fi + # Soak parsing failed. + # Probably not a perf test at all (or a failed one), + # but we can still bisect by passrate. + parse_results_passrate "${in_file}" "${out_file}" || die +} + + +function parse_results_mrr () { + + # Parse MRR test message(s) into JSON-readable output. + # + # Return non-zero if parsing fails. + # + # Arguments: + # - ${1} - Path to (existing) input file. Required. + # - ${2} - Path to (overwritten if exists) output file. Required. + # Files read: + # - output.xml - The input file from argument location. + # Files updated: + # - results.txt - (Re)created, in argument location. + # Functions called: + # - die - Print to stderr and exit, defined in common.sh + + set -exuo pipefail + + in_file="${1}" || die "Two arguments needed." + out_file="${2}" || die "Two arguments needed." + pattern='Maximum Receive Rate trial results in .*' || die + pattern+=' per second: .*\]' || die + # RC of the following line is returned. + grep -o "${pattern}" "${in_file}" | grep -o '\[.*\]' > "${out_file}" +} + + +function parse_results_ndrpdr () { + + # Parse NDRPDR test message(s) for PDR_LOWER, into JSON-readable output. + # + # Return non-zero if parsing fails. + # Parse for PDR, unless environment variable says NDR. + # + # Arguments: + # - ${1} - Path to (existing) input file. Required. + # - ${2} - Path to (overwritten if exists) output file. Required. + # Variables read: + # - FDIO_CSIT_PERF_PARSE_NDR - If defined and "yes", parse for NDR, not PDR. + # Files read: + # - output.xml - The input file from argument location. + # Files updated: + # - results.txt - (Re)created, in argument location. + # Functions called: + # - die - Print to stderr and exit, defined in common.sh + + set -exuo pipefail + + in_file="${1}" || die "Two arguments needed." + out_file="${2}" || die "Two arguments needed." + if [[ "${FDIO_CSIT_PERF_PARSE_NDR:-no}" == "yes" ]]; then + pattern1="Arguments: [ '\\nNDR_LOWER: " || die + else + pattern1="Arguments: [ '\\nPDR_LOWER: " || die + fi + # Adapted from https://superuser.com/a/377084 + pattern2='(?<=R: ).*(?= pps)' || die + if fgrep "${pattern1}" "${in_file}" | grep -Po "${pattern2}" >> "${out_file}" + then + # Add bracket https://www.shellhacks.com/sed-awk-add-end-beginning-line/ + sed -i 's/.*/[&]/' "${out_file}" + # Returns nonzero if fails. + return "$?" + fi + # Maybe it was CPS instead of pps? + pattern2='(?<=R: ).*(?= CPS)' || die + if fgrep "${pattern1}" "${in_file}" | grep -Po "${pattern2}" >> "${out_file}" + then + # Add bracket https://www.shellhacks.com/sed-awk-add-end-beginning-line/ + sed -i 's/.*/[&]/' "${out_file}" + # Returns nonzero if fails. + return "$?" + else + return 1 + fi +} + + +function parse_results_passrate () { + + # Create fake values for failed tests. + # + # This function always passes (or dies). + # + # A non-zero but small value is chosen for failed run, to distinguish from + # real nonzero perf (which are big in general) and real zero values. + # A medium sized value is chosen for a passed run. + # This way bisect can search for breakages and fixes in device tests. + # At least in theory, as device tests are bootstrapped too differently. + # + # The fake value is repeated according to BMRR multiplicity, + # because a single value can be lost in high stdev data. + # (And it does not hurt for single value outputs such as NDR.) + # + # TODO: Count number of tests and generate fake results for every one. + # Currently that would interfere with test retry logic. + # + # Arguments: + # - ${1} - Path to (existing) input file. Required. + # - ${2} - Path to (overwritten if exists) output file. Required. + # Variables read: + # - CSIT_PERF_TRIAL_MULTIPLICITY - To create fake results of this length. + # Files read: + # - output.xml - The input file from argument location. + # Files updated: + # - results.txt - (Re)created, in argument location. + # Functions called: + # - die - Print to stderr and exit, defined in common.sh + + set -exuo pipefail + + in_file="${1}" || die "Two arguments needed." + out_file="${2}" || die "Two arguments needed." + # The last status is the top level (global) robot status. + # It only passes if there were no (critical) test failures. + if fgrep ' "${out_file}" || die +} + + +function parse_results_soak () { + + # Parse soak test message(s) for lower bound, into JSON-readable output. + # + # Return non-zero if parsing fails. + # + # Arguments: + # - ${1} - Path to (existing) input file. Required. + # - ${2} - Path to (overwritten if exists) output file. Required. + # Files read: + # - output.xml - The input file from argument location. + # Files updated: + # - results.txt - (Re)created, in argument location. + # Functions called: + # - die - Print to stderr and exit, defined in common.sh + + set -exuo pipefail + + in_file="${1}" || die "Two arguments needed." + out_file="${2}" || die "Two arguments needed." + pattern1='PLRsearch lower bound: .*, .*<' || die + # Adapted from https://superuser.com/a/377084 + pattern2='(?<=: ).*(?= pps)' || die + if grep "${pattern1}" "${in_file}" | grep -Po "${pattern2}" >> "${out_file}" + then + # Add bracket https://www.shellhacks.com/sed-awk-add-end-beginning-line/ + sed -i 's/.*/[&]/' "${out_file}" + # Returns nonzero if fails. + else + return 1 + fi } -- 2.16.6