X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fbash%2Ffunction%2Fcommon.sh;h=e89bae9b3853beb5c38d1bad4b9a1f208f47034b;hp=879602674a84bdc63056520a00f1a1029c548fab;hb=7c1753296301b237c19d5c2fbf8f9102fde8d3c1;hpb=5e6145a4260ffce1c302e94b9b241851f90838e1 diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index 879602674a..e89bae9b38 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -33,6 +33,7 @@ function activate_docker_topology () { # - TOPOLOGIES - Available topologies. # - NODENESS - Node multiplicity of desired testbed. # - FLAVOR - Node flavor string, usually describing the processor. + # - IMAGE_VER_FILE - Name of file that contains the image version. # Variables set: # - WORKING_TOPOLOGY - Path to topology file. @@ -40,7 +41,7 @@ function activate_docker_topology () { die "Source failed!" } - device_image="$(< ${CSIT_DIR}/VPP_DEVICE_IMAGE)" + device_image="$(< ${CSIT_DIR}/${IMAGE_VER_FILE})" case_text="${NODENESS}_${FLAVOR}" case "${case_text}" in "1n_skx") @@ -503,29 +504,35 @@ function get_test_tag_string () { } -function reserve_testbed () { +function reserve_and_cleanup_testbed () { set -exuo pipefail # Reserve physical testbed, perform cleanup, register trap to unreserve. + # When cleanup fails, remove from topologies and keep retrying + # until all topologies are removed. # # Variables read: # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on. # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script. # Variables set: + # - TOPOLOGIES - Array of paths to topologies, with failed cleanups removed. # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. # Functions called: # - die - Print to stderr and exit. # Traps registered: # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}. - while true; do + while [[ ${TOPOLOGIES[@]} ]]; do for topo in "${TOPOLOGIES[@]}"; do set +e python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}" result="$?" set -e if [[ "${result}" == "0" ]]; then + # Trap unreservation before cleanup check, + # so multiple jobs showing failed cleanup improve chances + # of humans to notice and fix. WORKING_TOPOLOGY="${topo}" echo "Reserved: ${WORKING_TOPOLOGY}" trap "untrap_and_unreserve_testbed" EXIT || { @@ -535,9 +542,28 @@ function reserve_testbed () { } die "Trap attempt failed, unreserve succeeded. Aborting." } - cleanup_topo || { - die "Testbed cleanup failed." - } + # Cleanup check. + set +e + cleanup_topo + result="$?" + set -e + if [[ "${result}" == "0" ]]; then + break + fi + warn "Testbed cleanup failed: ${topo}" + untrap_and_unreserve_testbed "Fail of unreserve after cleanup." + # WORKING_TOPOLOGY is now empty again. + # Build new topology array. + # TOPOLOGIES=("${TOPOLOGIES[@]/$topo}") + # does not really work, see: + # https://stackoverflow.com/questions/16860877/remove-an-element-from-a-bash-array + new_topologies=() + for item in "${TOPOLOGIES[@]}"; do + if [[ "${item}" != "${topo}" ]]; then + new_topologies+=("${item}") + fi + done + TOPOLOGIES=("${new_topologies[@]}") break fi done @@ -554,6 +580,11 @@ function reserve_testbed () { echo "Sleeping ${sleep_time}" sleep "${sleep_time}" || die "Sleep failed." done + if [[ ${TOPOLOGIES[@]} ]]; then + echo "Reservation and cleanup successful." + else + die "Run out of operational testbeds!" + fi } @@ -580,6 +611,14 @@ function run_pybot () { pybot "${all_options[@]}" "${GENERATED_DIR}/tests/" PYBOT_EXIT_STATUS="$?" set -e + + # Generate INFO level output_info.xml for post-processing. + all_options=("--loglevel" "INFO") + all_options+=("--log" "none") + all_options+=("--report" "none") + all_options+=("--output" "${ARCHIVE_DIR}/output_info.xml") + all_options+=("${ARCHIVE_DIR}/output.xml") + rebot "${all_options[@]}" popd || die "Change directory operation failed." } @@ -650,14 +689,13 @@ function select_tags () { # Blacklisting certain tags per topology. case "${TEST_CODE}" in - *"3n-hsw"*) - test_tag_array+=("!drv_avf") - ;; *"2n-skx"*) test_tag_array+=("!ipsechw") ;; *"3n-skx"*) test_tag_array+=("!ipsechw") + # Not enough nic_intel-xxv710 to support double link tests. + test_tag_array+=("!3_node_double_link_topoANDnic_intel-xxv710") ;; *"3n-tsh"*) test_tag_array+=("!ipsechw") @@ -666,9 +704,21 @@ function select_tags () { test_tag_array+=("!vhost") test_tag_array+=("!vts") ;; + *"3n-hsw"*) + # TODO: Introduce NOIOMMU version of AVF tests. + # TODO: Make (both) AVF tests work on Haswell, + # or document why (some of) it is not possible. + # https://github.com/FDio/vpp/blob/master/src/plugins/avf/README.md + test_tag_array+=("!drv_avf") + # All cards have access to QAT. But only one card (xl710) + # resides in same NUMA as QAT. Other cards must go over QPI + # which we do not want to even run. + test_tag_array+=("!ipsechwNOTnic_intel-xl710") + ;; *) # Default to 3n-hsw due to compatibility. test_tag_array+=("!drv_avf") + test_tag_array+=("!ipsechwNOTnic_intel-xl710") ;; esac @@ -680,10 +730,15 @@ function select_tags () { # We will prefix with perftest to prevent running other tests # (e.g. Functional). prefix="perftestAND" + set +x if [[ "${TEST_CODE}" == "vpp-"* ]]; then # Automatic prefixing for VPP jobs to limit the NIC used and # traffic evaluation to MRR. - prefix="${prefix}mrrAND${DEFAULT_NIC}AND" + if [[ "${TEST_TAG_STRING-}" == *"nic_"* ]]; then + prefix="${prefix}mrrAND" + else + prefix="${prefix}mrrAND${DEFAULT_NIC}AND" + fi fi for tag in "${test_tag_array[@]}"; do if [[ "${tag}" == "!"* ]]; then @@ -695,6 +750,7 @@ function select_tags () { TAGS+=("${prefix}${tag}") fi done + set -x } @@ -743,6 +799,36 @@ function select_vpp_device_tags () { done } +function select_os () { + + set -exuo pipefail + + # Variables set: + # - VPP_VER_FILE - Name of File in CSIT dir containing vpp stable version. + # - IMAGE_VER_FILE - Name of File in CSIT dir containing the image name. + # - PKG_SUFFIX - Suffix of OS package file name, "rpm" or "deb." + + os_id=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') || { + die "Get OS release failed." + } + + case "${os_id}" in + "ubuntu"*) + IMAGE_VER_FILE="VPP_DEVICE_IMAGE_UBUNTU" + VPP_VER_FILE="VPP_STABLE_VER_UBUNTU_BIONIC" + PKG_SUFFIX="deb" + ;; + "centos"*) + IMAGE_VER_FILE="VPP_DEVICE_IMAGE_CENTOS" + VPP_VER_FILE="VPP_STABLE_VER_CENTOS" + PKG_SUFFIX="rpm" + ;; + *) + die "Unable to identify distro or os from ${OS}" + ;; + esac +} + function select_topology () { @@ -761,46 +847,29 @@ function select_topology () { case_text="${NODENESS}_${FLAVOR}" case "${case_text}" in + # TODO: Move tags to "# Blacklisting certain tags per topology" section. "1n_vbox") - TOPOLOGIES=( - "${TOPOLOGIES_DIR}/vpp_device.template" - ) + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template ) TOPOLOGIES_TAGS="2_node_single_link_topo" ;; "1n_skx") - TOPOLOGIES=( - "${TOPOLOGIES_DIR}/vpp_device.template" - ) + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template ) TOPOLOGIES_TAGS="2_node_single_link_topo" ;; "2n_skx") - TOPOLOGIES=( - "${TOPOLOGIES_DIR}/lf_2n_skx_testbed21.yaml" - #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed22.yaml" - "${TOPOLOGIES_DIR}/lf_2n_skx_testbed23.yaml" - "${TOPOLOGIES_DIR}/lf_2n_skx_testbed24.yaml" - ) + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_skx*.yaml ) TOPOLOGIES_TAGS="2_node_*_link_topo" ;; "3n_skx") - TOPOLOGIES=( - "${TOPOLOGIES_DIR}/lf_3n_skx_testbed31.yaml" - "${TOPOLOGIES_DIR}/lf_3n_skx_testbed32.yaml" - ) + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_skx*.yaml ) TOPOLOGIES_TAGS="3_node_*_link_topo" ;; "3n_hsw") - TOPOLOGIES=( - "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml" - "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml" - "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml" - ) + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_hsw*.yaml ) TOPOLOGIES_TAGS="3_node_single_link_topo" ;; "3n_tsh") - TOPOLOGIES=( - "${TOPOLOGIES_DIR}/lf_3n_tsh_testbed33.yaml" - ) + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_tsh*.yaml ) TOPOLOGIES_TAGS="3_node_*_link_topo" ;; *)