X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fbash%2Ffunction%2Fcommon.sh;h=5deab8aa40fa6b9e8f69687e452ceb1fe62af54d;hp=399b449fa0cde16ac3a778d2356952e9847c792a;hb=cac8f76ce1f9085e9eede19b250f4a4cb9b199e0;hpb=ea6e8be86d47b288df5e11ca5515f6bda71dbd08 diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index 399b449fa0..5deab8aa40 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -445,6 +445,14 @@ function get_test_code () { NODENESS="3n" FLAVOR="skx" ;; + *"2n-dnv"*) + NODENESS="2n" + FLAVOR="dnv" + ;; + *"3n-dnv"*) + NODENESS="3n" + FLAVOR="dnv" + ;; *"3n-tsh"*) NODENESS="3n" FLAVOR="tsh" @@ -491,6 +499,7 @@ function get_test_tag_string () { comment="${comment/perftest-3n/perftest}" comment="${comment/perftest-hsw/perftest}" comment="${comment/perftest-skx/perftest}" + comment="${comment/perftest-dnv/perftest}" comment="${comment/perftest-tsh/perftest}" tag_string="$(echo "${comment}" \ | grep -oE '(perftest$|perftest[[:space:]].+$)' || true)" @@ -504,29 +513,42 @@ function get_test_tag_string () { } -function reserve_testbed () { +function reserve_and_cleanup_testbed () { set -exuo pipefail # Reserve physical testbed, perform cleanup, register trap to unreserve. + # When cleanup fails, remove from topologies and keep retrying + # until all topologies are removed. # # Variables read: # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on. # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script. + # - BUILD_TAG - Any string suitable as filename, identifying + # test run executing this function. May be unset. + # - BUILD_URL - Any string suitable as URL, identifying + # test run executing this function. May be unset. # Variables set: + # - TOPOLOGIES - Array of paths to topologies, with failed cleanups removed. # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. # Functions called: # - die - Print to stderr and exit. # Traps registered: # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}. - while true; do + while [[ ${TOPOLOGIES[@]} ]]; do for topo in "${TOPOLOGIES[@]}"; do set +e - python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}" + scrpt="${PYTHON_SCRIPTS_DIR}/topo_reservation.py" + opts=("-t" "${topo}" "-r" "${BUILD_TAG:-Unknown}") + opts+=("-u" "${BUILD_URL:-Unknown}") + python "${scrpt}" "${opts[@]}" result="$?" set -e if [[ "${result}" == "0" ]]; then + # Trap unreservation before cleanup check, + # so multiple jobs showing failed cleanup improve chances + # of humans to notice and fix. WORKING_TOPOLOGY="${topo}" echo "Reserved: ${WORKING_TOPOLOGY}" trap "untrap_and_unreserve_testbed" EXIT || { @@ -536,9 +558,28 @@ function reserve_testbed () { } die "Trap attempt failed, unreserve succeeded. Aborting." } - cleanup_topo || { - die "Testbed cleanup failed." - } + # Cleanup check. + set +e + cleanup_topo + result="$?" + set -e + if [[ "${result}" == "0" ]]; then + break + fi + warn "Testbed cleanup failed: ${topo}" + untrap_and_unreserve_testbed "Fail of unreserve after cleanup." + # WORKING_TOPOLOGY is now empty again. + # Build new topology array. + # TOPOLOGIES=("${TOPOLOGIES[@]/$topo}") + # does not really work, see: + # https://stackoverflow.com/questions/16860877/remove-an-element-from-a-bash-array + new_topologies=() + for item in "${TOPOLOGIES[@]}"; do + if [[ "${item}" != "${topo}" ]]; then + new_topologies+=("${item}") + fi + done + TOPOLOGIES=("${new_topologies[@]}") break fi done @@ -555,6 +596,11 @@ function reserve_testbed () { echo "Sleeping ${sleep_time}" sleep "${sleep_time}" || die "Sleep failed." done + if [[ ${TOPOLOGIES[@]} ]]; then + echo "Reservation and cleanup successful." + else + die "Run out of operational testbeds!" + fi } @@ -622,6 +668,9 @@ function select_tags () { # Select default NIC case "${TEST_CODE}" in + *"3n-dnv"* | *"2n-dnv"*) + DEFAULT_NIC='nic_intel-x553' + ;; *"3n-tsh"*) DEFAULT_NIC='nic_intel-x520-da2' ;; @@ -659,10 +708,6 @@ function select_tags () { # Blacklisting certain tags per topology. case "${TEST_CODE}" in - *"3n-hsw"*) - test_tag_array+=("!drv_avf") - test_tag_array+=("!ipsechwNOTnic_intel-xl710") - ;; *"2n-skx"*) test_tag_array+=("!ipsechw") ;; @@ -671,6 +716,19 @@ function select_tags () { # Not enough nic_intel-xxv710 to support double link tests. test_tag_array+=("!3_node_double_link_topoANDnic_intel-xxv710") ;; + *"2n-dnv"*) + test_tag_array+=("!ipsechw") + test_tag_array+=("!memif") + test_tag_array+=("!srv6_proxy") + test_tag_array+=("!vhost") + test_tag_array+=("!vts") + ;; + *"3n-dnv"*) + test_tag_array+=("!memif") + test_tag_array+=("!srv6_proxy") + test_tag_array+=("!vhost") + test_tag_array+=("!vts") + ;; *"3n-tsh"*) test_tag_array+=("!ipsechw") test_tag_array+=("!memif") @@ -678,9 +736,21 @@ function select_tags () { test_tag_array+=("!vhost") test_tag_array+=("!vts") ;; + *"3n-hsw"*) + # TODO: Introduce NOIOMMU version of AVF tests. + # TODO: Make (both) AVF tests work on Haswell, + # or document why (some of) it is not possible. + # https://github.com/FDio/vpp/blob/master/src/plugins/avf/README.md + test_tag_array+=("!drv_avf") + # All cards have access to QAT. But only one card (xl710) + # resides in same NUMA as QAT. Other cards must go over QPI + # which we do not want to even run. + test_tag_array+=("!ipsechwNOTnic_intel-xl710") + ;; *) # Default to 3n-hsw due to compatibility. test_tag_array+=("!drv_avf") + test_tag_array+=("!ipsechwNOTnic_intel-xl710") ;; esac @@ -765,27 +835,29 @@ function select_os () { set -exuo pipefail - # Variables read: - # - OS - os or distro for selecting container image. # Variables set: # - VPP_VER_FILE - Name of File in CSIT dir containing vpp stable version. # - IMAGE_VER_FILE - Name of File in CSIT dir containing the image name. # - PKG_SUFFIX - Suffix of OS package file name, "rpm" or "deb." - case "${OS}" in - "ubuntu"*) - IMAGE_VER_FILE="VPP_DEVICE_IMAGE_UBUNTU" - VPP_VER_FILE="VPP_STABLE_VER_UBUNTU_BIONIC" - PKG_SUFFIX="deb" - ;; - "centos"*) - IMAGE_VER_FILE="VPP_DEVICE_IMAGE_CENTOS" - VPP_VER_FILE="VPP_STABLE_VER_CENTOS" - PKG_SUFFIX="rpm" - ;; - *) - die "Unable to identify distro or os from ${OS}" - ;; + os_id=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') || { + die "Get OS release failed." + } + + case "${os_id}" in + "ubuntu"*) + IMAGE_VER_FILE="VPP_DEVICE_IMAGE_UBUNTU" + VPP_VER_FILE="VPP_STABLE_VER_UBUNTU_BIONIC" + PKG_SUFFIX="deb" + ;; + "centos"*) + IMAGE_VER_FILE="VPP_DEVICE_IMAGE_CENTOS" + VPP_VER_FILE="VPP_STABLE_VER_CENTOS" + PKG_SUFFIX="rpm" + ;; + *) + die "Unable to identify distro or os from ${OS}" + ;; esac } @@ -807,6 +879,7 @@ function select_topology () { case_text="${NODENESS}_${FLAVOR}" case "${case_text}" in + # TODO: Move tags to "# Blacklisting certain tags per topology" section. "1n_vbox") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template ) TOPOLOGIES_TAGS="2_node_single_link_topo" @@ -823,6 +896,14 @@ function select_topology () { TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_skx*.yaml ) TOPOLOGIES_TAGS="3_node_*_link_topo" ;; + "2n_dnv") + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_dnv*.yaml ) + TOPOLOGIES_TAGS="2_node_single_link_topo" + ;; + "3n_dnv") + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_dnv*.yaml ) + TOPOLOGIES_TAGS="3_node_single_link_topo" + ;; "3n_hsw") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_hsw*.yaml ) TOPOLOGIES_TAGS="3_node_single_link_topo"