X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fbash%2Ffunction%2Fcommon.sh;h=5deab8aa40fa6b9e8f69687e452ceb1fe62af54d;hp=02c65c82c201f0559a5336123e623af471094f5a;hb=cac8f76ce1f9085e9eede19b250f4a4cb9b199e0;hpb=e5c9717d241c005b4546ada80b5b85a40cde2061 diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index 02c65c82c2..5deab8aa40 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -445,6 +445,14 @@ function get_test_code () { NODENESS="3n" FLAVOR="skx" ;; + *"2n-dnv"*) + NODENESS="2n" + FLAVOR="dnv" + ;; + *"3n-dnv"*) + NODENESS="3n" + FLAVOR="dnv" + ;; *"3n-tsh"*) NODENESS="3n" FLAVOR="tsh" @@ -491,6 +499,7 @@ function get_test_tag_string () { comment="${comment/perftest-3n/perftest}" comment="${comment/perftest-hsw/perftest}" comment="${comment/perftest-skx/perftest}" + comment="${comment/perftest-dnv/perftest}" comment="${comment/perftest-tsh/perftest}" tag_string="$(echo "${comment}" \ | grep -oE '(perftest$|perftest[[:space:]].+$)' || true)" @@ -504,29 +513,42 @@ function get_test_tag_string () { } -function reserve_testbed () { +function reserve_and_cleanup_testbed () { set -exuo pipefail # Reserve physical testbed, perform cleanup, register trap to unreserve. + # When cleanup fails, remove from topologies and keep retrying + # until all topologies are removed. # # Variables read: # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on. # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script. + # - BUILD_TAG - Any string suitable as filename, identifying + # test run executing this function. May be unset. + # - BUILD_URL - Any string suitable as URL, identifying + # test run executing this function. May be unset. # Variables set: + # - TOPOLOGIES - Array of paths to topologies, with failed cleanups removed. # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. # Functions called: # - die - Print to stderr and exit. # Traps registered: # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}. - while true; do + while [[ ${TOPOLOGIES[@]} ]]; do for topo in "${TOPOLOGIES[@]}"; do set +e - python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}" + scrpt="${PYTHON_SCRIPTS_DIR}/topo_reservation.py" + opts=("-t" "${topo}" "-r" "${BUILD_TAG:-Unknown}") + opts+=("-u" "${BUILD_URL:-Unknown}") + python "${scrpt}" "${opts[@]}" result="$?" set -e if [[ "${result}" == "0" ]]; then + # Trap unreservation before cleanup check, + # so multiple jobs showing failed cleanup improve chances + # of humans to notice and fix. WORKING_TOPOLOGY="${topo}" echo "Reserved: ${WORKING_TOPOLOGY}" trap "untrap_and_unreserve_testbed" EXIT || { @@ -536,9 +558,28 @@ function reserve_testbed () { } die "Trap attempt failed, unreserve succeeded. Aborting." } - cleanup_topo || { - die "Testbed cleanup failed." - } + # Cleanup check. + set +e + cleanup_topo + result="$?" + set -e + if [[ "${result}" == "0" ]]; then + break + fi + warn "Testbed cleanup failed: ${topo}" + untrap_and_unreserve_testbed "Fail of unreserve after cleanup." + # WORKING_TOPOLOGY is now empty again. + # Build new topology array. + # TOPOLOGIES=("${TOPOLOGIES[@]/$topo}") + # does not really work, see: + # https://stackoverflow.com/questions/16860877/remove-an-element-from-a-bash-array + new_topologies=() + for item in "${TOPOLOGIES[@]}"; do + if [[ "${item}" != "${topo}" ]]; then + new_topologies+=("${item}") + fi + done + TOPOLOGIES=("${new_topologies[@]}") break fi done @@ -555,6 +596,11 @@ function reserve_testbed () { echo "Sleeping ${sleep_time}" sleep "${sleep_time}" || die "Sleep failed." done + if [[ ${TOPOLOGIES[@]} ]]; then + echo "Reservation and cleanup successful." + else + die "Run out of operational testbeds!" + fi } @@ -622,6 +668,9 @@ function select_tags () { # Select default NIC case "${TEST_CODE}" in + *"3n-dnv"* | *"2n-dnv"*) + DEFAULT_NIC='nic_intel-x553' + ;; *"3n-tsh"*) DEFAULT_NIC='nic_intel-x520-da2' ;; @@ -659,10 +708,6 @@ function select_tags () { # Blacklisting certain tags per topology. case "${TEST_CODE}" in - *"3n-hsw"*) - test_tag_array+=("!drv_avf") - test_tag_array+=("!ipsechwNOTnic_intel-xl710") - ;; *"2n-skx"*) test_tag_array+=("!ipsechw") ;; @@ -671,6 +716,19 @@ function select_tags () { # Not enough nic_intel-xxv710 to support double link tests. test_tag_array+=("!3_node_double_link_topoANDnic_intel-xxv710") ;; + *"2n-dnv"*) + test_tag_array+=("!ipsechw") + test_tag_array+=("!memif") + test_tag_array+=("!srv6_proxy") + test_tag_array+=("!vhost") + test_tag_array+=("!vts") + ;; + *"3n-dnv"*) + test_tag_array+=("!memif") + test_tag_array+=("!srv6_proxy") + test_tag_array+=("!vhost") + test_tag_array+=("!vts") + ;; *"3n-tsh"*) test_tag_array+=("!ipsechw") test_tag_array+=("!memif") @@ -678,9 +736,21 @@ function select_tags () { test_tag_array+=("!vhost") test_tag_array+=("!vts") ;; + *"3n-hsw"*) + # TODO: Introduce NOIOMMU version of AVF tests. + # TODO: Make (both) AVF tests work on Haswell, + # or document why (some of) it is not possible. + # https://github.com/FDio/vpp/blob/master/src/plugins/avf/README.md + test_tag_array+=("!drv_avf") + # All cards have access to QAT. But only one card (xl710) + # resides in same NUMA as QAT. Other cards must go over QPI + # which we do not want to even run. + test_tag_array+=("!ipsechwNOTnic_intel-xl710") + ;; *) # Default to 3n-hsw due to compatibility. test_tag_array+=("!drv_avf") + test_tag_array+=("!ipsechwNOTnic_intel-xl710") ;; esac @@ -809,6 +879,7 @@ function select_topology () { case_text="${NODENESS}_${FLAVOR}" case "${case_text}" in + # TODO: Move tags to "# Blacklisting certain tags per topology" section. "1n_vbox") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template ) TOPOLOGIES_TAGS="2_node_single_link_topo" @@ -825,6 +896,14 @@ function select_topology () { TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_skx*.yaml ) TOPOLOGIES_TAGS="3_node_*_link_topo" ;; + "2n_dnv") + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_dnv*.yaml ) + TOPOLOGIES_TAGS="2_node_single_link_topo" + ;; + "3n_dnv") + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_dnv*.yaml ) + TOPOLOGIES_TAGS="3_node_single_link_topo" + ;; "3n_hsw") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_hsw*.yaml ) TOPOLOGIES_TAGS="3_node_single_link_topo"