X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fbash%2Ffunction%2Fcommon.sh;h=e89bae9b3853beb5c38d1bad4b9a1f208f47034b;hp=399b449fa0cde16ac3a778d2356952e9847c792a;hb=7c1753296301b237c19d5c2fbf8f9102fde8d3c1;hpb=ea6e8be86d47b288df5e11ca5515f6bda71dbd08

diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index 399b449fa0..e89bae9b38 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -504,29 +504,35 @@ function get_test_tag_string () {
 }
 
 
-function reserve_testbed () {
+function reserve_and_cleanup_testbed () {
 
     set -exuo pipefail
 
     # Reserve physical testbed, perform cleanup, register trap to unreserve.
+    # When cleanup fails, remove from topologies and keep retrying
+    # until all topologies are removed.
     #
     # Variables read:
     # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on.
     # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
     # Variables set:
+    # - TOPOLOGIES - Array of paths to topologies, with failed cleanups removed.
     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
     # Functions called:
     # - die - Print to stderr and exit.
     # Traps registered:
     # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
 
-    while true; do
+    while [[ ${TOPOLOGIES[@]} ]]; do
         for topo in "${TOPOLOGIES[@]}"; do
             set +e
             python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}"
             result="$?"
             set -e
             if [[ "${result}" == "0" ]]; then
+                # Trap unreservation before cleanup check,
+                # so multiple jobs showing failed cleanup improve chances
+                # of humans to notice and fix.
                 WORKING_TOPOLOGY="${topo}"
                 echo "Reserved: ${WORKING_TOPOLOGY}"
                 trap "untrap_and_unreserve_testbed" EXIT || {
@@ -536,9 +542,28 @@ function reserve_testbed () {
                     }
                     die "Trap attempt failed, unreserve succeeded. Aborting."
                 }
-                cleanup_topo || {
-                    die "Testbed cleanup failed."
-                }
+                # Cleanup check.
+                set +e
+                cleanup_topo
+                result="$?"
+                set -e
+                if [[ "${result}" == "0" ]]; then
+                    break
+                fi
+                warn "Testbed cleanup failed: ${topo}"
+                untrap_and_unreserve_testbed "Fail of unreserve after cleanup."
+                # WORKING_TOPOLOGY is now empty again.
+                # Build new topology array.
+                #   TOPOLOGIES=("${TOPOLOGIES[@]/$topo}")
+                # does not really work, see:
+                # https://stackoverflow.com/questions/16860877/remove-an-element-from-a-bash-array
+                new_topologies=()
+                for item in "${TOPOLOGIES[@]}"; do
+                    if [[ "${item}" != "${topo}" ]]; then
+                        new_topologies+=("${item}")
+                    fi
+                done
+                TOPOLOGIES=("${new_topologies[@]}")
                 break
             fi
         done
@@ -555,6 +580,11 @@ function reserve_testbed () {
         echo "Sleeping ${sleep_time}"
         sleep "${sleep_time}" || die "Sleep failed."
     done
+    if [[ ${TOPOLOGIES[@]} ]]; then
+        echo "Reservation and cleanup successful."
+    else
+        die "Run out of operational testbeds!"
+    fi
 }
 
 
@@ -659,10 +689,6 @@ function select_tags () {
 
     # Blacklisting certain tags per topology.
     case "${TEST_CODE}" in
-        *"3n-hsw"*)
-            test_tag_array+=("!drv_avf")
-            test_tag_array+=("!ipsechwNOTnic_intel-xl710")
-            ;;
         *"2n-skx"*)
             test_tag_array+=("!ipsechw")
             ;;
@@ -678,9 +704,21 @@ function select_tags () {
             test_tag_array+=("!vhost")
             test_tag_array+=("!vts")
             ;;
+        *"3n-hsw"*)
+            # TODO: Introduce NOIOMMU version of AVF tests.
+            # TODO: Make (both) AVF tests work on Haswell,
+            # or document why (some of) it is not possible.
+            # https://github.com/FDio/vpp/blob/master/src/plugins/avf/README.md
+            test_tag_array+=("!drv_avf")
+            # All cards have access to QAT. But only one card (xl710)
+            # resides in same NUMA as QAT. Other cards must go over QPI
+            # which we do not want to even run.
+            test_tag_array+=("!ipsechwNOTnic_intel-xl710")
+            ;;
         *)
             # Default to 3n-hsw due to compatibility.
             test_tag_array+=("!drv_avf")
+            test_tag_array+=("!ipsechwNOTnic_intel-xl710")
             ;;
     esac
 
@@ -765,27 +803,29 @@ function select_os () {
 
     set -exuo pipefail
 
-    # Variables read:
-    # - OS - os or distro for selecting container image.
     # Variables set:
     # - VPP_VER_FILE - Name of File in CSIT dir containing vpp stable version.
     # - IMAGE_VER_FILE - Name of File in CSIT dir containing the image name.
     # - PKG_SUFFIX - Suffix of OS package file name, "rpm" or "deb."
 
-    case "${OS}" in
-    "ubuntu"*)
-        IMAGE_VER_FILE="VPP_DEVICE_IMAGE_UBUNTU"
-        VPP_VER_FILE="VPP_STABLE_VER_UBUNTU_BIONIC"
-        PKG_SUFFIX="deb"
-        ;;
-    "centos"*)
-        IMAGE_VER_FILE="VPP_DEVICE_IMAGE_CENTOS"
-        VPP_VER_FILE="VPP_STABLE_VER_CENTOS"
-        PKG_SUFFIX="rpm"
-        ;;
-    *)
-        die "Unable to identify distro or os from ${OS}"
-        ;;
+    os_id=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') || {
+        die "Get OS release failed."
+    }
+
+    case "${os_id}" in
+        "ubuntu"*)
+            IMAGE_VER_FILE="VPP_DEVICE_IMAGE_UBUNTU"
+            VPP_VER_FILE="VPP_STABLE_VER_UBUNTU_BIONIC"
+            PKG_SUFFIX="deb"
+            ;;
+        "centos"*)
+            IMAGE_VER_FILE="VPP_DEVICE_IMAGE_CENTOS"
+            VPP_VER_FILE="VPP_STABLE_VER_CENTOS"
+            PKG_SUFFIX="rpm"
+            ;;
+        *)
+            die "Unable to identify distro or os from ${OS}"
+            ;;
     esac
 }
 
@@ -807,6 +847,7 @@ function select_topology () {
 
     case_text="${NODENESS}_${FLAVOR}"
     case "${case_text}" in
+        # TODO: Move tags to "# Blacklisting certain tags per topology" section.
         "1n_vbox")
             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template )
             TOPOLOGIES_TAGS="2_node_single_link_topo"