# - TOPOLOGIES - Available topologies.
# - NODENESS - Node multiplicity of desired testbed.
# - FLAVOR - Node flavor string, usually describing the processor.
+ # - IMAGE_VER_FILE - Name of file that contains the image version.
# Variables set:
# - WORKING_TOPOLOGY - Path to topology file.
die "Source failed!"
}
- device_image="$(< ${CSIT_DIR}/VPP_DEVICE_IMAGE)"
+ device_image="$(< ${CSIT_DIR}/${IMAGE_VER_FILE})"
case_text="${NODENESS}_${FLAVOR}"
case "${case_text}" in
"1n_skx")
NODENESS="3n"
FLAVOR="skx"
;;
+ *"2n-dnv"*)
+ NODENESS="2n"
+ FLAVOR="dnv"
+ ;;
+ *"3n-dnv"*)
+ NODENESS="3n"
+ FLAVOR="dnv"
+ ;;
*"3n-tsh"*)
NODENESS="3n"
FLAVOR="tsh"
comment="${comment/perftest-3n/perftest}"
comment="${comment/perftest-hsw/perftest}"
comment="${comment/perftest-skx/perftest}"
+ comment="${comment/perftest-dnv/perftest}"
comment="${comment/perftest-tsh/perftest}"
tag_string="$(echo "${comment}" \
| grep -oE '(perftest$|perftest[[:space:]].+$)' || true)"
}
-function reserve_testbed () {
+function reserve_and_cleanup_testbed () {
set -exuo pipefail
# Reserve physical testbed, perform cleanup, register trap to unreserve.
+ # When cleanup fails, remove from topologies and keep retrying
+ # until all topologies are removed.
#
# Variables read:
# - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on.
# - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
+ # - BUILD_TAG - Any string suitable as filename, identifying
+ # test run executing this function. May be unset.
+ # - BUILD_URL - Any string suitable as URL, identifying
+ # test run executing this function. May be unset.
# Variables set:
+ # - TOPOLOGIES - Array of paths to topologies, with failed cleanups removed.
# - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
# Functions called:
# - die - Print to stderr and exit.
# Traps registered:
# - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
- while true; do
+ while [[ ${TOPOLOGIES[@]} ]]; do
for topo in "${TOPOLOGIES[@]}"; do
set +e
- python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}"
+ scrpt="${PYTHON_SCRIPTS_DIR}/topo_reservation.py"
+ opts=("-t" "${topo}" "-r" "${BUILD_TAG:-Unknown}")
+ opts+=("-u" "${BUILD_URL:-Unknown}")
+ python "${scrpt}" "${opts[@]}"
result="$?"
set -e
if [[ "${result}" == "0" ]]; then
+ # Trap unreservation before cleanup check,
+ # so multiple jobs showing failed cleanup improve chances
+ # of humans to notice and fix.
WORKING_TOPOLOGY="${topo}"
echo "Reserved: ${WORKING_TOPOLOGY}"
trap "untrap_and_unreserve_testbed" EXIT || {
}
die "Trap attempt failed, unreserve succeeded. Aborting."
}
- cleanup_topo || {
- die "Testbed cleanup failed."
- }
+ # Cleanup check.
+ set +e
+ cleanup_topo
+ result="$?"
+ set -e
+ if [[ "${result}" == "0" ]]; then
+ break
+ fi
+ warn "Testbed cleanup failed: ${topo}"
+ untrap_and_unreserve_testbed "Fail of unreserve after cleanup."
+ # WORKING_TOPOLOGY is now empty again.
+ # Build new topology array.
+ # TOPOLOGIES=("${TOPOLOGIES[@]/$topo}")
+ # does not really work, see:
+ # https://stackoverflow.com/questions/16860877/remove-an-element-from-a-bash-array
+ new_topologies=()
+ for item in "${TOPOLOGIES[@]}"; do
+ if [[ "${item}" != "${topo}" ]]; then
+ new_topologies+=("${item}")
+ fi
+ done
+ TOPOLOGIES=("${new_topologies[@]}")
break
fi
done
echo "Sleeping ${sleep_time}"
sleep "${sleep_time}" || die "Sleep failed."
done
+ if [[ ${TOPOLOGIES[@]} ]]; then
+ echo "Reservation and cleanup successful."
+ else
+ die "Run out of operational testbeds!"
+ fi
}
# Select default NIC
case "${TEST_CODE}" in
+ *"3n-dnv"* | *"2n-dnv"*)
+ DEFAULT_NIC='nic_intel-x553'
+ ;;
*"3n-tsh"*)
DEFAULT_NIC='nic_intel-x520-da2'
;;
# Blacklisting certain tags per topology.
case "${TEST_CODE}" in
- *"3n-hsw"*)
- test_tag_array+=("!drv_avf")
- ;;
*"2n-skx"*)
test_tag_array+=("!ipsechw")
;;
# Not enough nic_intel-xxv710 to support double link tests.
test_tag_array+=("!3_node_double_link_topoANDnic_intel-xxv710")
;;
+ *"2n-dnv"*)
+ test_tag_array+=("!ipsechw")
+ test_tag_array+=("!memif")
+ test_tag_array+=("!srv6_proxy")
+ test_tag_array+=("!vhost")
+ test_tag_array+=("!vts")
+ ;;
+ *"3n-dnv"*)
+ test_tag_array+=("!memif")
+ test_tag_array+=("!srv6_proxy")
+ test_tag_array+=("!vhost")
+ test_tag_array+=("!vts")
+ ;;
*"3n-tsh"*)
test_tag_array+=("!ipsechw")
test_tag_array+=("!memif")
test_tag_array+=("!vhost")
test_tag_array+=("!vts")
;;
+ *"3n-hsw"*)
+ # TODO: Introduce NOIOMMU version of AVF tests.
+ # TODO: Make (both) AVF tests work on Haswell,
+ # or document why (some of) it is not possible.
+ # https://github.com/FDio/vpp/blob/master/src/plugins/avf/README.md
+ test_tag_array+=("!drv_avf")
+ # All cards have access to QAT. But only one card (xl710)
+ # resides in same NUMA as QAT. Other cards must go over QPI
+ # which we do not want to even run.
+ test_tag_array+=("!ipsechwNOTnic_intel-xl710")
+ ;;
*)
# Default to 3n-hsw due to compatibility.
test_tag_array+=("!drv_avf")
+ test_tag_array+=("!ipsechwNOTnic_intel-xl710")
;;
esac
# We will prefix with perftest to prevent running other tests
# (e.g. Functional).
prefix="perftestAND"
+ set +x
if [[ "${TEST_CODE}" == "vpp-"* ]]; then
# Automatic prefixing for VPP jobs to limit the NIC used and
# traffic evaluation to MRR.
- prefix="${prefix}mrrAND${DEFAULT_NIC}AND"
+ if [[ "${TEST_TAG_STRING-}" == *"nic_"* ]]; then
+ prefix="${prefix}mrrAND"
+ else
+ prefix="${prefix}mrrAND${DEFAULT_NIC}AND"
+ fi
fi
for tag in "${test_tag_array[@]}"; do
if [[ "${tag}" == "!"* ]]; then
TAGS+=("${prefix}${tag}")
fi
done
+ set -x
}
done
}
+function select_os () {
+
+ set -exuo pipefail
+
+ # Variables set:
+ # - VPP_VER_FILE - Name of File in CSIT dir containing vpp stable version.
+ # - IMAGE_VER_FILE - Name of File in CSIT dir containing the image name.
+ # - PKG_SUFFIX - Suffix of OS package file name, "rpm" or "deb."
+
+ os_id=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') || {
+ die "Get OS release failed."
+ }
+
+ case "${os_id}" in
+ "ubuntu"*)
+ IMAGE_VER_FILE="VPP_DEVICE_IMAGE_UBUNTU"
+ VPP_VER_FILE="VPP_STABLE_VER_UBUNTU_BIONIC"
+ PKG_SUFFIX="deb"
+ ;;
+ "centos"*)
+ IMAGE_VER_FILE="VPP_DEVICE_IMAGE_CENTOS"
+ VPP_VER_FILE="VPP_STABLE_VER_CENTOS"
+ PKG_SUFFIX="rpm"
+ ;;
+ *)
+ die "Unable to identify distro or os from ${OS}"
+ ;;
+ esac
+}
+
function select_topology () {
case_text="${NODENESS}_${FLAVOR}"
case "${case_text}" in
+ # TODO: Move tags to "# Blacklisting certain tags per topology" section.
"1n_vbox")
- TOPOLOGIES=(
- "${TOPOLOGIES_DIR}/vpp_device.template"
- )
+ TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template )
TOPOLOGIES_TAGS="2_node_single_link_topo"
;;
"1n_skx")
- TOPOLOGIES=(
- "${TOPOLOGIES_DIR}/vpp_device.template"
- )
+ TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template )
TOPOLOGIES_TAGS="2_node_single_link_topo"
;;
"2n_skx")
- TOPOLOGIES=(
- "${TOPOLOGIES_DIR}/lf_2n_skx_testbed21.yaml"
- #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed22.yaml"
- "${TOPOLOGIES_DIR}/lf_2n_skx_testbed23.yaml"
- "${TOPOLOGIES_DIR}/lf_2n_skx_testbed24.yaml"
- )
+ TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_skx*.yaml )
TOPOLOGIES_TAGS="2_node_*_link_topo"
;;
"3n_skx")
- TOPOLOGIES=(
- "${TOPOLOGIES_DIR}/lf_3n_skx_testbed31.yaml"
- "${TOPOLOGIES_DIR}/lf_3n_skx_testbed32.yaml"
- )
+ TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_skx*.yaml )
TOPOLOGIES_TAGS="3_node_*_link_topo"
;;
+ "2n_dnv")
+ TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_dnv*.yaml )
+ TOPOLOGIES_TAGS="2_node_single_link_topo"
+ ;;
+ "3n_dnv")
+ TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_dnv*.yaml )
+ TOPOLOGIES_TAGS="3_node_single_link_topo"
+ ;;
"3n_hsw")
- TOPOLOGIES=(
- "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml"
- "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml"
- "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml"
- )
+ TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_hsw*.yaml )
TOPOLOGIES_TAGS="3_node_single_link_topo"
;;
"3n_tsh")
- TOPOLOGIES=(
- "${TOPOLOGIES_DIR}/lf_3n_tsh_testbed33.yaml"
- )
+ TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_tsh*.yaml )
TOPOLOGIES_TAGS="3_node_*_link_topo"
;;
*)
Code Review / csit.git / blobdiff