resources/libraries/bash/function/common.sh

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 set -exuo pipefail
  15
  16 # This library defines functions used by multiple entry scripts.
  17 # Keep functions ordered alphabetically, please.
  18
  19 # TODO: Add a link to bash style guide.
  20 # TODO: Consider putting every die into a {} block,
  21 #   the code might become more readable (but longer).
  22
  23
  24 function activate_virtualenv () {
  25
  26     set -exuo pipefail
  27
  28     # Arguments:
  29     # - ${1} - Non-empty path to existing directory for creating virtualenv in.
  30     # Variables read:
  31     # - CSIT_DIR - Path to existing root of local CSIT git repository.
  32     # Variables set:
  33     # - ENV_DIR - Path to the created virtualenv subdirectory.
  34     # Variables exported:
  35     # - PYTHONPATH - CSIT_DIR, as CSIT Python scripts usually need this.
  36     # Functions called:
  37     # - die - Print to stderr and exit.
  38
  39     # TODO: Do we really need to have ENV_DIR available as a global variable?
  40
  41     if [[ "${1-}" == "" ]]; then
  42         die "Root location of virtualenv to create is not specified."
  43     fi
  44     ENV_DIR="${1}/env"
  45     rm -rf "${ENV_DIR}" || die "Failed to clean previous virtualenv."
  46
  47     pip install --upgrade virtualenv || {
  48         die "Virtualenv package install failed."
  49     }
  50     virtualenv "${ENV_DIR}" || {
  51         die "Virtualenv creation failed."
  52     }
  53     set +u
  54     source "${ENV_DIR}/bin/activate" || die "Virtualenv activation failed."
  55     set -u
  56     pip install -r "${CSIT_DIR}/requirements.txt" || {
  57         die "CSIT requirements installation failed."
  58     }
  59
  60     # Most CSIT Python scripts assume PYTHONPATH is set and exported.
  61     export PYTHONPATH="${CSIT_DIR}" || die "Export failed."
  62 }
  63
  64
  65 function check_download_dir () {
  66
  67     set -exuo pipefail
  68
  69     # Fail if there are no files visible in ${DOWNLOAD_DIR}.
  70     # TODO: Do we need this as a function, if it is (almost) a one-liner?
  71     #
  72     # Variables read:
  73     # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
  74     # Directories read:
  75     # - ${DOWNLOAD_DIR} - Has to be non-empty to proceed.
  76     # Functions called:
  77     # - die - Print to stderr and exit.
  78
  79     if [[ ! "$(ls -A "${DOWNLOAD_DIR}")" ]]; then
  80         die "No artifacts downloaded!"
  81     fi
  82 }
  83
  84
  85 function common_dirs () {
  86
  87     set -exuo pipefail
  88
  89     # Variables set:
  90     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
  91     # - CSIT_DIR - Path to existing root of local CSIT git repository.
  92     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
  93     # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources".
  94     # - TOOLS_DIR - Path to existing resources subdirectory "tools".
  95     # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".
  96     # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive".
  97     # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
  98     # Functions called:
  99     # - die - Print to stderr and exit.
 100
 101     BASH_FUNCTION_DIR="$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")" || {
 102         die "Some error during localizing this source directory."
 103     }
 104     # Current working directory could be in a different repo, e.g. VPP.
 105     pushd "${BASH_FUNCTION_DIR}" || die "Pushd failed"
 106     CSIT_DIR="$(readlink -e "$(git rev-parse --show-toplevel)")" || {
 107         die "Readlink or git rev-parse failed."
 108     }
 109     popd || die "Popd failed."
 110     TOPOLOGIES_DIR="$(readlink -e "${CSIT_DIR}/topologies/available")" || {
 111         die "Readlink failed."
 112     }
 113     RESOURCES_DIR="$(readlink -e "${CSIT_DIR}/resources")" || {
 114         die "Readlink failed."
 115     }
 116     TOOLS_DIR="$(readlink -e "${RESOURCES_DIR}/tools")" || {
 117         die "Readlink failed."
 118     }
 119     PYTHON_SCRIPTS_DIR="$(readlink -e "${TOOLS_DIR}/scripts")" || {
 120         die "Readlink failed."
 121     }
 122
 123     ARCHIVE_DIR="$(readlink -f "${CSIT_DIR}/archive")" || {
 124         die "Readlink failed."
 125     }
 126     mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed."
 127     DOWNLOAD_DIR="$(readlink -f "${CSIT_DIR}/download_dir")" || {
 128         die "Readlink failed."
 129     }
 130     mkdir -p "${DOWNLOAD_DIR}" || die "Mkdir failed."
 131 }
 132
 133
 134 function compose_pybot_arguments () {
 135
 136     set -exuo pipefail
 137
 138     # Variables read:
 139     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 140     # - DUT - CSIT test/ subdirectory, set while processing tags.
 141     # - TAGS - Array variable holding selected tag boolean expressions.
 142     # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology.
 143     # Variables set:
 144     # - PYBOT_ARGS - String holding part of all arguments for pybot.
 145     # - EXPANDED_TAGS - Array of strings pybot arguments compiled from tags.
 146
 147     # No explicit check needed with "set -u".
 148     PYBOT_ARGS=("--loglevel" "TRACE" "--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}")
 149     PYBOT_ARGS+=("--suite" "tests.${DUT}.perf")
 150
 151     EXPANDED_TAGS=()
 152     for tag in "${TAGS[@]}"; do
 153         if [[ ${tag} == "!"* ]]; then
 154             EXPANDED_TAGS+=("--exclude" "${tag#$"!"}")
 155         else
 156             EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}")
 157         fi
 158     done
 159 }
 160
 161
 162 function copy_archives () {
 163
 164     set -exuo pipefail
 165
 166     # Variables read:
 167     # - WORKSPACE - Jenkins workspace, copy only if the value is not empty.
 168     #   Can be unset, then it speeds up manual testing.
 169     # - ARCHIVE_DIR - Path to directory with content to be copied.
 170     # Directories updated:
 171     # - ${WORKSPACE}/archives/ - Created if does not exist.
 172     #   Content of ${ARCHIVE_DIR}/ is copied here.
 173     # Functions called:
 174     # - die - Print to stderr and exit.
 175
 176     # We will create additional archive if workspace variable is set.
 177     # This way if script is running in jenkins all will be
 178     # automatically archived to logs.fd.io.
 179     if [[ -n "${WORKSPACE-}" ]]; then
 180         mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed."
 181         cp -r "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed."
 182     fi
 183 }
 184
 185
 186 function die () {
 187     # Print the message to standard error end exit with error code specified
 188     # by the second argument.
 189     #
 190     # Hardcoded values:
 191     # - The default error message.
 192     # Arguments:
 193     # - ${1} - The whole error message, be sure to quote. Optional
 194     # - ${2} - the code to exit with, default: 1.
 195
 196     set -x
 197     set +eu
 198     warn "${1:-Unspecified run-time error occurred!}"
 199     exit "${2:-1}"
 200 }
 201
 202
 203 function die_on_pybot_error () {
 204
 205     set -exuo pipefail
 206
 207     # Source this fragment if you want to abort on any failed test case.
 208     #
 209     # Variables read:
 210     # - PYBOT_EXIT_STATUS - Set by a pybot running fragment.
 211     # Functions called:
 212     # - die - Print to stderr and exit.
 213
 214     if [[ "${PYBOT_EXIT_STATUS}" != "0" ]]; then
 215         die "${PYBOT_EXIT_STATUS}" "Test failures are present!"
 216     fi
 217 }
 218
 219
 220 function get_test_code () {
 221
 222     set -exuo pipefail
 223
 224     # Arguments:
 225     # - ${1} - Optional, argument of entry script (or empty as unset).
 226     #   Test code value to override job name from environment.
 227     # Variables read:
 228     # - JOB_NAME - String affecting test selection, default if not argument.
 229     # Variables set:
 230     # - TEST_CODE - The test selection string from environment or argument.
 231     # - NODENESS - Node multiplicity of desired testbed.
 232     # - FLAVOR - Node flavor string, usually describing the processor.
 233
 234     TEST_CODE="${1-}" || die "Reading optional argument failed, somehow."
 235     if [[ -z "${TEST_CODE}" ]]; then
 236         TEST_CODE="${JOB_NAME-}" || die "Reading job name failed, somehow."
 237     fi
 238
 239     case "${TEST_CODE}" in
 240         *"2n-skx"*)
 241             NODENESS="2n"
 242             FLAVOR="skx"
 243             ;;
 244         *"3n-skx"*)
 245             NODENESS="3n"
 246             FLAVOR="skx"
 247             ;;
 248         *)
 249             # Fallback to 3-node Haswell by default (backward compatibility)
 250             NODENESS="3n"
 251             FLAVOR="hsw"
 252             ;;
 253     esac
 254 }
 255
 256
 257 function get_test_tag_string () {
 258
 259     set -exuo pipefail
 260
 261     # Variables read:
 262     # - GERRIT_EVENT_TYPE - Event type set by gerrit, can be unset.
 263     # - GERRIT_EVENT_COMMENT_TEXT - Comment text, read for "comment-added" type.
 264     # Variables set:
 265     # - TEST_TAG_STRING - The string following "perftest" in gerrit comment,
 266     #   or empty.
 267
 268     # TODO: ci-management scripts no longer need to perform this.
 269
 270     trigger=""
 271     if [[ "${GERRIT_EVENT_TYPE-}" == "comment-added" ]]; then
 272         # On parsing error, ${trigger} stays empty.
 273         trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \
 274             | grep -oE '(perftest$|perftest[[:space:]].+$)')" || true
 275     fi
 276     # Set test tags as string.
 277     TEST_TAG_STRING="${trigger#$"perftest"}"
 278 }
 279
 280
 281 function reserve_testbed () {
 282
 283     set -exuo pipefail
 284
 285     # Reserve physical testbed, perform cleanup, register trap to unreserve.
 286     #
 287     # Variables read:
 288     # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on.
 289     # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
 290     # Variables set:
 291     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 292     # Functions called:
 293     # - die - Print to stderr and exit.
 294     # Traps registered:
 295     # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
 296
 297     while true; do
 298         for topo in "${TOPOLOGIES[@]}"; do
 299             set +e
 300             python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}"
 301             result="$?"
 302             set -e
 303             if [[ "${result}" == "0" ]]; then
 304                 WORKING_TOPOLOGY="${topo}"
 305                 echo "Reserved: ${WORKING_TOPOLOGY}"
 306                 trap "untrap_and_unreserve_testbed" EXIT || {
 307                     message="TRAP ATTEMPT AND UNRESERVE FAILED, FIX MANUALLY."
 308                     untrap_and_unreserve_testbed "${message}" || {
 309                         die "Teardown should have died, not failed."
 310                     }
 311                     die "Trap attempt failed, unreserve succeeded. Aborting."
 312                 }
 313                 python "${PYTHON_SCRIPTS_DIR}/topo_cleanup.py" -t "${topo}" || {
 314                     die "Testbed cleanup failed."
 315                 }
 316                 break
 317             fi
 318         done
 319
 320         if [[ -n "${WORKING_TOPOLOGY-}" ]]; then
 321             # Exit the infinite while loop if we made a reservation.
 322             break
 323         fi
 324
 325         # Wait ~3minutes before next try.
 326         sleep_time="$[ ( $RANDOM % 20 ) + 180 ]s" || {
 327             die "Sleep time calculation failed."
 328         }
 329         echo "Sleeping ${sleep_time}"
 330         sleep "${sleep_time}" || die "Sleep failed."
 331     done
 332 }
 333
 334
 335 function run_pybot () {
 336
 337     set -exuo pipefail
 338
 339     # Currently, VPP-1361 causes occasional test failures.
 340     # If real result is more important than time, we can retry few times.
 341     # TODO: We should be retrying on test case level instead.
 342
 343     # Arguments:
 344     # - ${1} - Optional number of pybot invocations to try to avoid failures.
 345     #   Default: 1.
 346     # Variables read:
 347     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 348     # - ARCHIVE_DIR - Path to store robot result files in.
 349     # - PYBOT_ARGS, EXPANDED_TAGS - See compose_pybot_arguments.sh
 350     # Variables set:
 351     # - PYBOT_EXIT_STATUS - Exit status of most recent pybot invocation.
 352     # Functions called:
 353     # - die - Print to stderr and exit.
 354
 355     # Set ${tries} as an integer variable, to fail on non-numeric input.
 356     local -i "tries" || die "Setting type of variable failed."
 357     tries="${1:-1}" || die "Argument evaluation failed."
 358     all_options=("--outputdir" "${ARCHIVE_DIR}" "${PYBOT_ARGS[@]}")
 359     all_options+=("${EXPANDED_TAGS[@]}")
 360
 361     while true; do
 362         if [[ "${tries}" -le 0 ]]; then
 363             break
 364         else
 365             tries="$((${tries} - 1))"
 366         fi
 367         pushd "${CSIT_DIR}" || die "Change directory operation failed."
 368         set +e
 369         # TODO: Make robot tests not require "$(pwd)" == "${CSIT_DIR}".
 370         pybot "${all_options[@]}" "${CSIT_DIR}/tests/"
 371         PYBOT_EXIT_STATUS="$?"
 372         set -e
 373         popd || die "Change directory operation failed."
 374         if [[ "${PYBOT_EXIT_STATUS}" == "0" ]]; then
 375             break
 376         fi
 377     done
 378 }
 379
 380
 381 function select_tags () {
 382
 383     set -exuo pipefail
 384
 385     # Variables read:
 386     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 387     # - TEST_CODE - String affecting test selection, usually jenkins job name.
 388     # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
 389     #   Can be unset.
 390     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
 391     # Variables set:
 392     # - TAGS - Array of processed tag boolean expressions.
 393
 394     # NIC SELECTION
 395     # All topologies NICs
 396     available=$(grep -hoPR "model: \K.*" "${TOPOLOGIES_DIR}"/* | sort -u)
 397     # Selected topology NICs
 398     reserved=$(grep -hoPR "model: \K.*" "${WORKING_TOPOLOGY}" | sort -u)
 399     # All topologies NICs - Selected topology NICs
 400     exclude_nics=($(comm -13 <(echo "${reserved}") <(echo "${available}")))
 401
 402     case "${TEST_CODE}" in
 403         # Select specific performance tests based on jenkins job type variable.
 404         *"ndrpdr-weekly"* )
 405             test_tag_array=("ndrpdrAND64bAND1c"
 406                             "ndrpdrAND78bAND1c")
 407             ;;
 408         *"mrr-daily"* | *"mrr-weekly"* )
 409             test_tag_array=(# vic
 410                             "mrrANDnic_cisco-vic-1227AND64b"
 411                             "mrrANDnic_cisco-vic-1385AND64b"
 412                             # memif
 413                             "mrrANDmemifAND64b"
 414                             "mrrANDmemifANDimix"
 415                             # crypto
 416                             "mrrANDipsecAND64b"
 417                             # ip4 base
 418                             "mrrANDip4baseAND64b"
 419                             # ip4 scale FIB 2M
 420                             "mrrANDip4fwdANDfib_2mAND64b"
 421                             # ip4 scale FIB 200k
 422                             "mrrANDip4fwdANDfib_200kANDnic_intel-*710AND64b"
 423                             # ip4 scale FIB 20k
 424                             "mrrANDip4fwdANDfib_20kANDnic_intel-*710AND64b"
 425                             # ip4 scale ACL
 426                             "mrrANDip4fwdANDacl1AND10k_flowsAND64b"
 427                             "mrrANDip4fwdANDacl50AND10k_flowsAND64b"
 428                             # ip4 scale NAT44
 429                             "mrrANDip4fwdANDnat44ANDbaseAND64b"
 430                             "mrrANDip4fwdANDnat44ANDsrc_user_4000AND64b"
 431                             # ip4 features
 432                             "mrrANDip4fwdANDfeatureANDnic_intel-*710AND64b"
 433                             # ip4 tunnels
 434                             "mrrANDip4fwdANDencapANDip6unrlayANDip4ovrlayANDnic_intel-x520-da2AND64b"
 435                             "mrrANDip4fwdANDencapANDnic_intel-*710AND64b"
 436                             "mrrANDl2ovrlayANDencapANDnic_intel-*710AND64b"
 437                             # ip6 base
 438                             "mrrANDip6baseANDethAND78b"
 439                             # ip6 features
 440                             "mrrANDip6fwdANDfeatureANDnic_intel-*710AND78b"
 441                             # ip6 scale FIB 2M
 442                             "mrrANDip6fwdANDfib_2mANDnic_intel-*710AND78b"
 443                             # ip6 scale FIB 200k
 444                             "mrrANDip6fwdANDfib_200kANDnic_intel-*710AND78b"
 445                             # ip6 scale FIB 20k
 446                             "mrrANDip6fwdANDfib_20kANDnic_intel-*710AND78b"
 447                             # ip6 tunnels
 448                             "mrrANDip6fwdANDencapANDnic_intel-x520-da2AND78b"
 449                             # l2xc base
 450                             "mrrANDl2xcfwdANDbaseAND64b"
 451                             # l2xc scale ACL
 452                             "mrrANDl2xcANDacl1AND10k_flowsAND64b"
 453                             "mrrANDl2xcANDacl50AND10k_flowsAND64b"
 454                             # l2xc scale FIB 2M
 455                             "mrrANDl2xcANDfib_2mAND64b"
 456                             # l2xc scale FIB 200k
 457                             "mrrANDl2xcANDfib_200kANDnic_intel-*710AND64b"
 458                             # l2xc scale FIB 20k
 459                             "mrrANDl2xcANDfib_20kANDnic_intel-*710AND64b"
 460                             # l2bd base
 461                             "mrrANDl2bdmaclrnANDbaseAND64b"
 462                             # l2bd scale ACL
 463                             "mrrANDl2bdmaclrnANDacl1AND10k_flowsAND64b"
 464                             "mrrANDl2bdmaclrnANDacl50AND10k_flowsAND64b"
 465                             # l2bd scale FIB 2M
 466                             "mrrANDl2bdmaclrnANDfib_2mAND64b"
 467                             # l2bd scale FIB 200k
 468                             "mrrANDl2bdmaclrnANDfib_200kANDnic_intel-*710AND64b"
 469                             # l2bd scale FIB 20k
 470                             "mrrANDl2bdmaclrnANDfib_20kANDnic_intel-*710AND64b"
 471                             # l2 patch base
 472                             "mrrANDl2patchAND64b"
 473                             # srv6
 474                             "mrrANDsrv6ANDnic_intel-x520-da2AND78b"
 475                             # vts
 476                             "mrrANDvtsANDnic_intel-x520-da2AND114b"
 477                             # vm vhost l2xc base
 478                             "mrrANDvhostANDl2xcfwdANDbaseAND64b"
 479                             "mrrANDvhostANDl2xcfwdANDbaseANDimix"
 480                             # vm vhost l2bd base
 481                             "mrrANDvhostANDl2bdmaclrnANDbaseAND64b"
 482                             "mrrANDvhostANDl2bdmaclrnANDbaseANDimix"
 483                             # vm vhost ip4 base
 484                             "mrrANDvhostANDip4baseAND64b"
 485                             "mrrANDvhostANDip4baseANDimix"
 486                             # Exclude
 487                             "!vhost_256ANDnic_intel-x520-da2"
 488                             "!vhostANDnic_intel-xl710"
 489                             "!cfs_opt"
 490                             "!lbond_dpdk")
 491             ;;
 492         * )
 493             if [[ -z "${TEST_TAG_STRING-}" ]]; then
 494                 # If nothing is specified, we will run pre-selected tests by
 495                 # following tags.
 496                 test_tag_array=("mrrANDnic_intel-x710AND1cAND64bANDip4base"
 497                                 "mrrANDnic_intel-x710AND1cAND78bANDip6base"
 498                                 "mrrANDnic_intel-x710AND1cAND64bANDl2bdbase"
 499                                 "mrrANDnic_intel-x710AND1cAND64bANDl2xcbase"
 500                                 "!dot1q")
 501             else
 502                 # If trigger contains tags, split them into array.
 503                 test_tag_array=(${TEST_TAG_STRING//:/ })
 504             fi
 505             ;;
 506     esac
 507
 508     # Blacklisting certain tags per topology.
 509     case "${TEST_CODE}" in
 510         *"3n-hsw"*)
 511             test_tag_array+=("!drv_avf")
 512             ;;
 513         *"2n-skx"*)
 514             test_tag_array+=("!ipsechw")
 515             ;;
 516         *"3n-skx"*)
 517             test_tag_array+=("!ipsechw")
 518             ;;
 519         *)
 520             # Default to 3n-hsw due to compatibility.
 521             test_tag_array+=("!drv_avf")
 522             ;;
 523     esac
 524
 525     # We will add excluded NICs.
 526     test_tag_array+=("${exclude_nics[@]/#/!NIC_}")
 527
 528     TAGS=()
 529
 530     # We will prefix with perftest to prevent running other tests
 531     # (e.g. Functional).
 532     prefix="perftestAND"
 533     if [[ "${TEST_CODE}" == "vpp-"* ]]; then
 534         # Automatic prefixing for VPP jobs to limit the NIC used and
 535         # traffic evaluation to MRR.
 536         prefix="${prefix}mrrANDnic_intel-x710AND"
 537     fi
 538     for tag in "${test_tag_array[@]}"; do
 539         if [[ ${tag} == "!"* ]]; then
 540             # Exclude tags are not prefixed.
 541             TAGS+=("${tag}")
 542         else
 543             TAGS+=("${prefix}${tag}")
 544         fi
 545     done
 546 }
 547
 548
 549 function select_topology () {
 550
 551     set -exuo pipefail
 552
 553     # Variables read:
 554     # - NODENESS - Node multiplicity of testbed, either "2n" or "3n".
 555     # - FLAVOR - Node flavor string, currently either "hsw" or "skx".
 556     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 557     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
 558     # Variables set:
 559     # - TOPOLOGIES - Array of paths to suitable topology yaml files.
 560     # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology.
 561     # Functions called:
 562     # - die - Print to stderr and exit.
 563
 564     case_text="${NODENESS}_${FLAVOR}"
 565     case "${case_text}" in
 566         "3n_hsw")
 567             TOPOLOGIES=(
 568                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml"
 569                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml"
 570                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml"
 571                        )
 572             TOPOLOGIES_TAGS="3_node_single_link_topo"
 573             ;;
 574         "2n_skx")
 575             TOPOLOGIES=(
 576                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed21.yaml"
 577                         #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed22.yaml"
 578                         #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed23.yaml"
 579                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed24.yaml"
 580                        )
 581             TOPOLOGIES_TAGS="2_node_*_link_topo"
 582             ;;
 583         "3n_skx")
 584             TOPOLOGIES=(
 585                         "${TOPOLOGIES_DIR}/lf_3n_skx_testbed31.yaml"
 586                         "${TOPOLOGIES_DIR}/lf_3n_skx_testbed32.yaml"
 587                        )
 588             TOPOLOGIES_TAGS="3_node_*_link_topo"
 589             ;;
 590         *)
 591             # No falling back to 3n_hsw default, that should have been done
 592             # by the function which has set NODENESS and FLAVOR.
 593             die "Unknown specification: ${case_text}"
 594     esac
 595
 596     if [[ -z "${TOPOLOGIES-}" ]]; then
 597         die "No applicable topology found!"
 598     fi
 599 }
 600
 601
 602 function untrap_and_unreserve_testbed () {
 603     # Use this as a trap function to ensure testbed does not remain reserved.
 604     # Perhaps call directly before script exit, to free testbed for other jobs.
 605     # This function is smart enough to avoid multiple unreservations (so safe).
 606     # Topo cleanup is executed (call it best practice), ignoring failures.
 607     #
 608     # Hardcoded values:
 609     # - default message to die with if testbed might remain reserved.
 610     # Arguments:
 611     # - ${1} - Message to die with if unreservation fails. Default hardcoded.
 612     # Variables read (by inner function):
 613     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 614     # - PYTHON_SCRIPTS_DIR - Path to directory holding Python scripts.
 615     # Variables written:
 616     # - WORKING_TOPOLOGY - Set to empty string on successful unreservation.
 617     # Trap unregistered:
 618     # - EXIT - Failure to untrap is reported, but ignored otherwise.
 619     # Functions called:
 620     # - die - Print to stderr and exit.
 621
 622     set -xo pipefail
 623     set +eu  # We do not want to exit early in a "teardown" function.
 624     trap - EXIT || echo "Trap deactivation failed, continuing anyway."
 625     wt="${WORKING_TOPOLOGY}"  # Just to avoid too long lines.
 626     if [[ -z "${wt-}" ]]; then
 627         set -eu
 628         echo "Testbed looks unreserved already. Trap removal failed before?"
 629     else
 630         python "${PYTHON_SCRIPTS_DIR}/topo_cleanup.py" -t "${wt}" || true
 631         python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || {
 632             die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2
 633         }
 634         WORKING_TOPOLOGY=""
 635         set -eu
 636     fi
 637 }
 638
 639
 640 function warn () {
 641     # Print the message to standard error.
 642     #
 643     # Arguments:
 644     # - ${@} - The text of the message.
 645
 646     echo "$@" >&2
 647 }