resources/libraries/bash/function/common.sh

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 set -exuo pipefail
  15
  16 # This library defines functions used by multiple entry scripts.
  17 # Keep functions ordered alphabetically, please.
  18
  19 # TODO: Add a link to bash style guide.
  20 # TODO: Consider putting every die into a {} block,
  21 #   the code might become more readable (but longer).
  22
  23
  24 function activate_docker_topology () {
  25     # Create virtual vpp-device topology. Output of the function is topology
  26     # file describing created environment saved to a file.
  27     #
  28     # Variables read:
  29     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
  30     # - TOPOLOGIES - Available topologies.
  31     # - NODENESS - Node multiplicity of desired testbed.
  32     # - FLAVOR - Node flavor string, usually describing the processor.
  33     # Variables set:
  34     # - WORKING_TOPOLOGY - Path to topology file.
  35
  36     set -exuo pipefail
  37
  38     source "${BASH_FUNCTION_DIR}/device.sh" || {
  39         die "Source failed!"
  40     }
  41
  42     device_image="$(< ${CSIT_DIR}/VPP_DEVICE_IMAGE)"
  43     case_text="${NODENESS}_${FLAVOR}"
  44     case "${case_text}" in
  45         "1n_skx")
  46             # We execute reservation over csit-shim-dcr (ssh) which runs sourced
  47             # script's functions. Env variables are read from ssh output
  48             # back to localhost for further processing.
  49             hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3)
  50             ssh="ssh root@${hostname} -p 6022"
  51             run="activate_wrapper ${NODENESS} ${FLAVOR} ${device_image}"
  52             env_vars=$(${ssh} "$(declare -f); ${run}") || {
  53                 die "Topology reservation via shim-dcr failed!"
  54             }
  55             set -a
  56             source <(echo "$env_vars" | grep -v /usr/bin/docker) || {
  57                 die "Source failed!"
  58             }
  59             set +a
  60             ;;
  61         "1n_vbox")
  62             # We execute reservation on localhost. Sourced script automatially
  63             # sets environment variables for further processing.
  64             activate_wrapper "${NODENESS}" "${FLAVOR}" "${device_image}" || die
  65             ;;
  66         *)
  67             die "Unknown specification: ${case_text}!"
  68     esac
  69
  70     trap 'deactivate_docker_topology' EXIT || {
  71          die "Trap attempt failed, please cleanup manually. Aborting!"
  72     }
  73
  74     # Replace all variables in template with those in environment.
  75     source <(echo 'cat <<EOF >topo.yml'; cat ${TOPOLOGIES[0]}; echo EOF;) || {
  76         die "Topology file create failed!"
  77     }
  78
  79     WORKING_TOPOLOGY="/tmp/topology.yaml"
  80     mv topo.yml "${WORKING_TOPOLOGY}" || {
  81         die "Topology move failed!"
  82     }
  83     cat ${WORKING_TOPOLOGY} | grep -v password || {
  84         die "Topology read failed!"
  85     }
  86 }
  87
  88
  89 function activate_virtualenv () {
  90
  91     set -exuo pipefail
  92
  93     # Update virtualenv pip package, delete and create virtualenv directory,
  94     # activate the virtualenv, install requirements, set PYTHONPATH.
  95
  96     # Arguments:
  97     # - ${1} - Path to existing directory for creating virtualenv in.
  98     #          If missing or empty, ${CSIT_DIR} is used.
  99     # - ${2} - Path to requirements file, ${CSIT_DIR}/requirements.txt if empty.
 100     # Variables read:
 101     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 102     # Variables exported:
 103     # - PYTHONPATH - CSIT_DIR, as CSIT Python scripts usually need this.
 104     # Functions called:
 105     # - die - Print to stderr and exit.
 106
 107     # TODO: Do we want the callers to be able to set the env dir name?
 108     # TODO: + In that case, do we want to support env switching?
 109     # TODO:   + In that case we want to make env_dir global.
 110     # TODO: Do we want the callers to override PYTHONPATH loaction?
 111
 112     root_path="${1-$CSIT_DIR}"
 113     env_dir="${root_path}/env"
 114     req_path=${2-$CSIT_DIR/requirements.txt}
 115     rm -rf "${env_dir}" || die "Failed to clean previous virtualenv."
 116     pip install --upgrade virtualenv || {
 117         die "Virtualenv package install failed."
 118     }
 119     virtualenv "${env_dir}" || {
 120         die "Virtualenv creation failed."
 121     }
 122     set +u
 123     source "${env_dir}/bin/activate" || die "Virtualenv activation failed."
 124     set -u
 125     pip install --upgrade -r "${req_path}" || {
 126         die "Requirements installation failed."
 127     }
 128     # Most CSIT Python scripts assume PYTHONPATH is set and exported.
 129     export PYTHONPATH="${root_path}" || die "Export failed."
 130 }
 131
 132
 133 function check_download_dir () {
 134
 135     set -exuo pipefail
 136
 137     # Fail if there are no files visible in ${DOWNLOAD_DIR}.
 138     # TODO: Do we need this as a function, if it is (almost) a one-liner?
 139     #
 140     # Variables read:
 141     # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
 142     # Directories read:
 143     # - ${DOWNLOAD_DIR} - Has to be non-empty to proceed.
 144     # Functions called:
 145     # - die - Print to stderr and exit.
 146
 147     if [[ ! "$(ls -A "${DOWNLOAD_DIR}")" ]]; then
 148         die "No artifacts downloaded!"
 149     fi
 150 }
 151
 152
 153 function cleanup_topo () {
 154
 155     set -exuo pipefail
 156
 157     # Variables read:
 158     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 159     # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
 160
 161     python "${PYTHON_SCRIPTS_DIR}/topo_cleanup.py" -t "${WORKING_TOPOLOGY}"
 162     # Not using "|| die" as some callers might want to ignore errors,
 163     # e.g. in teardowns, such as unreserve.
 164 }
 165
 166
 167 function common_dirs () {
 168
 169     set -exuo pipefail
 170
 171     # Variables set:
 172     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
 173     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 174     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
 175     # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources".
 176     # - TOOLS_DIR - Path to existing resources subdirectory "tools".
 177     # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".
 178     # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive".
 179     # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
 180     # Functions called:
 181     # - die - Print to stderr and exit.
 182
 183     BASH_FUNCTION_DIR="$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")" || {
 184         die "Some error during localizing this source directory."
 185     }
 186     # Current working directory could be in a different repo, e.g. VPP.
 187     pushd "${BASH_FUNCTION_DIR}" || die "Pushd failed"
 188     CSIT_DIR="$(readlink -e "$(git rev-parse --show-toplevel)")" || {
 189         die "Readlink or git rev-parse failed."
 190     }
 191     popd || die "Popd failed."
 192     TOPOLOGIES_DIR="$(readlink -e "${CSIT_DIR}/topologies/available")" || {
 193         die "Readlink failed."
 194     }
 195     RESOURCES_DIR="$(readlink -e "${CSIT_DIR}/resources")" || {
 196         die "Readlink failed."
 197     }
 198     TOOLS_DIR="$(readlink -e "${RESOURCES_DIR}/tools")" || {
 199         die "Readlink failed."
 200     }
 201     PYTHON_SCRIPTS_DIR="$(readlink -e "${TOOLS_DIR}/scripts")" || {
 202         die "Readlink failed."
 203     }
 204
 205     ARCHIVE_DIR="$(readlink -f "${CSIT_DIR}/archive")" || {
 206         die "Readlink failed."
 207     }
 208     mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed."
 209     DOWNLOAD_DIR="$(readlink -f "${CSIT_DIR}/download_dir")" || {
 210         die "Readlink failed."
 211     }
 212     mkdir -p "${DOWNLOAD_DIR}" || die "Mkdir failed."
 213 }
 214
 215
 216 function compose_pybot_arguments () {
 217
 218     set -exuo pipefail
 219
 220     # Variables read:
 221     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 222     # - DUT - CSIT test/ subdirectory, set while processing tags.
 223     # - TAGS - Array variable holding selected tag boolean expressions.
 224     # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology.
 225     # - TEST_CODE - The test selection string from environment or argument.
 226     # Variables set:
 227     # - PYBOT_ARGS - String holding part of all arguments for pybot.
 228     # - EXPANDED_TAGS - Array of strings pybot arguments compiled from tags.
 229
 230     # No explicit check needed with "set -u".
 231     PYBOT_ARGS=("--loglevel" "TRACE")
 232     PYBOT_ARGS+=("--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}")
 233
 234     case "${TEST_CODE}" in
 235         *"device"*)
 236             PYBOT_ARGS+=("--suite" "tests.${DUT}.device")
 237             ;;
 238         *"func"*)
 239             PYBOT_ARGS+=("--suite" "tests.${DUT}.func")
 240             ;;
 241         *"perf"*)
 242             PYBOT_ARGS+=("--suite" "tests.${DUT}.perf")
 243             ;;
 244         *)
 245             die "Unknown specification: ${TEST_CODE}"
 246     esac
 247
 248     EXPANDED_TAGS=()
 249     for tag in "${TAGS[@]}"; do
 250         if [[ ${tag} == "!"* ]]; then
 251             EXPANDED_TAGS+=("--exclude" "${tag#$"!"}")
 252         else
 253             EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}")
 254         fi
 255     done
 256 }
 257
 258
 259 function copy_archives () {
 260
 261     set -exuo pipefail
 262
 263     # Variables read:
 264     # - WORKSPACE - Jenkins workspace, copy only if the value is not empty.
 265     #   Can be unset, then it speeds up manual testing.
 266     # - ARCHIVE_DIR - Path to directory with content to be copied.
 267     # Directories updated:
 268     # - ${WORKSPACE}/archives/ - Created if does not exist.
 269     #   Content of ${ARCHIVE_DIR}/ is copied here.
 270     # Functions called:
 271     # - die - Print to stderr and exit.
 272
 273     # We will create additional archive if workspace variable is set.
 274     # This way if script is running in jenkins all will be
 275     # automatically archived to logs.fd.io.
 276     if [[ -n "${WORKSPACE-}" ]]; then
 277         mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed."
 278         cp -rf "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed."
 279     fi
 280 }
 281
 282
 283 function deactivate_docker_topology () {
 284     # Deactivate virtual vpp-device topology by removing containers.
 285     #
 286     # Variables read:
 287     # - NODENESS - Node multiplicity of desired testbed.
 288     # - FLAVOR - Node flavor string, usually describing the processor.
 289
 290     set -exuo pipefail
 291
 292     case_text="${NODENESS}_${FLAVOR}"
 293     case "${case_text}" in
 294         "1n_skx")
 295             hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3)
 296             ssh="ssh root@${hostname} -p 6022"
 297             env_vars="$(env | grep CSIT_ | tr '\n' ' ' )"
 298             ${ssh} "$(declare -f); deactivate_wrapper ${env_vars}" || {
 299                 die "Topology cleanup via shim-dcr failed!"
 300             }
 301             ;;
 302         "1n_vbox")
 303             enter_mutex || die
 304             clean_environment || {
 305                 die "Topology cleanup locally failed!"
 306             }
 307             exit_mutex || die
 308             ;;
 309         *)
 310             die "Unknown specification: ${case_text}!"
 311     esac
 312 }
 313
 314
 315 function die () {
 316     # Print the message to standard error end exit with error code specified
 317     # by the second argument.
 318     #
 319     # Hardcoded values:
 320     # - The default error message.
 321     # Arguments:
 322     # - ${1} - The whole error message, be sure to quote. Optional
 323     # - ${2} - the code to exit with, default: 1.
 324
 325     set -x
 326     set +eu
 327     warn "${1:-Unspecified run-time error occurred!}"
 328     exit "${2:-1}"
 329 }
 330
 331
 332 function die_on_pybot_error () {
 333
 334     set -exuo pipefail
 335
 336     # Source this fragment if you want to abort on any failed test case.
 337     #
 338     # Variables read:
 339     # - PYBOT_EXIT_STATUS - Set by a pybot running fragment.
 340     # Functions called:
 341     # - die - Print to stderr and exit.
 342
 343     if [[ "${PYBOT_EXIT_STATUS}" != "0" ]]; then
 344         die "${PYBOT_EXIT_STATUS}" "Test failures are present!"
 345     fi
 346 }
 347
 348
 349 function get_test_code () {
 350
 351     set -exuo pipefail
 352
 353     # Arguments:
 354     # - ${1} - Optional, argument of entry script (or empty as unset).
 355     #   Test code value to override job name from environment.
 356     # Variables read:
 357     # - JOB_NAME - String affecting test selection, default if not argument.
 358     # Variables set:
 359     # - TEST_CODE - The test selection string from environment or argument.
 360     # - NODENESS - Node multiplicity of desired testbed.
 361     # - FLAVOR - Node flavor string, usually describing the processor.
 362
 363     TEST_CODE="${1-}" || die "Reading optional argument failed, somehow."
 364     if [[ -z "${TEST_CODE}" ]]; then
 365         TEST_CODE="${JOB_NAME-}" || die "Reading job name failed, somehow."
 366     fi
 367
 368     case "${TEST_CODE}" in
 369         *"1n-vbox"*)
 370             NODENESS="1n"
 371             FLAVOR="vbox"
 372             ;;
 373         *"1n-skx"*)
 374             NODENESS="1n"
 375             FLAVOR="skx"
 376             ;;
 377         *"2n-skx"*)
 378             NODENESS="2n"
 379             FLAVOR="skx"
 380             ;;
 381         *"3n-skx"*)
 382             NODENESS="3n"
 383             FLAVOR="skx"
 384             ;;
 385         *)
 386             # Fallback to 3-node Haswell by default (backward compatibility)
 387             NODENESS="3n"
 388             FLAVOR="hsw"
 389             ;;
 390     esac
 391 }
 392
 393
 394 function get_test_tag_string () {
 395
 396     set -exuo pipefail
 397
 398     # Variables read:
 399     # - GERRIT_EVENT_TYPE - Event type set by gerrit, can be unset.
 400     # - GERRIT_EVENT_COMMENT_TEXT - Comment text, read for "comment-added" type.
 401     # - TEST_CODE - The test selection string from environment or argument.
 402     # Variables set:
 403     # - TEST_TAG_STRING - The string following "perftest" in gerrit comment,
 404     #   or empty.
 405
 406     # TODO: ci-management scripts no longer need to perform this.
 407
 408     trigger=""
 409     if [[ "${GERRIT_EVENT_TYPE-}" == "comment-added" ]]; then
 410         case "${TEST_CODE}" in
 411             *"device"*)
 412                 # On parsing error, ${trigger} stays empty.
 413                 trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \
 414                     | grep -oE '(devicetest$|devicetest[[:space:]].+$)')" \
 415                     || true
 416                 # Set test tags as string.
 417                 TEST_TAG_STRING="${trigger#$"devicetest"}"
 418                 ;;
 419             *"perf"*)
 420                 # On parsing error, ${trigger} stays empty.
 421                 comment="${GERRIT_EVENT_COMMENT_TEXT}"
 422                 # As "perftest" can be followed by something, we substitute it.
 423                 comment="${comment/perftest-2n/perftest}"
 424                 comment="${comment/perftest-3n/perftest}"
 425                 comment="${comment/perftest-hsw/perftest}"
 426                 comment="${comment/perftest-skx/perftest}"
 427                 tag_string="$(echo "${comment}" \
 428                     | grep -oE '(perftest$|perftest[[:space:]].+$)' || true)"
 429                 # Set test tags as string.
 430                 TEST_TAG_STRING="${tag_string#$"perftest"}"
 431                 ;;
 432             *)
 433                 die "Unknown specification: ${TEST_CODE}"
 434         esac
 435     fi
 436 }
 437
 438
 439 function reserve_testbed () {
 440
 441     set -exuo pipefail
 442
 443     # Reserve physical testbed, perform cleanup, register trap to unreserve.
 444     #
 445     # Variables read:
 446     # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on.
 447     # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
 448     # Variables set:
 449     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 450     # Functions called:
 451     # - die - Print to stderr and exit.
 452     # Traps registered:
 453     # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
 454
 455     while true; do
 456         for topo in "${TOPOLOGIES[@]}"; do
 457             set +e
 458             python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}"
 459             result="$?"
 460             set -e
 461             if [[ "${result}" == "0" ]]; then
 462                 WORKING_TOPOLOGY="${topo}"
 463                 echo "Reserved: ${WORKING_TOPOLOGY}"
 464                 trap "untrap_and_unreserve_testbed" EXIT || {
 465                     message="TRAP ATTEMPT AND UNRESERVE FAILED, FIX MANUALLY."
 466                     untrap_and_unreserve_testbed "${message}" || {
 467                         die "Teardown should have died, not failed."
 468                     }
 469                     die "Trap attempt failed, unreserve succeeded. Aborting."
 470                 }
 471                 cleanup_topo || {
 472                     die "Testbed cleanup failed."
 473                 }
 474                 break
 475             fi
 476         done
 477
 478         if [[ -n "${WORKING_TOPOLOGY-}" ]]; then
 479             # Exit the infinite while loop if we made a reservation.
 480             break
 481         fi
 482
 483         # Wait ~3minutes before next try.
 484         sleep_time="$[ ( $RANDOM % 20 ) + 180 ]s" || {
 485             die "Sleep time calculation failed."
 486         }
 487         echo "Sleeping ${sleep_time}"
 488         sleep "${sleep_time}" || die "Sleep failed."
 489     done
 490 }
 491
 492
 493 function run_pybot () {
 494
 495     set -exuo pipefail
 496
 497     # Currently, VPP-1361 causes occasional test failures.
 498     # If real result is more important than time, we can retry few times.
 499     # TODO: We should be retrying on test case level instead.
 500
 501     # Arguments:
 502     # - ${1} - Optional number of pybot invocations to try to avoid failures.
 503     #   Default: 1.
 504     # Variables read:
 505     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 506     # - ARCHIVE_DIR - Path to store robot result files in.
 507     # - PYBOT_ARGS, EXPANDED_TAGS - See compose_pybot_arguments.sh
 508     # Variables set:
 509     # - PYBOT_EXIT_STATUS - Exit status of most recent pybot invocation.
 510     # Functions called:
 511     # - die - Print to stderr and exit.
 512
 513     # Set ${tries} as an integer variable, to fail on non-numeric input.
 514     local -i "tries" || die "Setting type of variable failed."
 515     tries="${1:-1}" || die "Argument evaluation failed."
 516     all_options=("--outputdir" "${ARCHIVE_DIR}" "${PYBOT_ARGS[@]}")
 517     all_options+=("${EXPANDED_TAGS[@]}")
 518
 519     while true; do
 520         if [[ "${tries}" -le 0 ]]; then
 521             break
 522         else
 523             tries="$((${tries} - 1))"
 524         fi
 525         pushd "${CSIT_DIR}" || die "Change directory operation failed."
 526         set +e
 527         # TODO: Make robot tests not require "$(pwd)" == "${CSIT_DIR}".
 528         pybot "${all_options[@]}" "${CSIT_DIR}/tests/"
 529         PYBOT_EXIT_STATUS="$?"
 530         set -e
 531         popd || die "Change directory operation failed."
 532         if [[ "${PYBOT_EXIT_STATUS}" == "0" ]]; then
 533             break
 534         fi
 535     done
 536 }
 537
 538
 539 function select_tags () {
 540
 541     set -exuo pipefail
 542
 543     # Variables read:
 544     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 545     # - TEST_CODE - String affecting test selection, usually jenkins job name.
 546     # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
 547     #   Can be unset.
 548     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
 549     # Variables set:
 550     # - TAGS - Array of processed tag boolean expressions.
 551
 552     # NIC SELECTION
 553     # All topologies NICs
 554     available=$(grep -hoPR "model: \K.*" "${TOPOLOGIES_DIR}"/* | sort -u)
 555     # Selected topology NICs
 556     reserved=$(grep -hoPR "model: \K.*" "${WORKING_TOPOLOGY}" | sort -u)
 557     # All topologies NICs - Selected topology NICs
 558     exclude_nics=($(comm -13 <(echo "${reserved}") <(echo "${available}")))
 559
 560     case "${TEST_CODE}" in
 561         # Select specific performance tests based on jenkins job type variable.
 562         *"ndrpdr-weekly"* )
 563             test_tag_array=("ndrpdrAND64bAND1c"
 564                             "ndrpdrAND78bAND1c")
 565             ;;
 566         *"mrr-daily"* )
 567             test_tag_array=(# vic
 568                             "mrrANDnic_cisco-vic-1227AND64b"
 569                             "mrrANDnic_cisco-vic-1385AND64b"
 570                             # memif
 571                             "mrrANDmemifANDethAND64b"
 572                             "mrrANDmemifANDethANDimix"
 573                             # crypto
 574                             "mrrANDipsecAND64b"
 575                             # ip4 base
 576                             "mrrANDip4baseAND64b"
 577                             # ip4 scale FIB 2M
 578                             "mrrANDip4fwdANDfib_2mAND64b"
 579                             # ip4 scale FIB 200k
 580                             "mrrANDip4fwdANDfib_200kANDnic_intel-*710AND64b"
 581                             # ip4 scale FIB 20k
 582                             "mrrANDip4fwdANDfib_20kANDnic_intel-*710AND64b"
 583                             # ip4 scale ACL
 584                             "mrrANDip4fwdANDacl1AND10k_flowsAND64b"
 585                             "mrrANDip4fwdANDacl50AND10k_flowsAND64b"
 586                             # ip4 scale NAT44
 587                             "mrrANDip4fwdANDnat44ANDbaseAND64b"
 588                             "mrrANDip4fwdANDnat44ANDsrc_user_4000AND64b"
 589                             # ip4 features
 590                             "mrrANDip4fwdANDfeatureANDnic_intel-*710AND64b"
 591                             # TODO: Remove when tags in
 592                             # tests/vpp/perf/ip4/*-ipolicemarkbase-*.robot
 593                             # are fixed
 594                             "mrrANDip4fwdANDpolice_markANDnic_intel-*710AND64b"
 595                             # ip4 tunnels
 596                             "mrrANDip4fwdANDencapANDip6unrlayANDip4ovrlayANDnic_intel-x520-da2AND64b"
 597                             "mrrANDip4fwdANDencapANDnic_intel-*710AND64b"
 598                             "mrrANDl2ovrlayANDencapANDnic_intel-*710AND64b"
 599                             # ip6 base
 600                             "mrrANDip6baseANDethAND78b"
 601                             # ip6 features
 602                             "mrrANDip6fwdANDfeatureANDnic_intel-*710AND78b"
 603                             # ip6 scale FIB 2M
 604                             "mrrANDip6fwdANDfib_2mANDnic_intel-*710AND78b"
 605                             # ip6 scale FIB 200k
 606                             "mrrANDip6fwdANDfib_200kANDnic_intel-*710AND78b"
 607                             # ip6 scale FIB 20k
 608                             "mrrANDip6fwdANDfib_20kANDnic_intel-*710AND78b"
 609                             # ip6 tunnels
 610                             "mrrANDip6fwdANDencapANDnic_intel-x520-da2AND78b"
 611                             # l2xc base
 612                             "mrrANDl2xcfwdANDbaseAND64b"
 613                             # l2xc scale ACL
 614                             "mrrANDl2xcANDacl1AND10k_flowsAND64b"
 615                             "mrrANDl2xcANDacl50AND10k_flowsAND64b"
 616                             # l2xc scale FIB 2M
 617                             "mrrANDl2xcANDfib_2mAND64b"
 618                             # l2xc scale FIB 200k
 619                             "mrrANDl2xcANDfib_200kANDnic_intel-*710AND64b"
 620                             # l2xc scale FIB 20k
 621                             "mrrANDl2xcANDfib_20kANDnic_intel-*710AND64b"
 622                             # l2bd base
 623                             "mrrANDl2bdmaclrnANDbaseAND64b"
 624                             # l2bd scale ACL
 625                             "mrrANDl2bdmaclrnANDacl1AND10k_flowsAND64b"
 626                             "mrrANDl2bdmaclrnANDacl50AND10k_flowsAND64b"
 627                             # l2bd scale FIB 2M
 628                             "mrrANDl2bdmaclrnANDfib_1mAND64b"
 629                             # l2bd scale FIB 200k
 630                             "mrrANDl2bdmaclrnANDfib_100kANDnic_intel-*710AND64b"
 631                             # l2bd scale FIB 20k
 632                             "mrrANDl2bdmaclrnANDfib_10kANDnic_intel-*710AND64b"
 633                             # l2 patch base
 634                             "mrrANDl2patchAND64b"
 635                             # srv6
 636                             "mrrANDsrv6ANDnic_intel-x520-da2AND78b"
 637                             # vts
 638                             "mrrANDvtsANDnic_intel-x520-da2AND114b"
 639                             # vm vhost l2xc base
 640                             "mrrANDvhostANDl2xcfwdANDbaseAND64b"
 641                             "mrrANDvhostANDl2xcfwdANDbaseANDimix"
 642                             # vm vhost l2bd base
 643                             "mrrANDvhostANDl2bdmaclrnANDbaseAND64b"
 644                             "mrrANDvhostANDl2bdmaclrnANDbaseANDimix"
 645                             # vm vhost ip4 base
 646                             "mrrANDvhostANDip4fwdANDbaseAND64b"
 647                             "mrrANDvhostANDip4fwdANDbaseANDimix"
 648                             # DPDK
 649                             "mrrANDdpdkAND64b"
 650                             # Exclude
 651                             "!mrrANDip6baseANDdot1qAND78b"
 652                             "!vhost_256ANDnic_intel-x520-da2"
 653                             "!vhostANDnic_intel-xl710"
 654                             "!cfs_opt"
 655                             "!lbond_dpdk"
 656                             "!nf_density")
 657             ;;
 658         *"mrr-weekly"* )
 659             test_tag_array=(# NF Density tests
 660                             "mrrANDnf_densityAND64b"
 661                             "mrrANDnf_densityANDimix"
 662                             # DPDK
 663                             "mrrANDdpdkAND64b")
 664             ;;
 665         * )
 666             if [[ -z "${TEST_TAG_STRING-}" ]]; then
 667                 # If nothing is specified, we will run pre-selected tests by
 668                 # following tags.
 669                 test_tag_array=("mrrANDnic_intel-x710AND1cAND64bANDip4base"
 670                                 "mrrANDnic_intel-x710AND1cAND78bANDip6base"
 671                                 "mrrANDnic_intel-x710AND1cAND64bANDl2bdbase"
 672                                 "mrrANDnic_intel-x710AND1cAND64bANDl2xcbase"
 673                                 "!dot1q")
 674             else
 675                 # If trigger contains tags, split them into array.
 676                 test_tag_array=(${TEST_TAG_STRING//:/ })
 677             fi
 678             ;;
 679     esac
 680
 681     # Blacklisting certain tags per topology.
 682     case "${TEST_CODE}" in
 683         *"3n-hsw"*)
 684             test_tag_array+=("!drv_avf")
 685             ;;
 686         *"2n-skx"*)
 687             test_tag_array+=("!ipsechw")
 688             ;;
 689         *"3n-skx"*)
 690             test_tag_array+=("!ipsechw")
 691             ;;
 692         *)
 693             # Default to 3n-hsw due to compatibility.
 694             test_tag_array+=("!drv_avf")
 695             ;;
 696     esac
 697
 698     # We will add excluded NICs.
 699     test_tag_array+=("${exclude_nics[@]/#/!NIC_}")
 700
 701     TAGS=()
 702
 703     # We will prefix with perftest to prevent running other tests
 704     # (e.g. Functional).
 705     prefix="perftestAND"
 706     if [[ "${TEST_CODE}" == "vpp-"* ]]; then
 707         # Automatic prefixing for VPP jobs to limit the NIC used and
 708         # traffic evaluation to MRR.
 709         prefix="${prefix}mrrANDnic_intel-x710AND"
 710     fi
 711     for tag in "${test_tag_array[@]}"; do
 712         if [[ ${tag} == "!"* ]]; then
 713             # Exclude tags are not prefixed.
 714             TAGS+=("${tag}")
 715         else
 716             TAGS+=("${prefix}${tag}")
 717         fi
 718     done
 719 }
 720
 721
 722 function select_vpp_device_tags () {
 723
 724     set -exuo pipefail
 725
 726     # Variables read:
 727     # - TEST_CODE - String affecting test selection, usually jenkins job name.
 728     # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
 729     #   Can be unset.
 730     # Variables set:
 731     # - TAGS - Array of processed tag boolean expressions.
 732
 733     case "${TEST_CODE}" in
 734         # Select specific performance tests based on jenkins job type variable.
 735         * )
 736             if [[ -z "${TEST_TAG_STRING-}" ]]; then
 737                 # If nothing is specified, we will run pre-selected tests by
 738                 # following tags. Items of array will be concatenated by OR
 739                 # in Robot Framework.
 740                 test_tag_array=()
 741             else
 742                 # If trigger contains tags, split them into array.
 743                 test_tag_array=(${TEST_TAG_STRING//:/ })
 744             fi
 745             ;;
 746     esac
 747
 748     TAGS=()
 749
 750     # We will prefix with perftest to prevent running other tests
 751     # (e.g. Functional).
 752     prefix="devicetestAND"
 753     if [[ "${TEST_CODE}" == "vpp-"* ]]; then
 754         # Automatic prefixing for VPP jobs to limit testing.
 755         prefix="${prefix}"
 756     fi
 757     for tag in "${test_tag_array[@]}"; do
 758         if [[ ${tag} == "!"* ]]; then
 759             # Exclude tags are not prefixed.
 760             TAGS+=("${tag}")
 761         else
 762             TAGS+=("${prefix}${tag}")
 763         fi
 764     done
 765 }
 766
 767
 768 function select_topology () {
 769
 770     set -exuo pipefail
 771
 772     # Variables read:
 773     # - NODENESS - Node multiplicity of testbed, either "2n" or "3n".
 774     # - FLAVOR - Node flavor string, currently either "hsw" or "skx".
 775     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 776     # - TOPOLOGIES_DIR - Path to existing directory with available topologies.
 777     # Variables set:
 778     # - TOPOLOGIES - Array of paths to suitable topology yaml files.
 779     # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology.
 780     # Functions called:
 781     # - die - Print to stderr and exit.
 782
 783     case_text="${NODENESS}_${FLAVOR}"
 784     case "${case_text}" in
 785         "1n_vbox")
 786             TOPOLOGIES=(
 787                         "${TOPOLOGIES_DIR}/vpp_device.template"
 788                        )
 789             TOPOLOGIES_TAGS="2_node_single_link_topo"
 790             ;;
 791         "1n_skx")
 792             TOPOLOGIES=(
 793                         "${TOPOLOGIES_DIR}/vpp_device.template"
 794                        )
 795             TOPOLOGIES_TAGS="2_node_single_link_topo"
 796             ;;
 797         "2n_skx")
 798             TOPOLOGIES=(
 799                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed21.yaml"
 800                         #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed22.yaml"
 801                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed23.yaml"
 802                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed24.yaml"
 803                        )
 804             TOPOLOGIES_TAGS="2_node_*_link_topo"
 805             ;;
 806         "3n_skx")
 807             TOPOLOGIES=(
 808                         "${TOPOLOGIES_DIR}/lf_3n_skx_testbed31.yaml"
 809                         "${TOPOLOGIES_DIR}/lf_3n_skx_testbed32.yaml"
 810                        )
 811             TOPOLOGIES_TAGS="3_node_*_link_topo"
 812             ;;
 813         "3n_hsw")
 814             TOPOLOGIES=(
 815                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml"
 816                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml"
 817                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml"
 818                        )
 819             TOPOLOGIES_TAGS="3_node_single_link_topo"
 820             ;;
 821         *)
 822             # No falling back to 3n_hsw default, that should have been done
 823             # by the function which has set NODENESS and FLAVOR.
 824             die "Unknown specification: ${case_text}"
 825     esac
 826
 827     if [[ -z "${TOPOLOGIES-}" ]]; then
 828         die "No applicable topology found!"
 829     fi
 830 }
 831
 832
 833 function untrap_and_unreserve_testbed () {
 834     # Use this as a trap function to ensure testbed does not remain reserved.
 835     # Perhaps call directly before script exit, to free testbed for other jobs.
 836     # This function is smart enough to avoid multiple unreservations (so safe).
 837     # Topo cleanup is executed (call it best practice), ignoring failures.
 838     #
 839     # Hardcoded values:
 840     # - default message to die with if testbed might remain reserved.
 841     # Arguments:
 842     # - ${1} - Message to die with if unreservation fails. Default hardcoded.
 843     # Variables read (by inner function):
 844     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 845     # - PYTHON_SCRIPTS_DIR - Path to directory holding Python scripts.
 846     # Variables written:
 847     # - WORKING_TOPOLOGY - Set to empty string on successful unreservation.
 848     # Trap unregistered:
 849     # - EXIT - Failure to untrap is reported, but ignored otherwise.
 850     # Functions called:
 851     # - die - Print to stderr and exit.
 852
 853     set -xo pipefail
 854     set +eu  # We do not want to exit early in a "teardown" function.
 855     trap - EXIT || echo "Trap deactivation failed, continuing anyway."
 856     wt="${WORKING_TOPOLOGY}"  # Just to avoid too long lines.
 857     if [[ -z "${wt-}" ]]; then
 858         set -eu
 859         warn "Testbed looks unreserved already. Trap removal failed before?"
 860     else
 861         cleanup_topo || true
 862         python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || {
 863             die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2
 864         }
 865         WORKING_TOPOLOGY=""
 866         set -eu
 867     fi
 868 }
 869
 870
 871 function warn () {
 872     # Print the message to standard error.
 873     #
 874     # Arguments:
 875     # - ${@} - The text of the message.
 876
 877     echo "$@" >&2
 878 }