resources/libraries/bash/function/common.sh

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 set -exuo pipefail
  15
  16 # This library defines functions used by multiple entry scripts.
  17 # Keep functions ordered alphabetically, please.
  18
  19 # TODO: Add a link to bash style guide.
  20 # TODO: Consider putting every die into a {} block,
  21 #   the code might become more readable (but longer).
  22
  23
  24 function activate_docker_topology () {
  25     # Create virtual vpp-device topology. Output of the function is topology
  26     # file describing created environment saved to a file.
  27     #
  28     # Variables read:
  29     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
  30     # - TOPOLOGIES - Available topologies.
  31     # - NODENESS - Node multiplicity of desired testbed.
  32     # - FLAVOR - Node flavor string, usually describing the processor.
  33     # Variables set:
  34     # - WORKING_TOPOLOGY - Path to topology file.
  35
  36     set -exuo pipefail
  37
  38     source "${BASH_FUNCTION_DIR}/device.sh" || {
  39         die "Source failed!"
  40     }
  41
  42     device_image="$(< ${CSIT_DIR}/VPP_DEVICE_IMAGE)"
  43     case_text="${NODENESS}_${FLAVOR}"
  44     case "${case_text}" in
  45         "1n_skx")
  46             # We execute reservation over csit-shim-dcr (ssh) which runs sourced
  47             # script's functions. Env variables are read from ssh output
  48             # back to localhost for further processing.
  49             hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3)
  50             ssh="ssh root@${hostname} -p 6022"
  51             run="activate_wrapper ${NODENESS} ${FLAVOR} ${device_image}"
  52             env_vars=$(${ssh} "$(declare -f); ${run}") || {
  53                 die "Topology reservation via shim-dcr failed!"
  54             }
  55             set -a
  56             source <(echo "$env_vars" | grep -v /usr/bin/docker) || {
  57                 die "Source failed!"
  58             }
  59             set +a
  60             ;;
  61         "1n_vbox")
  62             # We execute reservation on localhost. Sourced script automatially
  63             # sets environment variables for further processing.
  64             activate_wrapper "${NODENESS}" "${FLAVOR}" "${device_image}" || die
  65             ;;
  66         *)
  67             die "Unknown specification: ${case_text}!"
  68     esac
  69
  70     trap 'deactivate_docker_topology' EXIT || {
  71          die "Trap attempt failed, please cleanup manually. Aborting!"
  72     }
  73
  74     # Replace all variables in template with those in environment.
  75     source <(echo 'cat <<EOF >topo.yml'; cat ${TOPOLOGIES[0]}; echo EOF;) || {
  76         die "Topology file create failed!"
  77     }
  78
  79     WORKING_TOPOLOGY="/tmp/topology.yaml"
  80     mv topo.yml "${WORKING_TOPOLOGY}" || {
  81         die "Topology move failed!"
  82     }
  83     cat ${WORKING_TOPOLOGY} | grep -v password || {
  84         die "Topology read failed!"
  85     }
  86 }
  87
  88
  89 function activate_virtualenv () {
  90
  91     set -exuo pipefail
  92
  93     # Arguments:
  94     # - ${1} - Non-empty path to existing directory for creating virtualenv in.
  95     # Variables read:
  96     # - CSIT_DIR - Path to existing root of local CSIT git repository.
  97     # Variables set:
  98     # - ENV_DIR - Path to the created virtualenv subdirectory.
  99     # Variables exported:
 100     # - PYTHONPATH - CSIT_DIR, as CSIT Python scripts usually need this.
 101     # Functions called:
 102     # - die - Print to stderr and exit.
 103
 104     # TODO: Do we really need to have ENV_DIR available as a global variable?
 105
 106     if [[ "${1-}" == "" ]]; then
 107         die "Root location of virtualenv to create is not specified."
 108     fi
 109     ENV_DIR="${1}/env"
 110     rm -rf "${ENV_DIR}" || die "Failed to clean previous virtualenv."
 111
 112     pip install --upgrade virtualenv || {
 113         die "Virtualenv package install failed."
 114     }
 115     virtualenv "${ENV_DIR}" || {
 116         die "Virtualenv creation failed."
 117     }
 118     set +u
 119     source "${ENV_DIR}/bin/activate" || die "Virtualenv activation failed."
 120     set -u
 121     pip install -r "${CSIT_DIR}/requirements.txt" || {
 122         die "CSIT requirements installation failed."
 123     }
 124
 125     # Most CSIT Python scripts assume PYTHONPATH is set and exported.
 126     export PYTHONPATH="${CSIT_DIR}" || die "Export failed."
 127 }
 128
 129
 130 function check_download_dir () {
 131
 132     set -exuo pipefail
 133
 134     # Fail if there are no files visible in ${DOWNLOAD_DIR}.
 135     # TODO: Do we need this as a function, if it is (almost) a one-liner?
 136     #
 137     # Variables read:
 138     # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
 139     # Directories read:
 140     # - ${DOWNLOAD_DIR} - Has to be non-empty to proceed.
 141     # Functions called:
 142     # - die - Print to stderr and exit.
 143
 144     if [[ ! "$(ls -A "${DOWNLOAD_DIR}")" ]]; then
 145         die "No artifacts downloaded!"
 146     fi
 147 }
 148
 149
 150 function cleanup_topo () {
 151
 152     set -exuo pipefail
 153
 154     # Variables read:
 155     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 156     # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
 157
 158     python "${PYTHON_SCRIPTS_DIR}/topo_cleanup.py" -t "${WORKING_TOPOLOGY}"
 159     # Not using "|| die" as some callers might want to ignore errors,
 160     # e.g. in teardowns, such as unreserve.
 161 }
 162
 163
 164 function common_dirs () {
 165
 166     set -exuo pipefail
 167
 168     # Variables set:
 169     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
 170     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 171     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
 172     # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources".
 173     # - TOOLS_DIR - Path to existing resources subdirectory "tools".
 174     # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".
 175     # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive".
 176     # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
 177     # Functions called:
 178     # - die - Print to stderr and exit.
 179
 180     BASH_FUNCTION_DIR="$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")" || {
 181         die "Some error during localizing this source directory."
 182     }
 183     # Current working directory could be in a different repo, e.g. VPP.
 184     pushd "${BASH_FUNCTION_DIR}" || die "Pushd failed"
 185     CSIT_DIR="$(readlink -e "$(git rev-parse --show-toplevel)")" || {
 186         die "Readlink or git rev-parse failed."
 187     }
 188     popd || die "Popd failed."
 189     TOPOLOGIES_DIR="$(readlink -e "${CSIT_DIR}/topologies/available")" || {
 190         die "Readlink failed."
 191     }
 192     RESOURCES_DIR="$(readlink -e "${CSIT_DIR}/resources")" || {
 193         die "Readlink failed."
 194     }
 195     TOOLS_DIR="$(readlink -e "${RESOURCES_DIR}/tools")" || {
 196         die "Readlink failed."
 197     }
 198     PYTHON_SCRIPTS_DIR="$(readlink -e "${TOOLS_DIR}/scripts")" || {
 199         die "Readlink failed."
 200     }
 201
 202     ARCHIVE_DIR="$(readlink -f "${CSIT_DIR}/archive")" || {
 203         die "Readlink failed."
 204     }
 205     mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed."
 206     DOWNLOAD_DIR="$(readlink -f "${CSIT_DIR}/download_dir")" || {
 207         die "Readlink failed."
 208     }
 209     mkdir -p "${DOWNLOAD_DIR}" || die "Mkdir failed."
 210 }
 211
 212
 213 function compose_pybot_arguments () {
 214
 215     set -exuo pipefail
 216
 217     # Variables read:
 218     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 219     # - DUT - CSIT test/ subdirectory, set while processing tags.
 220     # - TAGS - Array variable holding selected tag boolean expressions.
 221     # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology.
 222     # - TEST_CODE - The test selection string from environment or argument.
 223     # Variables set:
 224     # - PYBOT_ARGS - String holding part of all arguments for pybot.
 225     # - EXPANDED_TAGS - Array of strings pybot arguments compiled from tags.
 226
 227     # No explicit check needed with "set -u".
 228     PYBOT_ARGS=("--loglevel" "TRACE")
 229     PYBOT_ARGS+=("--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}")
 230
 231     case "${TEST_CODE}" in
 232         *"device"*)
 233             PYBOT_ARGS+=("--suite" "tests.${DUT}.device")
 234             ;;
 235         *"func"*)
 236             PYBOT_ARGS+=("--suite" "tests.${DUT}.func")
 237             ;;
 238         *"perf"*)
 239             PYBOT_ARGS+=("--suite" "tests.${DUT}.perf")
 240             ;;
 241         *)
 242             die "Unknown specification: ${TEST_CODE}"
 243     esac
 244
 245     EXPANDED_TAGS=()
 246     for tag in "${TAGS[@]}"; do
 247         if [[ ${tag} == "!"* ]]; then
 248             EXPANDED_TAGS+=("--exclude" "${tag#$"!"}")
 249         else
 250             EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}")
 251         fi
 252     done
 253 }
 254
 255
 256 function copy_archives () {
 257
 258     set -exuo pipefail
 259
 260     # Variables read:
 261     # - WORKSPACE - Jenkins workspace, copy only if the value is not empty.
 262     #   Can be unset, then it speeds up manual testing.
 263     # - ARCHIVE_DIR - Path to directory with content to be copied.
 264     # Directories updated:
 265     # - ${WORKSPACE}/archives/ - Created if does not exist.
 266     #   Content of ${ARCHIVE_DIR}/ is copied here.
 267     # Functions called:
 268     # - die - Print to stderr and exit.
 269
 270     # We will create additional archive if workspace variable is set.
 271     # This way if script is running in jenkins all will be
 272     # automatically archived to logs.fd.io.
 273     if [[ -n "${WORKSPACE-}" ]]; then
 274         mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed."
 275         cp -rf "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed."
 276     fi
 277 }
 278
 279
 280 function deactivate_docker_topology () {
 281     # Deactivate virtual vpp-device topology by removing containers.
 282     #
 283     # Variables read:
 284     # - NODENESS - Node multiplicity of desired testbed.
 285     # - FLAVOR - Node flavor string, usually describing the processor.
 286
 287     set -exuo pipefail
 288
 289     case_text="${NODENESS}_${FLAVOR}"
 290     case "${case_text}" in
 291         "1n_skx")
 292             hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3)
 293             ssh="ssh root@${hostname} -p 6022"
 294             env_vars="$(env | grep CSIT_ | tr '\n' ' ' )"
 295             ${ssh} "$(declare -f); deactivate_wrapper ${env_vars}" || {
 296                 die "Topology cleanup via shim-dcr failed!"
 297             }
 298             ;;
 299         "1n_vbox")
 300             enter_mutex || die
 301             clean_environment || {
 302                 die "Topology cleanup locally failed!"
 303             }
 304             exit_mutex || die
 305             ;;
 306         *)
 307             die "Unknown specification: ${case_text}!"
 308     esac
 309 }
 310
 311
 312 function die () {
 313     # Print the message to standard error end exit with error code specified
 314     # by the second argument.
 315     #
 316     # Hardcoded values:
 317     # - The default error message.
 318     # Arguments:
 319     # - ${1} - The whole error message, be sure to quote. Optional
 320     # - ${2} - the code to exit with, default: 1.
 321
 322     set -x
 323     set +eu
 324     warn "${1:-Unspecified run-time error occurred!}"
 325     exit "${2:-1}"
 326 }
 327
 328
 329 function die_on_pybot_error () {
 330
 331     set -exuo pipefail
 332
 333     # Source this fragment if you want to abort on any failed test case.
 334     #
 335     # Variables read:
 336     # - PYBOT_EXIT_STATUS - Set by a pybot running fragment.
 337     # Functions called:
 338     # - die - Print to stderr and exit.
 339
 340     if [[ "${PYBOT_EXIT_STATUS}" != "0" ]]; then
 341         die "${PYBOT_EXIT_STATUS}" "Test failures are present!"
 342     fi
 343 }
 344
 345
 346 function get_test_code () {
 347
 348     set -exuo pipefail
 349
 350     # Arguments:
 351     # - ${1} - Optional, argument of entry script (or empty as unset).
 352     #   Test code value to override job name from environment.
 353     # Variables read:
 354     # - JOB_NAME - String affecting test selection, default if not argument.
 355     # Variables set:
 356     # - TEST_CODE - The test selection string from environment or argument.
 357     # - NODENESS - Node multiplicity of desired testbed.
 358     # - FLAVOR - Node flavor string, usually describing the processor.
 359
 360     TEST_CODE="${1-}" || die "Reading optional argument failed, somehow."
 361     if [[ -z "${TEST_CODE}" ]]; then
 362         TEST_CODE="${JOB_NAME-}" || die "Reading job name failed, somehow."
 363     fi
 364
 365     case "${TEST_CODE}" in
 366         *"1n-vbox"*)
 367             NODENESS="1n"
 368             FLAVOR="vbox"
 369             ;;
 370         *"1n-skx"*)
 371             NODENESS="1n"
 372             FLAVOR="skx"
 373             ;;
 374         *"2n-skx"*)
 375             NODENESS="2n"
 376             FLAVOR="skx"
 377             ;;
 378         *"3n-skx"*)
 379             NODENESS="3n"
 380             FLAVOR="skx"
 381             ;;
 382         *)
 383             # Fallback to 3-node Haswell by default (backward compatibility)
 384             NODENESS="3n"
 385             FLAVOR="hsw"
 386             ;;
 387     esac
 388 }
 389
 390
 391 function get_test_tag_string () {
 392
 393     set -exuo pipefail
 394
 395     # Variables read:
 396     # - GERRIT_EVENT_TYPE - Event type set by gerrit, can be unset.
 397     # - GERRIT_EVENT_COMMENT_TEXT - Comment text, read for "comment-added" type.
 398     # - TEST_CODE - The test selection string from environment or argument.
 399     # Variables set:
 400     # - TEST_TAG_STRING - The string following "perftest" in gerrit comment,
 401     #   or empty.
 402
 403     # TODO: ci-management scripts no longer need to perform this.
 404
 405     trigger=""
 406     if [[ "${GERRIT_EVENT_TYPE-}" == "comment-added" ]]; then
 407         case "${TEST_CODE}" in
 408             *"device"*)
 409                 # On parsing error, ${trigger} stays empty.
 410                 trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \
 411                     | grep -oE '(devicetest$|devicetest[[:space:]].+$)')" \
 412                     || true
 413                 # Set test tags as string.
 414                 TEST_TAG_STRING="${trigger#$"devicetest"}"
 415                 ;;
 416             *"perf"*)
 417                 # On parsing error, ${trigger} stays empty.
 418                 trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \
 419                     | grep -oE '(perftest$|perftest[[:space:]].+$)')" \
 420                     || true
 421                 # Set test tags as string.
 422                 TEST_TAG_STRING="${trigger#$"perftest"}"
 423                 ;;
 424             *)
 425                 die "Unknown specification: ${TEST_CODE}"
 426         esac
 427     fi
 428 }
 429
 430
 431 function reserve_testbed () {
 432
 433     set -exuo pipefail
 434
 435     # Reserve physical testbed, perform cleanup, register trap to unreserve.
 436     #
 437     # Variables read:
 438     # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on.
 439     # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
 440     # Variables set:
 441     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 442     # Functions called:
 443     # - die - Print to stderr and exit.
 444     # Traps registered:
 445     # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
 446
 447     while true; do
 448         for topo in "${TOPOLOGIES[@]}"; do
 449             set +e
 450             python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}"
 451             result="$?"
 452             set -e
 453             if [[ "${result}" == "0" ]]; then
 454                 WORKING_TOPOLOGY="${topo}"
 455                 echo "Reserved: ${WORKING_TOPOLOGY}"
 456                 trap "untrap_and_unreserve_testbed" EXIT || {
 457                     message="TRAP ATTEMPT AND UNRESERVE FAILED, FIX MANUALLY."
 458                     untrap_and_unreserve_testbed "${message}" || {
 459                         die "Teardown should have died, not failed."
 460                     }
 461                     die "Trap attempt failed, unreserve succeeded. Aborting."
 462                 }
 463                 cleanup_topo || {
 464                     die "Testbed cleanup failed."
 465                 }
 466                 break
 467             fi
 468         done
 469
 470         if [[ -n "${WORKING_TOPOLOGY-}" ]]; then
 471             # Exit the infinite while loop if we made a reservation.
 472             break
 473         fi
 474
 475         # Wait ~3minutes before next try.
 476         sleep_time="$[ ( $RANDOM % 20 ) + 180 ]s" || {
 477             die "Sleep time calculation failed."
 478         }
 479         echo "Sleeping ${sleep_time}"
 480         sleep "${sleep_time}" || die "Sleep failed."
 481     done
 482 }
 483
 484
 485 function run_pybot () {
 486
 487     set -exuo pipefail
 488
 489     # Currently, VPP-1361 causes occasional test failures.
 490     # If real result is more important than time, we can retry few times.
 491     # TODO: We should be retrying on test case level instead.
 492
 493     # Arguments:
 494     # - ${1} - Optional number of pybot invocations to try to avoid failures.
 495     #   Default: 1.
 496     # Variables read:
 497     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 498     # - ARCHIVE_DIR - Path to store robot result files in.
 499     # - PYBOT_ARGS, EXPANDED_TAGS - See compose_pybot_arguments.sh
 500     # Variables set:
 501     # - PYBOT_EXIT_STATUS - Exit status of most recent pybot invocation.
 502     # Functions called:
 503     # - die - Print to stderr and exit.
 504
 505     # Set ${tries} as an integer variable, to fail on non-numeric input.
 506     local -i "tries" || die "Setting type of variable failed."
 507     tries="${1:-1}" || die "Argument evaluation failed."
 508     all_options=("--outputdir" "${ARCHIVE_DIR}" "${PYBOT_ARGS[@]}")
 509     all_options+=("${EXPANDED_TAGS[@]}")
 510
 511     while true; do
 512         if [[ "${tries}" -le 0 ]]; then
 513             break
 514         else
 515             tries="$((${tries} - 1))"
 516         fi
 517         pushd "${CSIT_DIR}" || die "Change directory operation failed."
 518         set +e
 519         # TODO: Make robot tests not require "$(pwd)" == "${CSIT_DIR}".
 520         pybot "${all_options[@]}" "${CSIT_DIR}/tests/"
 521         PYBOT_EXIT_STATUS="$?"
 522         set -e
 523         popd || die "Change directory operation failed."
 524         if [[ "${PYBOT_EXIT_STATUS}" == "0" ]]; then
 525             break
 526         fi
 527     done
 528 }
 529
 530
 531 function select_tags () {
 532
 533     set -exuo pipefail
 534
 535     # Variables read:
 536     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 537     # - TEST_CODE - String affecting test selection, usually jenkins job name.
 538     # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
 539     #   Can be unset.
 540     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
 541     # Variables set:
 542     # - TAGS - Array of processed tag boolean expressions.
 543
 544     # NIC SELECTION
 545     # All topologies NICs
 546     available=$(grep -hoPR "model: \K.*" "${TOPOLOGIES_DIR}"/* | sort -u)
 547     # Selected topology NICs
 548     reserved=$(grep -hoPR "model: \K.*" "${WORKING_TOPOLOGY}" | sort -u)
 549     # All topologies NICs - Selected topology NICs
 550     exclude_nics=($(comm -13 <(echo "${reserved}") <(echo "${available}")))
 551
 552     case "${TEST_CODE}" in
 553         # Select specific performance tests based on jenkins job type variable.
 554         *"ndrpdr-weekly"* )
 555             test_tag_array=("ndrpdrAND64bAND1c"
 556                             "ndrpdrAND78bAND1c")
 557             ;;
 558         *"mrr-daily"* )
 559             test_tag_array=(# vic
 560                             "mrrANDnic_cisco-vic-1227AND64b"
 561                             "mrrANDnic_cisco-vic-1385AND64b"
 562                             # memif
 563                             "mrrANDmemifANDethAND64b"
 564                             "mrrANDmemifANDethANDimix"
 565                             # crypto
 566                             "mrrANDipsecAND64b"
 567                             # ip4 base
 568                             "mrrANDip4baseAND64b"
 569                             # ip4 scale FIB 2M
 570                             "mrrANDip4fwdANDfib_2mAND64b"
 571                             # ip4 scale FIB 200k
 572                             "mrrANDip4fwdANDfib_200kANDnic_intel-*710AND64b"
 573                             # ip4 scale FIB 20k
 574                             "mrrANDip4fwdANDfib_20kANDnic_intel-*710AND64b"
 575                             # ip4 scale ACL
 576                             "mrrANDip4fwdANDacl1AND10k_flowsAND64b"
 577                             "mrrANDip4fwdANDacl50AND10k_flowsAND64b"
 578                             # ip4 scale NAT44
 579                             "mrrANDip4fwdANDnat44ANDbaseAND64b"
 580                             "mrrANDip4fwdANDnat44ANDsrc_user_4000AND64b"
 581                             # ip4 features
 582                             "mrrANDip4fwdANDfeatureANDnic_intel-*710AND64b"
 583                             # TODO: Remove when tags in
 584                             # tests/vpp/perf/ip4/*-ipolicemarkbase-*.robot
 585                             # are fixed
 586                             "mrrANDip4fwdANDpolice_markANDnic_intel-*710AND64b"
 587                             # ip4 tunnels
 588                             "mrrANDip4fwdANDencapANDip6unrlayANDip4ovrlayANDnic_intel-x520-da2AND64b"
 589                             "mrrANDip4fwdANDencapANDnic_intel-*710AND64b"
 590                             "mrrANDl2ovrlayANDencapANDnic_intel-*710AND64b"
 591                             # ip6 base
 592                             "mrrANDip6baseANDethAND78b"
 593                             # ip6 features
 594                             "mrrANDip6fwdANDfeatureANDnic_intel-*710AND78b"
 595                             # ip6 scale FIB 2M
 596                             "mrrANDip6fwdANDfib_2mANDnic_intel-*710AND78b"
 597                             # ip6 scale FIB 200k
 598                             "mrrANDip6fwdANDfib_200kANDnic_intel-*710AND78b"
 599                             # ip6 scale FIB 20k
 600                             "mrrANDip6fwdANDfib_20kANDnic_intel-*710AND78b"
 601                             # ip6 tunnels
 602                             "mrrANDip6fwdANDencapANDnic_intel-x520-da2AND78b"
 603                             # l2xc base
 604                             "mrrANDl2xcfwdANDbaseAND64b"
 605                             # l2xc scale ACL
 606                             "mrrANDl2xcANDacl1AND10k_flowsAND64b"
 607                             "mrrANDl2xcANDacl50AND10k_flowsAND64b"
 608                             # l2xc scale FIB 2M
 609                             "mrrANDl2xcANDfib_2mAND64b"
 610                             # l2xc scale FIB 200k
 611                             "mrrANDl2xcANDfib_200kANDnic_intel-*710AND64b"
 612                             # l2xc scale FIB 20k
 613                             "mrrANDl2xcANDfib_20kANDnic_intel-*710AND64b"
 614                             # l2bd base
 615                             "mrrANDl2bdmaclrnANDbaseAND64b"
 616                             # l2bd scale ACL
 617                             "mrrANDl2bdmaclrnANDacl1AND10k_flowsAND64b"
 618                             "mrrANDl2bdmaclrnANDacl50AND10k_flowsAND64b"
 619                             # l2bd scale FIB 2M
 620                             "mrrANDl2bdmaclrnANDfib_1mAND64b"
 621                             # l2bd scale FIB 200k
 622                             "mrrANDl2bdmaclrnANDfib_100kANDnic_intel-*710AND64b"
 623                             # l2bd scale FIB 20k
 624                             "mrrANDl2bdmaclrnANDfib_10kANDnic_intel-*710AND64b"
 625                             # l2 patch base
 626                             "mrrANDl2patchAND64b"
 627                             # srv6
 628                             "mrrANDsrv6ANDnic_intel-x520-da2AND78b"
 629                             # vts
 630                             "mrrANDvtsANDnic_intel-x520-da2AND114b"
 631                             # vm vhost l2xc base
 632                             "mrrANDvhostANDl2xcfwdANDbaseAND64b"
 633                             "mrrANDvhostANDl2xcfwdANDbaseANDimix"
 634                             # vm vhost l2bd base
 635                             "mrrANDvhostANDl2bdmaclrnANDbaseAND64b"
 636                             "mrrANDvhostANDl2bdmaclrnANDbaseANDimix"
 637                             # vm vhost ip4 base
 638                             "mrrANDvhostANDip4fwdANDbaseAND64b"
 639                             "mrrANDvhostANDip4fwdANDbaseANDimix"
 640                             # DPDK
 641                             "mrrANDdpdkAND64b"
 642                             # Exclude
 643                             "!mrrANDip6baseANDdot1qAND78b"
 644                             "!vhost_256ANDnic_intel-x520-da2"
 645                             "!vhostANDnic_intel-xl710"
 646                             "!cfs_opt"
 647                             "!lbond_dpdk"
 648                             "!nf_density")
 649             ;;
 650         *"mrr-weekly"* )
 651             test_tag_array=(# NF Density tests
 652                             "mrrANDnf_densityAND64b"
 653                             "mrrANDnf_densityANDimix"
 654                             # DPDK
 655                             "mrrANDdpdkAND64b")
 656             ;;
 657         * )
 658             if [[ -z "${TEST_TAG_STRING-}" ]]; then
 659                 # If nothing is specified, we will run pre-selected tests by
 660                 # following tags.
 661                 test_tag_array=("mrrANDnic_intel-x710AND1cAND64bANDip4base"
 662                                 "mrrANDnic_intel-x710AND1cAND78bANDip6base"
 663                                 "mrrANDnic_intel-x710AND1cAND64bANDl2bdbase"
 664                                 "mrrANDnic_intel-x710AND1cAND64bANDl2xcbase"
 665                                 "!dot1q")
 666             else
 667                 # If trigger contains tags, split them into array.
 668                 test_tag_array=(${TEST_TAG_STRING//:/ })
 669             fi
 670             ;;
 671     esac
 672
 673     # Blacklisting certain tags per topology.
 674     case "${TEST_CODE}" in
 675         *"3n-hsw"*)
 676             test_tag_array+=("!drv_avf")
 677             ;;
 678         *"2n-skx"*)
 679             test_tag_array+=("!ipsechw")
 680             ;;
 681         *"3n-skx"*)
 682             test_tag_array+=("!ipsechw")
 683             ;;
 684         *)
 685             # Default to 3n-hsw due to compatibility.
 686             test_tag_array+=("!drv_avf")
 687             ;;
 688     esac
 689
 690     # We will add excluded NICs.
 691     test_tag_array+=("${exclude_nics[@]/#/!NIC_}")
 692
 693     TAGS=()
 694
 695     # We will prefix with perftest to prevent running other tests
 696     # (e.g. Functional).
 697     prefix="perftestAND"
 698     if [[ "${TEST_CODE}" == "vpp-"* ]]; then
 699         # Automatic prefixing for VPP jobs to limit the NIC used and
 700         # traffic evaluation to MRR.
 701         prefix="${prefix}mrrANDnic_intel-x710AND"
 702     fi
 703     for tag in "${test_tag_array[@]}"; do
 704         if [[ ${tag} == "!"* ]]; then
 705             # Exclude tags are not prefixed.
 706             TAGS+=("${tag}")
 707         else
 708             TAGS+=("${prefix}${tag}")
 709         fi
 710     done
 711 }
 712
 713
 714 function select_vpp_device_tags () {
 715
 716     set -exuo pipefail
 717
 718     # Variables read:
 719     # - TEST_CODE - String affecting test selection, usually jenkins job name.
 720     # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
 721     #   Can be unset.
 722     # Variables set:
 723     # - TAGS - Array of processed tag boolean expressions.
 724
 725     case "${TEST_CODE}" in
 726         # Select specific performance tests based on jenkins job type variable.
 727         * )
 728             if [[ -z "${TEST_TAG_STRING-}" ]]; then
 729                 # If nothing is specified, we will run pre-selected tests by
 730                 # following tags. Items of array will be concatenated by OR
 731                 # in Robot Framework.
 732                 test_tag_array=()
 733             else
 734                 # If trigger contains tags, split them into array.
 735                 test_tag_array=(${TEST_TAG_STRING//:/ })
 736             fi
 737             ;;
 738     esac
 739
 740     TAGS=()
 741
 742     # We will prefix with perftest to prevent running other tests
 743     # (e.g. Functional).
 744     prefix="devicetestAND"
 745     if [[ "${TEST_CODE}" == "vpp-"* ]]; then
 746         # Automatic prefixing for VPP jobs to limit testing.
 747         prefix="${prefix}"
 748     fi
 749     for tag in "${test_tag_array[@]}"; do
 750         if [[ ${tag} == "!"* ]]; then
 751             # Exclude tags are not prefixed.
 752             TAGS+=("${tag}")
 753         else
 754             TAGS+=("${prefix}${tag}")
 755         fi
 756     done
 757 }
 758
 759
 760 function select_topology () {
 761
 762     set -exuo pipefail
 763
 764     # Variables read:
 765     # - NODENESS - Node multiplicity of testbed, either "2n" or "3n".
 766     # - FLAVOR - Node flavor string, currently either "hsw" or "skx".
 767     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 768     # - TOPOLOGIES_DIR - Path to existing directory with available topologies.
 769     # Variables set:
 770     # - TOPOLOGIES - Array of paths to suitable topology yaml files.
 771     # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology.
 772     # Functions called:
 773     # - die - Print to stderr and exit.
 774
 775     case_text="${NODENESS}_${FLAVOR}"
 776     case "${case_text}" in
 777         "1n_vbox")
 778             TOPOLOGIES=(
 779                         "${TOPOLOGIES_DIR}/vpp_device.template"
 780                        )
 781             TOPOLOGIES_TAGS="2_node_single_link_topo"
 782             ;;
 783         "1n_skx")
 784             TOPOLOGIES=(
 785                         "${TOPOLOGIES_DIR}/vpp_device.template"
 786                        )
 787             TOPOLOGIES_TAGS="2_node_single_link_topo"
 788             ;;
 789         "2n_skx")
 790             TOPOLOGIES=(
 791                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed21.yaml"
 792                         #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed22.yaml"
 793                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed23.yaml"
 794                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed24.yaml"
 795                        )
 796             TOPOLOGIES_TAGS="2_node_*_link_topo"
 797             ;;
 798         "3n_skx")
 799             TOPOLOGIES=(
 800                         "${TOPOLOGIES_DIR}/lf_3n_skx_testbed31.yaml"
 801                         "${TOPOLOGIES_DIR}/lf_3n_skx_testbed32.yaml"
 802                        )
 803             TOPOLOGIES_TAGS="3_node_*_link_topo"
 804             ;;
 805         "3n_hsw")
 806             TOPOLOGIES=(
 807                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml"
 808                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml"
 809                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml"
 810                        )
 811             TOPOLOGIES_TAGS="3_node_single_link_topo"
 812             ;;
 813         *)
 814             # No falling back to 3n_hsw default, that should have been done
 815             # by the function which has set NODENESS and FLAVOR.
 816             die "Unknown specification: ${case_text}"
 817     esac
 818
 819     if [[ -z "${TOPOLOGIES-}" ]]; then
 820         die "No applicable topology found!"
 821     fi
 822 }
 823
 824
 825 function untrap_and_unreserve_testbed () {
 826     # Use this as a trap function to ensure testbed does not remain reserved.
 827     # Perhaps call directly before script exit, to free testbed for other jobs.
 828     # This function is smart enough to avoid multiple unreservations (so safe).
 829     # Topo cleanup is executed (call it best practice), ignoring failures.
 830     #
 831     # Hardcoded values:
 832     # - default message to die with if testbed might remain reserved.
 833     # Arguments:
 834     # - ${1} - Message to die with if unreservation fails. Default hardcoded.
 835     # Variables read (by inner function):
 836     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 837     # - PYTHON_SCRIPTS_DIR - Path to directory holding Python scripts.
 838     # Variables written:
 839     # - WORKING_TOPOLOGY - Set to empty string on successful unreservation.
 840     # Trap unregistered:
 841     # - EXIT - Failure to untrap is reported, but ignored otherwise.
 842     # Functions called:
 843     # - die - Print to stderr and exit.
 844
 845     set -xo pipefail
 846     set +eu  # We do not want to exit early in a "teardown" function.
 847     trap - EXIT || echo "Trap deactivation failed, continuing anyway."
 848     wt="${WORKING_TOPOLOGY}"  # Just to avoid too long lines.
 849     if [[ -z "${wt-}" ]]; then
 850         set -eu
 851         warn "Testbed looks unreserved already. Trap removal failed before?"
 852     else
 853         cleanup_topo || true
 854         python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || {
 855             die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2
 856         }
 857         WORKING_TOPOLOGY=""
 858         set -eu
 859     fi
 860 }
 861
 862
 863 function warn () {
 864     # Print the message to standard error.
 865     #
 866     # Arguments:
 867     # - ${@} - The text of the message.
 868
 869     echo "$@" >&2
 870 }