resources/libraries/bash/function/common.sh

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Copyright (c) 2019 PANTHEON.tech and/or its affiliates.
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at:
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 set -exuo pipefail
  16
  17 # This library defines functions used by multiple entry scripts.
  18 # Keep functions ordered alphabetically, please.
  19
  20 # TODO: Add a link to bash style guide.
  21 # TODO: Consider putting every die into a {} block,
  22 #   the code might become more readable (but longer).
  23
  24
  25 function activate_docker_topology () {
  26     # Create virtual vpp-device topology. Output of the function is topology
  27     # file describing created environment saved to a file.
  28     #
  29     # Variables read:
  30     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
  31     # - TOPOLOGIES - Available topologies.
  32     # - NODENESS - Node multiplicity of desired testbed.
  33     # - FLAVOR - Node flavor string, usually describing the processor.
  34     # Variables set:
  35     # - WORKING_TOPOLOGY - Path to topology file.
  36
  37     set -exuo pipefail
  38
  39     source "${BASH_FUNCTION_DIR}/device.sh" || {
  40         die "Source failed!"
  41     }
  42
  43     device_image="$(< ${CSIT_DIR}/VPP_DEVICE_IMAGE)"
  44     case_text="${NODENESS}_${FLAVOR}"
  45     case "${case_text}" in
  46         "1n_skx")
  47             # We execute reservation over csit-shim-dcr (ssh) which runs sourced
  48             # script's functions. Env variables are read from ssh output
  49             # back to localhost for further processing.
  50             hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3)
  51             ssh="ssh root@${hostname} -p 6022"
  52             run="activate_wrapper ${NODENESS} ${FLAVOR} ${device_image}"
  53             env_vars=$(${ssh} "$(declare -f); ${run}") || {
  54                 die "Topology reservation via shim-dcr failed!"
  55             }
  56             set -a
  57             source <(echo "$env_vars" | grep -v /usr/bin/docker) || {
  58                 die "Source failed!"
  59             }
  60             set +a
  61             ;;
  62         "1n_vbox")
  63             # We execute reservation on localhost. Sourced script automatially
  64             # sets environment variables for further processing.
  65             activate_wrapper "${NODENESS}" "${FLAVOR}" "${device_image}" || die
  66             ;;
  67         *)
  68             die "Unknown specification: ${case_text}!"
  69     esac
  70
  71     trap 'deactivate_docker_topology' EXIT || {
  72          die "Trap attempt failed, please cleanup manually. Aborting!"
  73     }
  74
  75     # Replace all variables in template with those in environment.
  76     source <(echo 'cat <<EOF >topo.yml'; cat ${TOPOLOGIES[0]}; echo EOF;) || {
  77         die "Topology file create failed!"
  78     }
  79
  80     WORKING_TOPOLOGY="/tmp/topology.yaml"
  81     mv topo.yml "${WORKING_TOPOLOGY}" || {
  82         die "Topology move failed!"
  83     }
  84     cat ${WORKING_TOPOLOGY} | grep -v password || {
  85         die "Topology read failed!"
  86     }
  87 }
  88
  89
  90 function activate_virtualenv () {
  91
  92     set -exuo pipefail
  93
  94     # Update virtualenv pip package, delete and create virtualenv directory,
  95     # activate the virtualenv, install requirements, set PYTHONPATH.
  96
  97     # Arguments:
  98     # - ${1} - Path to existing directory for creating virtualenv in.
  99     #          If missing or empty, ${CSIT_DIR} is used.
 100     # - ${2} - Path to requirements file, ${CSIT_DIR}/requirements.txt if empty.
 101     # Variables read:
 102     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 103     # Variables exported:
 104     # - PYTHONPATH - CSIT_DIR, as CSIT Python scripts usually need this.
 105     # Functions called:
 106     # - die - Print to stderr and exit.
 107
 108     # TODO: Do we want the callers to be able to set the env dir name?
 109     # TODO: + In that case, do we want to support env switching?
 110     # TODO:   + In that case we want to make env_dir global.
 111     # TODO: Do we want the callers to override PYTHONPATH loaction?
 112
 113     root_path="${1-$CSIT_DIR}"
 114     env_dir="${root_path}/env"
 115     req_path=${2-$CSIT_DIR/requirements.txt}
 116     rm -rf "${env_dir}" || die "Failed to clean previous virtualenv."
 117     pip install --upgrade virtualenv || {
 118         die "Virtualenv package install failed."
 119     }
 120     virtualenv "${env_dir}" || {
 121         die "Virtualenv creation failed."
 122     }
 123     set +u
 124     source "${env_dir}/bin/activate" || die "Virtualenv activation failed."
 125     set -u
 126     pip install --upgrade -r "${req_path}" || {
 127         die "Requirements installation failed."
 128     }
 129     # Most CSIT Python scripts assume PYTHONPATH is set and exported.
 130     export PYTHONPATH="${CSIT_DIR}" || die "Export failed."
 131 }
 132
 133
 134 function check_download_dir () {
 135
 136     set -exuo pipefail
 137
 138     # Fail if there are no files visible in ${DOWNLOAD_DIR}.
 139     # TODO: Do we need this as a function, if it is (almost) a one-liner?
 140     #
 141     # Variables read:
 142     # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
 143     # Directories read:
 144     # - ${DOWNLOAD_DIR} - Has to be non-empty to proceed.
 145     # Functions called:
 146     # - die - Print to stderr and exit.
 147
 148     if [[ ! "$(ls -A "${DOWNLOAD_DIR}")" ]]; then
 149         die "No artifacts downloaded!"
 150     fi
 151 }
 152
 153
 154 function cleanup_topo () {
 155
 156     set -exuo pipefail
 157
 158     # Variables read:
 159     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 160     # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
 161
 162     python "${PYTHON_SCRIPTS_DIR}/topo_cleanup.py" -t "${WORKING_TOPOLOGY}"
 163     # Not using "|| die" as some callers might want to ignore errors,
 164     # e.g. in teardowns, such as unreserve.
 165 }
 166
 167
 168 function common_dirs () {
 169
 170     set -exuo pipefail
 171
 172     # Variables set:
 173     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
 174     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 175     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
 176     # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources".
 177     # - TOOLS_DIR - Path to existing resources subdirectory "tools".
 178     # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".
 179     # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive".
 180     # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
 181     # Functions called:
 182     # - die - Print to stderr and exit.
 183
 184     BASH_FUNCTION_DIR="$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")" || {
 185         die "Some error during localizing this source directory."
 186     }
 187     # Current working directory could be in a different repo, e.g. VPP.
 188     pushd "${BASH_FUNCTION_DIR}" || die "Pushd failed"
 189     CSIT_DIR="$(readlink -e "$(git rev-parse --show-toplevel)")" || {
 190         die "Readlink or git rev-parse failed."
 191     }
 192     popd || die "Popd failed."
 193     TOPOLOGIES_DIR="$(readlink -e "${CSIT_DIR}/topologies/available")" || {
 194         die "Readlink failed."
 195     }
 196     RESOURCES_DIR="$(readlink -e "${CSIT_DIR}/resources")" || {
 197         die "Readlink failed."
 198     }
 199     TOOLS_DIR="$(readlink -e "${RESOURCES_DIR}/tools")" || {
 200         die "Readlink failed."
 201     }
 202     PYTHON_SCRIPTS_DIR="$(readlink -e "${TOOLS_DIR}/scripts")" || {
 203         die "Readlink failed."
 204     }
 205
 206     ARCHIVE_DIR="$(readlink -f "${CSIT_DIR}/archive")" || {
 207         die "Readlink failed."
 208     }
 209     mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed."
 210     DOWNLOAD_DIR="$(readlink -f "${CSIT_DIR}/download_dir")" || {
 211         die "Readlink failed."
 212     }
 213     mkdir -p "${DOWNLOAD_DIR}" || die "Mkdir failed."
 214 }
 215
 216
 217 function compose_pybot_arguments () {
 218
 219     set -exuo pipefail
 220
 221     # Variables read:
 222     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 223     # - DUT - CSIT test/ subdirectory, set while processing tags.
 224     # - TAGS - Array variable holding selected tag boolean expressions.
 225     # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology.
 226     # - TEST_CODE - The test selection string from environment or argument.
 227     # Variables set:
 228     # - PYBOT_ARGS - String holding part of all arguments for pybot.
 229     # - EXPANDED_TAGS - Array of strings pybot arguments compiled from tags.
 230
 231     # No explicit check needed with "set -u".
 232     PYBOT_ARGS=("--loglevel" "TRACE")
 233     PYBOT_ARGS+=("--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}")
 234
 235     case "${TEST_CODE}" in
 236         *"device"*)
 237             PYBOT_ARGS+=("--suite" "tests.${DUT}.device")
 238             ;;
 239         *"func"*)
 240             PYBOT_ARGS+=("--suite" "tests.${DUT}.func")
 241             ;;
 242         *"perf"*)
 243             PYBOT_ARGS+=("--suite" "tests.${DUT}.perf")
 244             ;;
 245         *)
 246             die "Unknown specification: ${TEST_CODE}"
 247     esac
 248
 249     EXPANDED_TAGS=()
 250     for tag in "${TAGS[@]}"; do
 251         if [[ ${tag} == "!"* ]]; then
 252             EXPANDED_TAGS+=("--exclude" "${tag#$"!"}")
 253         else
 254             EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}")
 255         fi
 256     done
 257 }
 258
 259
 260 function copy_archives () {
 261
 262     set -exuo pipefail
 263
 264     # Variables read:
 265     # - WORKSPACE - Jenkins workspace, copy only if the value is not empty.
 266     #   Can be unset, then it speeds up manual testing.
 267     # - ARCHIVE_DIR - Path to directory with content to be copied.
 268     # Directories updated:
 269     # - ${WORKSPACE}/archives/ - Created if does not exist.
 270     #   Content of ${ARCHIVE_DIR}/ is copied here.
 271     # Functions called:
 272     # - die - Print to stderr and exit.
 273
 274     # We will create additional archive if workspace variable is set.
 275     # This way if script is running in jenkins all will be
 276     # automatically archived to logs.fd.io.
 277     if [[ -n "${WORKSPACE-}" ]]; then
 278         mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed."
 279         cp -rf "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed."
 280     fi
 281 }
 282
 283
 284 function deactivate_docker_topology () {
 285     # Deactivate virtual vpp-device topology by removing containers.
 286     #
 287     # Variables read:
 288     # - NODENESS - Node multiplicity of desired testbed.
 289     # - FLAVOR - Node flavor string, usually describing the processor.
 290
 291     set -exuo pipefail
 292
 293     case_text="${NODENESS}_${FLAVOR}"
 294     case "${case_text}" in
 295         "1n_skx")
 296             hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3)
 297             ssh="ssh root@${hostname} -p 6022"
 298             env_vars="$(env | grep CSIT_ | tr '\n' ' ' )"
 299             ${ssh} "$(declare -f); deactivate_wrapper ${env_vars}" || {
 300                 die "Topology cleanup via shim-dcr failed!"
 301             }
 302             ;;
 303         "1n_vbox")
 304             enter_mutex || die
 305             clean_environment || {
 306                 die "Topology cleanup locally failed!"
 307             }
 308             exit_mutex || die
 309             ;;
 310         *)
 311             die "Unknown specification: ${case_text}!"
 312     esac
 313 }
 314
 315
 316 function die () {
 317     # Print the message to standard error end exit with error code specified
 318     # by the second argument.
 319     #
 320     # Hardcoded values:
 321     # - The default error message.
 322     # Arguments:
 323     # - ${1} - The whole error message, be sure to quote. Optional
 324     # - ${2} - the code to exit with, default: 1.
 325
 326     set -x
 327     set +eu
 328     warn "${1:-Unspecified run-time error occurred!}"
 329     exit "${2:-1}"
 330 }
 331
 332
 333 function die_on_pybot_error () {
 334
 335     set -exuo pipefail
 336
 337     # Source this fragment if you want to abort on any failed test case.
 338     #
 339     # Variables read:
 340     # - PYBOT_EXIT_STATUS - Set by a pybot running fragment.
 341     # Functions called:
 342     # - die - Print to stderr and exit.
 343
 344     if [[ "${PYBOT_EXIT_STATUS}" != "0" ]]; then
 345         die "${PYBOT_EXIT_STATUS}" "Test failures are present!"
 346     fi
 347 }
 348
 349
 350 function get_test_code () {
 351
 352     set -exuo pipefail
 353
 354     # Arguments:
 355     # - ${1} - Optional, argument of entry script (or empty as unset).
 356     #   Test code value to override job name from environment.
 357     # Variables read:
 358     # - JOB_NAME - String affecting test selection, default if not argument.
 359     # Variables set:
 360     # - TEST_CODE - The test selection string from environment or argument.
 361     # - NODENESS - Node multiplicity of desired testbed.
 362     # - FLAVOR - Node flavor string, usually describing the processor.
 363
 364     TEST_CODE="${1-}" || die "Reading optional argument failed, somehow."
 365     if [[ -z "${TEST_CODE}" ]]; then
 366         TEST_CODE="${JOB_NAME-}" || die "Reading job name failed, somehow."
 367     fi
 368
 369     case "${TEST_CODE}" in
 370         *"1n-vbox"*)
 371             NODENESS="1n"
 372             FLAVOR="vbox"
 373             ;;
 374         *"1n-skx"*)
 375             NODENESS="1n"
 376             FLAVOR="skx"
 377             ;;
 378         *"2n-skx"*)
 379             NODENESS="2n"
 380             FLAVOR="skx"
 381             ;;
 382         *"3n-skx"*)
 383             NODENESS="3n"
 384             FLAVOR="skx"
 385             ;;
 386         *"3n-tsh"*)
 387             NODENESS="3n"
 388             FLAVOR="tsh"
 389             ;;
 390         *)
 391             # Fallback to 3-node Haswell by default (backward compatibility)
 392             NODENESS="3n"
 393             FLAVOR="hsw"
 394             ;;
 395     esac
 396 }
 397
 398
 399 function get_test_tag_string () {
 400
 401     set -exuo pipefail
 402
 403     # Variables read:
 404     # - GERRIT_EVENT_TYPE - Event type set by gerrit, can be unset.
 405     # - GERRIT_EVENT_COMMENT_TEXT - Comment text, read for "comment-added" type.
 406     # - TEST_CODE - The test selection string from environment or argument.
 407     # Variables set:
 408     # - TEST_TAG_STRING - The string following "perftest" in gerrit comment,
 409     #   or empty.
 410
 411     # TODO: ci-management scripts no longer need to perform this.
 412
 413     trigger=""
 414     if [[ "${GERRIT_EVENT_TYPE-}" == "comment-added" ]]; then
 415         case "${TEST_CODE}" in
 416             *"device"*)
 417                 # On parsing error, ${trigger} stays empty.
 418                 trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \
 419                     | grep -oE '(devicetest$|devicetest[[:space:]].+$)')" \
 420                     || true
 421                 # Set test tags as string.
 422                 TEST_TAG_STRING="${trigger#$"devicetest"}"
 423                 ;;
 424             *"perf"*)
 425                 # On parsing error, ${trigger} stays empty.
 426                 comment="${GERRIT_EVENT_COMMENT_TEXT}"
 427                 # As "perftest" can be followed by something, we substitute it.
 428                 comment="${comment/perftest-2n/perftest}"
 429                 comment="${comment/perftest-3n/perftest}"
 430                 comment="${comment/perftest-hsw/perftest}"
 431                 comment="${comment/perftest-skx/perftest}"
 432                 comment="${comment/perftest-tsh/perftest}"
 433                 tag_string="$(echo "${comment}" \
 434                     | grep -oE '(perftest$|perftest[[:space:]].+$)' || true)"
 435                 # Set test tags as string.
 436                 TEST_TAG_STRING="${tag_string#$"perftest"}"
 437                 ;;
 438             *)
 439                 die "Unknown specification: ${TEST_CODE}"
 440         esac
 441     fi
 442 }
 443
 444
 445 function reserve_testbed () {
 446
 447     set -exuo pipefail
 448
 449     # Reserve physical testbed, perform cleanup, register trap to unreserve.
 450     #
 451     # Variables read:
 452     # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on.
 453     # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
 454     # Variables set:
 455     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 456     # Functions called:
 457     # - die - Print to stderr and exit.
 458     # Traps registered:
 459     # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
 460
 461     while true; do
 462         for topo in "${TOPOLOGIES[@]}"; do
 463             set +e
 464             python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}"
 465             result="$?"
 466             set -e
 467             if [[ "${result}" == "0" ]]; then
 468                 WORKING_TOPOLOGY="${topo}"
 469                 echo "Reserved: ${WORKING_TOPOLOGY}"
 470                 trap "untrap_and_unreserve_testbed" EXIT || {
 471                     message="TRAP ATTEMPT AND UNRESERVE FAILED, FIX MANUALLY."
 472                     untrap_and_unreserve_testbed "${message}" || {
 473                         die "Teardown should have died, not failed."
 474                     }
 475                     die "Trap attempt failed, unreserve succeeded. Aborting."
 476                 }
 477                 cleanup_topo || {
 478                     die "Testbed cleanup failed."
 479                 }
 480                 break
 481             fi
 482         done
 483
 484         if [[ -n "${WORKING_TOPOLOGY-}" ]]; then
 485             # Exit the infinite while loop if we made a reservation.
 486             break
 487         fi
 488
 489         # Wait ~3minutes before next try.
 490         sleep_time="$[ ( $RANDOM % 20 ) + 180 ]s" || {
 491             die "Sleep time calculation failed."
 492         }
 493         echo "Sleeping ${sleep_time}"
 494         sleep "${sleep_time}" || die "Sleep failed."
 495     done
 496 }
 497
 498
 499 function run_pybot () {
 500
 501     set -exuo pipefail
 502
 503     # Currently, VPP-1361 causes occasional test failures.
 504     # If real result is more important than time, we can retry few times.
 505     # TODO: We should be retrying on test case level instead.
 506
 507     # Arguments:
 508     # - ${1} - Optional number of pybot invocations to try to avoid failures.
 509     #   Default: 1.
 510     # Variables read:
 511     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 512     # - ARCHIVE_DIR - Path to store robot result files in.
 513     # - PYBOT_ARGS, EXPANDED_TAGS - See compose_pybot_arguments.sh
 514     # Variables set:
 515     # - PYBOT_EXIT_STATUS - Exit status of most recent pybot invocation.
 516     # Functions called:
 517     # - die - Print to stderr and exit.
 518
 519     # Set ${tries} as an integer variable, to fail on non-numeric input.
 520     local -i "tries" || die "Setting type of variable failed."
 521     tries="${1:-1}" || die "Argument evaluation failed."
 522     all_options=("--outputdir" "${ARCHIVE_DIR}" "${PYBOT_ARGS[@]}")
 523     all_options+=("${EXPANDED_TAGS[@]}")
 524
 525     while true; do
 526         if [[ "${tries}" -le 0 ]]; then
 527             break
 528         else
 529             tries="$((${tries} - 1))"
 530         fi
 531         pushd "${CSIT_DIR}" || die "Change directory operation failed."
 532         set +e
 533         # TODO: Make robot tests not require "$(pwd)" == "${CSIT_DIR}".
 534         pybot "${all_options[@]}" "${CSIT_DIR}/tests/"
 535         PYBOT_EXIT_STATUS="$?"
 536         set -e
 537         popd || die "Change directory operation failed."
 538         if [[ "${PYBOT_EXIT_STATUS}" == "0" ]]; then
 539             break
 540         fi
 541     done
 542 }
 543
 544
 545 function select_tags () {
 546
 547     set -exuo pipefail
 548
 549     # Variables read:
 550     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 551     # - TEST_CODE - String affecting test selection, usually jenkins job name.
 552     # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
 553     #   Can be unset.
 554     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
 555     # Variables set:
 556     # - TAGS - Array of processed tag boolean expressions.
 557
 558     # NIC SELECTION
 559     # All topologies NICs
 560     available=$(grep -hoPR "model: \K.*" "${TOPOLOGIES_DIR}"/* | sort -u)
 561     # Selected topology NICs
 562     reserved=$(grep -hoPR "model: \K.*" "${WORKING_TOPOLOGY}" | sort -u)
 563     # All topologies NICs - Selected topology NICs
 564     exclude_nics=($(comm -13 <(echo "${reserved}") <(echo "${available}")))
 565
 566     # Select default NIC
 567     case "${TEST_CODE}" in
 568         *"3n-tsh"*)
 569             DEFAULT_NIC='nic_intel-82599es'
 570             ;;
 571         *)
 572             DEFAULT_NIC='nic_intel-x710'
 573             ;;
 574     esac
 575
 576     case "${TEST_CODE}" in
 577         # Select specific performance tests based on jenkins job type variable.
 578         *"ndrpdr-weekly"* )
 579             test_tag_array=("ndrpdrAND64bAND1c"
 580                             "ndrpdrAND78bAND1c")
 581             ;;
 582         *"mrr-daily"* )
 583             test_tag_array=(# vic
 584                             "mrrANDnic_cisco-vic-1227AND64b"
 585                             "mrrANDnic_cisco-vic-1385AND64b"
 586                             # memif
 587                             "mrrANDmemifANDethAND64b"
 588                             "mrrANDmemifANDethANDimix"
 589                             # crypto
 590                             "mrrANDipsecAND64b"
 591                             # ip4 base
 592                             "mrrANDip4baseAND64b"
 593                             # ip4 scale FIB 2M
 594                             "mrrANDip4fwdANDfib_2mAND64b"
 595                             # ip4 scale FIB 200k
 596                             "mrrANDip4fwdANDfib_200kANDnic_intel-*710AND64b"
 597                             # ip4 scale FIB 20k
 598                             "mrrANDip4fwdANDfib_20kANDnic_intel-*710AND64b"
 599                             # ip4 scale ACL
 600                             "mrrANDip4fwdANDacl1AND10k_flowsAND64b"
 601                             "mrrANDip4fwdANDacl50AND10k_flowsAND64b"
 602                             # ip4 scale NAT44
 603                             "mrrANDip4fwdANDnat44ANDbaseAND64b"
 604                             "mrrANDip4fwdANDnat44ANDsrc_user_4000AND64b"
 605                             # ip4 features
 606                             "mrrANDip4fwdANDfeatureANDnic_intel-*710AND64b"
 607                             # TODO: Remove when tags in
 608                             # tests/vpp/perf/ip4/*-ipolicemarkbase-*.robot
 609                             # are fixed
 610                             "mrrANDip4fwdANDpolice_markANDnic_intel-*710AND64b"
 611                             # ip4 tunnels
 612                             "mrrANDip4fwdANDencapANDip6unrlayANDip4ovrlayANDnic_intel-x520-da2AND64b"
 613                             "mrrANDip4fwdANDencapANDnic_intel-*710AND64b"
 614                             "mrrANDl2ovrlayANDencapANDnic_intel-*710AND64b"
 615                             # ip6 base
 616                             "mrrANDip6baseANDethAND78b"
 617                             # ip6 features
 618                             "mrrANDip6fwdANDfeatureANDnic_intel-*710AND78b"
 619                             # ip6 scale FIB 2M
 620                             "mrrANDip6fwdANDfib_2mANDnic_intel-*710AND78b"
 621                             # ip6 scale FIB 200k
 622                             "mrrANDip6fwdANDfib_200kANDnic_intel-*710AND78b"
 623                             # ip6 scale FIB 20k
 624                             "mrrANDip6fwdANDfib_20kANDnic_intel-*710AND78b"
 625                             # ip6 tunnels
 626                             "mrrANDip6fwdANDencapANDnic_intel-x520-da2AND78b"
 627                             # l2xc base
 628                             "mrrANDl2xcfwdANDbaseAND64b"
 629                             # l2xc scale ACL
 630                             "mrrANDl2xcANDacl1AND10k_flowsAND64b"
 631                             "mrrANDl2xcANDacl50AND10k_flowsAND64b"
 632                             # l2xc scale FIB 2M
 633                             "mrrANDl2xcANDfib_2mAND64b"
 634                             # l2xc scale FIB 200k
 635                             "mrrANDl2xcANDfib_200kANDnic_intel-*710AND64b"
 636                             # l2xc scale FIB 20k
 637                             "mrrANDl2xcANDfib_20kANDnic_intel-*710AND64b"
 638                             # l2bd base
 639                             "mrrANDl2bdmaclrnANDbaseAND64b"
 640                             # l2bd scale ACL
 641                             "mrrANDl2bdmaclrnANDacl1AND10k_flowsAND64b"
 642                             "mrrANDl2bdmaclrnANDacl50AND10k_flowsAND64b"
 643                             # l2bd scale FIB 2M
 644                             "mrrANDl2bdmaclrnANDfib_1mAND64b"
 645                             # l2bd scale FIB 200k
 646                             "mrrANDl2bdmaclrnANDfib_100kANDnic_intel-*710AND64b"
 647                             # l2bd scale FIB 20k
 648                             "mrrANDl2bdmaclrnANDfib_10kANDnic_intel-*710AND64b"
 649                             # l2 patch base
 650                             "mrrANDl2patchAND64b"
 651                             # srv6
 652                             "mrrANDsrv6ANDnic_intel-x520-da2AND78b"
 653                             # vts
 654                             "mrrANDvtsANDnic_intel-x520-da2AND114b"
 655                             # vm vhost l2xc base
 656                             "mrrANDvhostANDl2xcfwdANDbaseAND64b"
 657                             "mrrANDvhostANDl2xcfwdANDbaseANDimix"
 658                             # vm vhost l2bd base
 659                             "mrrANDvhostANDl2bdmaclrnANDbaseAND64b"
 660                             "mrrANDvhostANDl2bdmaclrnANDbaseANDimix"
 661                             # vm vhost ip4 base
 662                             "mrrANDvhostANDip4fwdANDbaseAND64b"
 663                             "mrrANDvhostANDip4fwdANDbaseANDimix"
 664                             # DPDK
 665                             "mrrANDdpdkAND64b"
 666                             # Exclude
 667                             "!mrrANDip6baseANDdot1qAND78b"
 668                             "!vhost_256ANDnic_intel-x520-da2"
 669                             "!vhostANDnic_intel-xl710"
 670                             "!cfs_opt"
 671                             "!lbond_dpdk"
 672                             "!nf_density")
 673             ;;
 674         *"mrr-weekly"* )
 675             test_tag_array=(# NF Density tests
 676                             "mrrANDnf_densityAND64b"
 677                             "mrrANDnf_densityANDimix"
 678                             # DPDK
 679                             "mrrANDdpdkAND64b")
 680             ;;
 681         * )
 682             if [[ -z "${TEST_TAG_STRING-}" ]]; then
 683                 # If nothing is specified, we will run pre-selected tests by
 684                 # following tags.
 685                 test_tag_array=("mrrAND${DEFAULT_NIC}AND1cAND64bANDip4base"
 686                                 "mrrAND${DEFAULT_NIC}AND1cAND78bANDip6base"
 687                                 "mrrAND${DEFAULT_NIC}AND1cAND64bANDl2bdbase"
 688                                 "mrrAND${DEFAULT_NIC}AND1cAND64bANDl2xcbase"
 689                                 "!dot1q")
 690             else
 691                 # If trigger contains tags, split them into array.
 692                 test_tag_array=(${TEST_TAG_STRING//:/ })
 693             fi
 694             ;;
 695     esac
 696
 697     # Blacklisting certain tags per topology.
 698     case "${TEST_CODE}" in
 699         *"3n-hsw"*)
 700             test_tag_array+=("!drv_avf")
 701             ;;
 702         *"2n-skx"*)
 703             test_tag_array+=("!ipsechw")
 704             ;;
 705         *"3n-skx"*)
 706             test_tag_array+=("!ipsechw")
 707             ;;
 708         *"3n-tsh"*)
 709             test_tag_array+=("!ipsechw")
 710             ;;
 711         *)
 712             # Default to 3n-hsw due to compatibility.
 713             test_tag_array+=("!drv_avf")
 714             ;;
 715     esac
 716
 717     # We will add excluded NICs.
 718     test_tag_array+=("${exclude_nics[@]/#/!NIC_}")
 719
 720     TAGS=()
 721
 722     # We will prefix with perftest to prevent running other tests
 723     # (e.g. Functional).
 724     prefix="perftestAND"
 725     if [[ "${TEST_CODE}" == "vpp-"* ]]; then
 726         # Automatic prefixing for VPP jobs to limit the NIC used and
 727         # traffic evaluation to MRR.
 728         prefix="${prefix}mrrAND${DEFAULT_NIC}AND"
 729     fi
 730     for tag in "${test_tag_array[@]}"; do
 731         if [[ ${tag} == "!"* ]]; then
 732             # Exclude tags are not prefixed.
 733             TAGS+=("${tag}")
 734         else
 735             TAGS+=("${prefix}${tag}")
 736         fi
 737     done
 738 }
 739
 740
 741 function select_vpp_device_tags () {
 742
 743     set -exuo pipefail
 744
 745     # Variables read:
 746     # - TEST_CODE - String affecting test selection, usually jenkins job name.
 747     # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
 748     #   Can be unset.
 749     # Variables set:
 750     # - TAGS - Array of processed tag boolean expressions.
 751
 752     case "${TEST_CODE}" in
 753         # Select specific performance tests based on jenkins job type variable.
 754         * )
 755             if [[ -z "${TEST_TAG_STRING-}" ]]; then
 756                 # If nothing is specified, we will run pre-selected tests by
 757                 # following tags. Items of array will be concatenated by OR
 758                 # in Robot Framework.
 759                 test_tag_array=()
 760             else
 761                 # If trigger contains tags, split them into array.
 762                 test_tag_array=(${TEST_TAG_STRING//:/ })
 763             fi
 764             ;;
 765     esac
 766
 767     TAGS=()
 768
 769     # We will prefix with perftest to prevent running other tests
 770     # (e.g. Functional).
 771     prefix="devicetestAND"
 772     if [[ "${TEST_CODE}" == "vpp-"* ]]; then
 773         # Automatic prefixing for VPP jobs to limit testing.
 774         prefix="${prefix}"
 775     fi
 776     for tag in "${test_tag_array[@]}"; do
 777         if [[ ${tag} == "!"* ]]; then
 778             # Exclude tags are not prefixed.
 779             TAGS+=("${tag}")
 780         else
 781             TAGS+=("${prefix}${tag}")
 782         fi
 783     done
 784 }
 785
 786
 787 function select_topology () {
 788
 789     set -exuo pipefail
 790
 791     # Variables read:
 792     # - NODENESS - Node multiplicity of testbed, either "2n" or "3n".
 793     # - FLAVOR - Node flavor string, currently either "hsw" or "skx".
 794     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 795     # - TOPOLOGIES_DIR - Path to existing directory with available topologies.
 796     # Variables set:
 797     # - TOPOLOGIES - Array of paths to suitable topology yaml files.
 798     # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology.
 799     # Functions called:
 800     # - die - Print to stderr and exit.
 801
 802     case_text="${NODENESS}_${FLAVOR}"
 803     case "${case_text}" in
 804         "1n_vbox")
 805             TOPOLOGIES=(
 806                         "${TOPOLOGIES_DIR}/vpp_device.template"
 807                        )
 808             TOPOLOGIES_TAGS="2_node_single_link_topo"
 809             ;;
 810         "1n_skx")
 811             TOPOLOGIES=(
 812                         "${TOPOLOGIES_DIR}/vpp_device.template"
 813                        )
 814             TOPOLOGIES_TAGS="2_node_single_link_topo"
 815             ;;
 816         "2n_skx")
 817             TOPOLOGIES=(
 818                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed21.yaml"
 819                         #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed22.yaml"
 820                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed23.yaml"
 821                         "${TOPOLOGIES_DIR}/lf_2n_skx_testbed24.yaml"
 822                        )
 823             TOPOLOGIES_TAGS="2_node_*_link_topo"
 824             ;;
 825         "3n_skx")
 826             TOPOLOGIES=(
 827                         "${TOPOLOGIES_DIR}/lf_3n_skx_testbed31.yaml"
 828                         "${TOPOLOGIES_DIR}/lf_3n_skx_testbed32.yaml"
 829                        )
 830             TOPOLOGIES_TAGS="3_node_*_link_topo"
 831             ;;
 832         "3n_hsw")
 833             TOPOLOGIES=(
 834                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml"
 835                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml"
 836                         "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml"
 837                        )
 838             TOPOLOGIES_TAGS="3_node_single_link_topo"
 839             ;;
 840         "3n_tsh")
 841             TOPOLOGIES=(
 842                         "${TOPOLOGIES_DIR}/lf_3n_tsh_testbed33.yaml"
 843                        )
 844             TOPOLOGIES_TAGS="3_node_*_link_topo"
 845             ;;
 846         *)
 847             # No falling back to 3n_hsw default, that should have been done
 848             # by the function which has set NODENESS and FLAVOR.
 849             die "Unknown specification: ${case_text}"
 850     esac
 851
 852     if [[ -z "${TOPOLOGIES-}" ]]; then
 853         die "No applicable topology found!"
 854     fi
 855 }
 856
 857
 858 function untrap_and_unreserve_testbed () {
 859     # Use this as a trap function to ensure testbed does not remain reserved.
 860     # Perhaps call directly before script exit, to free testbed for other jobs.
 861     # This function is smart enough to avoid multiple unreservations (so safe).
 862     # Topo cleanup is executed (call it best practice), ignoring failures.
 863     #
 864     # Hardcoded values:
 865     # - default message to die with if testbed might remain reserved.
 866     # Arguments:
 867     # - ${1} - Message to die with if unreservation fails. Default hardcoded.
 868     # Variables read (by inner function):
 869     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 870     # - PYTHON_SCRIPTS_DIR - Path to directory holding Python scripts.
 871     # Variables written:
 872     # - WORKING_TOPOLOGY - Set to empty string on successful unreservation.
 873     # Trap unregistered:
 874     # - EXIT - Failure to untrap is reported, but ignored otherwise.
 875     # Functions called:
 876     # - die - Print to stderr and exit.
 877
 878     set -xo pipefail
 879     set +eu  # We do not want to exit early in a "teardown" function.
 880     trap - EXIT || echo "Trap deactivation failed, continuing anyway."
 881     wt="${WORKING_TOPOLOGY}"  # Just to avoid too long lines.
 882     if [[ -z "${wt-}" ]]; then
 883         set -eu
 884         warn "Testbed looks unreserved already. Trap removal failed before?"
 885     else
 886         cleanup_topo || true
 887         python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || {
 888             die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2
 889         }
 890         WORKING_TOPOLOGY=""
 891         set -eu
 892     fi
 893 }
 894
 895
 896 function warn () {
 897     # Print the message to standard error.
 898     #
 899     # Arguments:
 900     # - ${@} - The text of the message.
 901
 902     echo "$@" >&2
 903 }