resources/libraries/bash/function/common.sh

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Copyright (c) 2023 PANTHEON.tech and/or its affiliates.
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at:
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 set -exuo pipefail
  16
  17 # This library defines functions used by multiple entry scripts.
  18 # Keep functions ordered alphabetically, please.
  19
  20 # TODO: Add a link to bash style guide.
  21 # TODO: Consider putting every die into a {} block,
  22 #   the code might become more readable (but longer).
  23
  24
  25 function activate_docker_topology () {
  26
  27     # Create virtual vpp-device topology. Output of the function is topology
  28     # file describing created environment saved to a file.
  29     #
  30     # Variables read:
  31     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
  32     # - TOPOLOGIES - Available topologies.
  33     # - NODENESS - Node multiplicity of desired testbed.
  34     # - FLAVOR - Node flavor string, usually describing the processor.
  35     # - IMAGE_VER_FILE - Name of file that contains the image version.
  36     # - CSIT_DIR - Directory where ${IMAGE_VER_FILE} is located.
  37     # Variables set:
  38     # - WORKING_TOPOLOGY - Path to topology file.
  39
  40     set -exuo pipefail
  41
  42     source "${BASH_FUNCTION_DIR}/device.sh" || {
  43         die "Source failed!"
  44     }
  45     device_image="$(< ${CSIT_DIR}/${IMAGE_VER_FILE})"
  46     case_text="${NODENESS}_${FLAVOR}"
  47     case "${case_text}" in
  48         "1n_skx" | "1n_tx2" | "1n_spr")
  49             # We execute reservation over csit-shim-dcr (ssh) which runs sourced
  50             # script's functions. Env variables are read from ssh output
  51             # back to localhost for further processing.
  52             # Shim and Jenkins executor are in the same network on the same host
  53             # Connect to docker's default gateway IP and shim's exposed port
  54             ssh="ssh root@172.17.0.1 -p 6022"
  55             run="activate_wrapper ${NODENESS} ${FLAVOR} ${device_image}"
  56             # The "declare -f" output is long and boring.
  57             set +x
  58             # backtics to avoid https://midnight-commander.org/ticket/2142
  59             env_vars=`${ssh} "$(declare -f); ${run}"` || {
  60                 die "Topology reservation via shim-dcr failed!"
  61             }
  62             set -x
  63             set -a
  64             source <(echo "$env_vars" | grep -v /usr/bin/docker) || {
  65                 die "Source failed!"
  66             }
  67             set +a
  68             ;;
  69         "1n_vbox")
  70             # We execute reservation on localhost. Sourced script automatially
  71             # sets environment variables for further processing.
  72             activate_wrapper "${NODENESS}" "${FLAVOR}" "${device_image}" || die
  73             ;;
  74         *)
  75             die "Unknown specification: ${case_text}!"
  76     esac
  77
  78     trap 'deactivate_docker_topology' EXIT || {
  79          die "Trap attempt failed, please cleanup manually. Aborting!"
  80     }
  81
  82     parse_env_variables || die "Parse of environment variables failed!"
  83
  84     # Replace all variables in template with those in environment.
  85     source <(echo 'cat <<EOF >topo.yml'; cat ${TOPOLOGIES[0]}; echo EOF;) || {
  86         die "Topology file create failed!"
  87     }
  88
  89     WORKING_TOPOLOGY="${CSIT_DIR}/topologies/available/vpp_device.yaml"
  90     mv topo.yml "${WORKING_TOPOLOGY}" || {
  91         die "Topology move failed!"
  92     }
  93     cat ${WORKING_TOPOLOGY} | grep -v password || {
  94         die "Topology read failed!"
  95     }
  96 }
  97
  98
  99 function activate_virtualenv () {
 100
 101     # Update virtualenv pip package, delete and create virtualenv directory,
 102     # activate the virtualenv, install requirements, set PYTHONPATH.
 103
 104     # Arguments:
 105     # - ${1} - Path to existing directory for creating virtualenv in.
 106     #          If missing or empty, ${CSIT_DIR} is used.
 107     # - ${2} - Path to requirements file, ${CSIT_DIR}/requirements.txt if empty.
 108     # Variables read:
 109     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 110     # Variables exported:
 111     # - PYTHONPATH - CSIT_DIR, as CSIT Python scripts usually need this.
 112     # Functions called:
 113     # - die - Print to stderr and exit.
 114
 115     set -exuo pipefail
 116
 117     root_path="${1-$CSIT_DIR}"
 118     env_dir="${root_path}/env"
 119     req_path=${2-$CSIT_DIR/requirements.txt}
 120     rm -rf "${env_dir}" || die "Failed to clean previous virtualenv."
 121     pip3 install virtualenv==20.15.1 || {
 122         die "Virtualenv package install failed."
 123     }
 124     virtualenv --no-download --python=$(which python3) "${env_dir}" || {
 125         die "Virtualenv creation for $(which python3) failed."
 126     }
 127     set +u
 128     source "${env_dir}/bin/activate" || die "Virtualenv activation failed."
 129     set -u
 130     pip3 install -r "${req_path}" || {
 131         die "Requirements installation failed."
 132     }
 133     # Most CSIT Python scripts assume PYTHONPATH is set and exported.
 134     export PYTHONPATH="${CSIT_DIR}" || die "Export failed."
 135 }
 136
 137
 138 function archive_tests () {
 139
 140     # Create .tar.gz of generated/tests for archiving.
 141     # To be run after generate_tests, kept separate to offer more flexibility.
 142
 143     # Directory read:
 144     # - ${GENERATED_DIR}/tests - Tree of executed suites to archive.
 145     # File rewriten:
 146     # - ${ARCHIVE_DIR}/generated_tests.tar.gz - Archive of generated tests.
 147
 148     set -exuo pipefail
 149
 150     pushd "${ARCHIVE_DIR}" || die
 151     tar czf "generated_tests.tar.gz" "${GENERATED_DIR}/tests" || true
 152     popd || die
 153 }
 154
 155
 156 function check_download_dir () {
 157
 158     # Fail if there are no files visible in ${DOWNLOAD_DIR}.
 159     #
 160     # Variables read:
 161     # - DOWNLOAD_DIR - Path to directory robot takes the build to test from.
 162     # Directories read:
 163     # - ${DOWNLOAD_DIR} - Has to be non-empty to proceed.
 164     # Functions called:
 165     # - die - Print to stderr and exit.
 166
 167     set -exuo pipefail
 168
 169     if [[ ! "$(ls -A "${DOWNLOAD_DIR}")" ]]; then
 170         die "No artifacts downloaded!"
 171     fi
 172 }
 173
 174
 175 function check_prerequisites () {
 176
 177     # Fail if prerequisites are not met.
 178     #
 179     # Functions called:
 180     # - installed - Check if application is installed/present in system.
 181     # - die - Print to stderr and exit.
 182
 183     set -exuo pipefail
 184
 185     if ! installed sshpass; then
 186         die "Please install sshpass before continue!"
 187     fi
 188 }
 189
 190
 191 function common_dirs () {
 192
 193     # Set global variables, create some directories (without touching content).
 194
 195     # Variables set:
 196     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
 197     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 198     # - TOPOLOGIES_DIR - Path to existing directory with available topologies.
 199     # - JOB_SPECS_DIR - Path to existing directory with job test specifications.
 200     # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources".
 201     # - TOOLS_DIR - Path to existing resources subdirectory "tools".
 202     # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".
 203     # - ARCHIVE_DIR - Path to created CSIT subdirectory "archives".
 204     #   The name is chosen to match what ci-management expects.
 205     # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
 206     # - GENERATED_DIR - Path to created CSIT subdirectory "generated".
 207     # Directories created if not present:
 208     # ARCHIVE_DIR, DOWNLOAD_DIR, GENERATED_DIR.
 209     # Functions called:
 210     # - die - Print to stderr and exit.
 211
 212     set -exuo pipefail
 213
 214     this_file=$(readlink -e "${BASH_SOURCE[0]}") || {
 215         die "Some error during locating of this source file."
 216     }
 217     BASH_FUNCTION_DIR=$(dirname "${this_file}") || {
 218         die "Some error during dirname call."
 219     }
 220     # Current working directory could be in a different repo, e.g. VPP.
 221     pushd "${BASH_FUNCTION_DIR}" || die "Pushd failed"
 222     relative_csit_dir=$(git rev-parse --show-toplevel) || {
 223         die "Git rev-parse failed."
 224     }
 225     CSIT_DIR=$(readlink -e "${relative_csit_dir}") || die "Readlink failed."
 226     popd || die "Popd failed."
 227     TOPOLOGIES_DIR=$(readlink -e "${CSIT_DIR}/topologies/available") || {
 228         die "Readlink failed."
 229     }
 230     JOB_SPECS_DIR=$(readlink -e "${CSIT_DIR}/resources/job_specs") || {
 231         die "Readlink failed."
 232     }
 233     RESOURCES_DIR=$(readlink -e "${CSIT_DIR}/resources") || {
 234         die "Readlink failed."
 235     }
 236     TOOLS_DIR=$(readlink -e "${RESOURCES_DIR}/tools") || {
 237         die "Readlink failed."
 238     }
 239     PYTHON_SCRIPTS_DIR=$(readlink -e "${TOOLS_DIR}/scripts") || {
 240         die "Readlink failed."
 241     }
 242
 243     ARCHIVE_DIR=$(readlink -f "${CSIT_DIR}/archives") || {
 244         die "Readlink failed."
 245     }
 246     mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed."
 247     DOWNLOAD_DIR=$(readlink -f "${CSIT_DIR}/download_dir") || {
 248         die "Readlink failed."
 249     }
 250     mkdir -p "${DOWNLOAD_DIR}" || die "Mkdir failed."
 251     GENERATED_DIR=$(readlink -f "${CSIT_DIR}/generated") || {
 252         die "Readlink failed."
 253     }
 254     mkdir -p "${GENERATED_DIR}" || die "Mkdir failed."
 255 }
 256
 257
 258 function compose_robot_arguments () {
 259
 260     # Variables read:
 261     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 262     # - DUT - CSIT test/ subdirectory, set while processing tags.
 263     # - TAGS - Array variable holding selected tag boolean expressions.
 264     # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology.
 265     # - TEST_CODE - The test selection string from environment or argument.
 266     # - SELECTION_MODE - Selection criteria [test, suite, include, exclude].
 267     # Variables set:
 268     # - ROBOT_ARGS - String holding part of all arguments for robot.
 269     # - EXPANDED_TAGS - Array of strings robot arguments compiled from tags.
 270
 271     set -exuo pipefail
 272
 273     # No explicit check needed with "set -u".
 274     ROBOT_ARGS=("--loglevel" "TRACE")
 275     ROBOT_ARGS+=("--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}")
 276
 277     case "${TEST_CODE}" in
 278         *"device"*)
 279             ROBOT_ARGS+=("--suite" "tests.${DUT}.device")
 280             ;;
 281         *"perf"*)
 282             ROBOT_ARGS+=("--suite" "tests.${DUT}.perf")
 283             ;;
 284         *)
 285             die "Unknown specification: ${TEST_CODE}"
 286     esac
 287
 288     EXPANDED_TAGS=()
 289     for tag in "${TAGS[@]}"; do
 290         if [[ ${tag} == "!"* ]]; then
 291             EXPANDED_TAGS+=("--exclude" "${tag#$"!"}")
 292         else
 293             if [[ ${SELECTION_MODE} == "--test" ]]; then
 294                 EXPANDED_TAGS+=("--test" "${tag}")
 295             else
 296                 EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}")
 297             fi
 298         fi
 299     done
 300
 301     if [[ ${SELECTION_MODE} == "--test" ]]; then
 302         EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}")
 303     fi
 304 }
 305
 306
 307 function deactivate_docker_topology () {
 308
 309     # Deactivate virtual vpp-device topology by removing containers.
 310     #
 311     # Variables read:
 312     # - NODENESS - Node multiplicity of desired testbed.
 313     # - FLAVOR - Node flavor string, usually describing the processor.
 314     # - CSIT_NO_CLEANUP - Variable to disable cleaning up the environment.
 315
 316     set -exuo pipefail
 317
 318     if [[ ${CSIT_NO_CLEANUP:-0} -eq 0 ]]; then
 319         case_text="${NODENESS}_${FLAVOR}"
 320         case "${case_text}" in
 321             "1n_skx" | "1n_tx2" | "1n_spr")
 322                 ssh="ssh root@172.17.0.1 -p 6022"
 323                 env_vars=$(env | grep CSIT_ | tr '\n' ' ' ) || die
 324                 # The "declare -f" output is long and boring.
 325                 set +x
 326                 ${ssh} "$(declare -f); deactivate_wrapper ${env_vars}" || {
 327                     die "Topology cleanup via shim-dcr failed!"
 328                 }
 329                 set -x
 330                 ;;
 331             "1n_vbox")
 332                 enter_mutex || die
 333                 clean_environment || {
 334                     die "Topology cleanup locally failed!"
 335                 }
 336                 exit_mutex || die
 337                 ;;
 338             *)
 339                 die "Unknown specification: ${case_text}!"
 340         esac
 341     else
 342         echo "CSIT_NO_CLEANUP environment variable is set"
 343         echo "Environment Cleanup Abandoned"
 344     fi
 345 }
 346
 347
 348 function die () {
 349
 350     # Print the message to standard error end exit with error code specified
 351     # by the second argument.
 352     #
 353     # Hardcoded values:
 354     # - The default error message.
 355     # Arguments:
 356     # - ${1} - The whole error message, be sure to quote. Optional
 357     # - ${2} - the code to exit with, default: 1.
 358
 359     set -x
 360     set +eu
 361     warn "${1:-Unspecified run-time error occurred!}"
 362     exit "${2:-1}"
 363 }
 364
 365
 366 function die_on_robot_error () {
 367
 368     # Source this fragment if you want to abort on any failed test case.
 369     #
 370     # Variables read:
 371     # - ROBOT_EXIT_STATUS - Set by a robot running fragment.
 372     # Functions called:
 373     # - die - Print to stderr and exit.
 374
 375     set -exuo pipefail
 376
 377     if [[ "${ROBOT_EXIT_STATUS}" != "0" ]]; then
 378         die "Test failures are present!" "${ROBOT_EXIT_STATUS}"
 379     fi
 380 }
 381
 382
 383 function generate_tests () {
 384
 385     # Populate ${GENERATED_DIR}/tests based on ${CSIT_DIR}/tests/.
 386     # Any previously existing content of ${GENERATED_DIR}/tests is wiped before.
 387     # The generation is done by executing any *.py executable
 388     # within any subdirectory after copying.
 389
 390     # This is a separate function, because this code is called
 391     # both by autogen checker and entries calling run_robot.
 392
 393     # Directories read:
 394     # - ${CSIT_DIR}/tests - Used as templates for the generated tests.
 395     # Directories replaced:
 396     # - ${GENERATED_DIR}/tests - Overwritten by the generated tests.
 397
 398     set -exuo pipefail
 399
 400     rm -rf "${GENERATED_DIR}/tests" || die
 401     cp -r "${CSIT_DIR}/tests" "${GENERATED_DIR}/tests" || die
 402     cmd_line=("find" "${GENERATED_DIR}/tests" "-type" "f")
 403     cmd_line+=("-executable" "-name" "*.py")
 404     # We sort the directories, so log output can be compared between runs.
 405     file_list=$("${cmd_line[@]}" | sort) || die
 406
 407     for gen in ${file_list}; do
 408         directory="$(dirname "${gen}")" || die
 409         filename="$(basename "${gen}")" || die
 410         pushd "${directory}" || die
 411         ./"${filename}" || die
 412         popd || die
 413     done
 414 }
 415
 416
 417 function get_test_code () {
 418
 419     # Arguments:
 420     # - ${1} - Optional, argument of entry script (or empty as unset).
 421     #   Test code value to override job name from environment.
 422     # Variables read:
 423     # - JOB_NAME - String affecting test selection, default if not argument.
 424     # Variables set:
 425     # - TEST_CODE - The test selection string from environment or argument.
 426     # - NODENESS - Node multiplicity of desired testbed.
 427     # - FLAVOR - Node flavor string, usually describing the processor.
 428
 429     set -exuo pipefail
 430
 431     TEST_CODE="${1-}" || die "Reading optional argument failed, somehow."
 432     if [[ -z "${TEST_CODE}" ]]; then
 433         TEST_CODE="${JOB_NAME-}" || die "Reading job name failed, somehow."
 434     fi
 435
 436     case "${TEST_CODE}" in
 437         *"1n-vbox"*)
 438             NODENESS="1n"
 439             FLAVOR="vbox"
 440             ;;
 441         *"1n-skx"*)
 442             NODENESS="1n"
 443             FLAVOR="skx"
 444             ;;
 445         *"1n-spr"*)
 446             NODENESS="1n"
 447             FLAVOR="spr"
 448             ;;
 449         *"1n-tx2"*)
 450             NODENESS="1n"
 451             FLAVOR="tx2"
 452             ;;
 453         *"1n-aws"*)
 454             NODENESS="1n"
 455             FLAVOR="aws"
 456             ;;
 457         *"2n-aws"*)
 458             NODENESS="2n"
 459             FLAVOR="aws"
 460             ;;
 461         *"3n-aws"*)
 462             NODENESS="3n"
 463             FLAVOR="aws"
 464             ;;
 465         *"1n-c6gn"*)
 466             NODENESS="1n"
 467             FLAVOR="c6gn"
 468             ;;
 469         *"2n-c6gn"*)
 470             NODENESS="2n"
 471             FLAVOR="c6gn"
 472             ;;
 473         *"3n-c6gn"*)
 474             NODENESS="3n"
 475             FLAVOR="c6gn"
 476             ;;
 477         *"1n-c6in"*)
 478             NODENESS="1n"
 479             FLAVOR="c6in"
 480             ;;
 481         *"2n-c6in"*)
 482             NODENESS="2n"
 483             FLAVOR="c6in"
 484             ;;
 485         *"3n-c6in"*)
 486             NODENESS="3n"
 487             FLAVOR="c6in"
 488             ;;
 489         *"2n-zn2"*)
 490             NODENESS="2n"
 491             FLAVOR="zn2"
 492             ;;
 493         *"2n-clx"*)
 494             NODENESS="2n"
 495             FLAVOR="clx"
 496             ;;
 497         *"2n-icx"*)
 498             NODENESS="2n"
 499             FLAVOR="icx"
 500             ;;
 501         *"2n-spr"*)
 502             NODENESS="2n"
 503             FLAVOR="spr"
 504             ;;
 505         *"3n-icx"*)
 506             NODENESS="3n"
 507             FLAVOR="icx"
 508             ;;
 509         *"3na-spr"*)
 510             NODENESS="3na"
 511             FLAVOR="spr"
 512             ;;
 513         *"3nb-spr"*)
 514             NODENESS="3nb"
 515             FLAVOR="spr"
 516             ;;
 517         *"3n-snr"*)
 518             NODENESS="3n"
 519             FLAVOR="snr"
 520             ;;
 521         *"2n-tx2"*)
 522             NODENESS="2n"
 523             FLAVOR="tx2"
 524             ;;
 525         *"3n-tsh"*)
 526             NODENESS="3n"
 527             FLAVOR="tsh"
 528             ;;
 529         *"3n-alt"*)
 530             NODENESS="3n"
 531             FLAVOR="alt"
 532             ;;
 533     esac
 534 }
 535
 536
 537 function get_test_tag_string () {
 538
 539     # Variables read:
 540     # - GERRIT_EVENT_TYPE - Event type set by gerrit, can be unset.
 541     # - GERRIT_EVENT_COMMENT_TEXT - Comment text, read for "comment-added" type.
 542     # - TEST_CODE - The test selection string from environment or argument.
 543     # Variables set:
 544     # - TEST_TAG_STRING - The string following trigger word in gerrit comment.
 545     #   May be empty, or even not set on event types not adding comment.
 546     # Variables exported optionally:
 547     # - GRAPH_NODE_VARIANT - Node variant to test with, set if found in trigger.
 548
 549     # TODO: ci-management scripts no longer need to perform this.
 550
 551     set -exuo pipefail
 552
 553     if [[ "${GERRIT_EVENT_TYPE-}" == "comment-added" ]]; then
 554         case "${TEST_CODE}" in
 555             *"device"*)
 556                 trigger="devicetest"
 557                 ;;
 558             *"perf"*)
 559                 trigger="perftest"
 560                 ;;
 561             *)
 562                 die "Unknown specification: ${TEST_CODE}"
 563         esac
 564         # Ignore lines not containing the trigger word.
 565         comment=$(fgrep "${trigger}" <<< "${GERRIT_EVENT_COMMENT_TEXT}" || true)
 566         # The vpp-csit triggers trail stuff we are not interested in.
 567         # Removing them and trigger word: https://unix.stackexchange.com/a/13472
 568         # (except relying on \s whitespace, \S non-whitespace and . both).
 569         # The last string is concatenated, only the middle part is expanded.
 570         cmd=("grep" "-oP" '\S*'"${trigger}"'\S*\s\K.+$') || die "Unset trigger?"
 571         # On parsing error, TEST_TAG_STRING probably stays empty.
 572         TEST_TAG_STRING=$("${cmd[@]}" <<< "${comment}" || true)
 573         if [[ -z "${TEST_TAG_STRING-}" ]]; then
 574             # Probably we got a base64 encoded comment.
 575             comment="${GERRIT_EVENT_COMMENT_TEXT}"
 576             comment=$(base64 --decode <<< "${comment}" || true)
 577             comment=$(fgrep "${trigger}" <<< "${comment}" || true)
 578             TEST_TAG_STRING=$("${cmd[@]}" <<< "${comment}" || true)
 579         fi
 580         if [[ -n "${TEST_TAG_STRING-}" ]]; then
 581             test_tag_array=(${TEST_TAG_STRING})
 582             if [[ "${test_tag_array[0]}" == "icl" ]]; then
 583                 export GRAPH_NODE_VARIANT="icl"
 584                 TEST_TAG_STRING="${test_tag_array[@]:1}" || true
 585             elif [[ "${test_tag_array[0]}" == "skx" ]]; then
 586                 export GRAPH_NODE_VARIANT="skx"
 587                 TEST_TAG_STRING="${test_tag_array[@]:1}" || true
 588             fi
 589         fi
 590     fi
 591 }
 592
 593
 594 function installed () {
 595
 596     # Check if the given utility is installed. Fail if not installed.
 597     #
 598     # Duplicate of common.sh function, as this file is also used standalone.
 599     #
 600     # Arguments:
 601     # - ${1} - Utility to check.
 602     # Returns:
 603     # - 0 - If command is installed.
 604     # - 1 - If command is not installed.
 605
 606     set -exuo pipefail
 607
 608     command -v "${1}"
 609 }
 610
 611
 612 function move_archives () {
 613
 614     # Move archive directory to top of workspace, if not already there.
 615     #
 616     # ARCHIVE_DIR is positioned relative to CSIT_DIR,
 617     # but in some jobs CSIT_DIR is not same as WORKSPACE
 618     # (e.g. under VPP_DIR). To simplify ci-management settings,
 619     # we want to move the data to the top. We do not want simple copy,
 620     # as ci-management is eager with recursive search.
 621     #
 622     # As some scripts may call this function multiple times,
 623     # the actual implementation use copying and deletion,
 624     # so the workspace gets "union" of contents (except overwrites on conflict).
 625     # The consequence is empty ARCHIVE_DIR remaining after this call.
 626     #
 627     # As the source directory is emptied,
 628     # the check for dirs being different is essential.
 629     #
 630     # Variables read:
 631     # - WORKSPACE - Jenkins workspace, move only if the value is not empty.
 632     #   Can be unset, then it speeds up manual testing.
 633     # - ARCHIVE_DIR - Path to directory with content to be moved.
 634     # Directories updated:
 635     # - ${WORKSPACE}/archives/ - Created if does not exist.
 636     #   Content of ${ARCHIVE_DIR}/ is moved.
 637     # Functions called:
 638     # - die - Print to stderr and exit.
 639
 640     set -exuo pipefail
 641
 642     if [[ -n "${WORKSPACE-}" ]]; then
 643         target=$(readlink -f "${WORKSPACE}/archives")
 644         if [[ "${target}" != "${ARCHIVE_DIR}" ]]; then
 645             mkdir -p "${target}" || die "Archives dir create failed."
 646             cp -rf "${ARCHIVE_DIR}"/* "${target}" || die "Copy failed."
 647             rm -rf "${ARCHIVE_DIR}"/* || die "Delete failed."
 648         fi
 649     fi
 650 }
 651
 652
 653 function prepare_topology () {
 654
 655     # Prepare virtual testbed topology if needed based on flavor.
 656
 657     # Variables read:
 658     # - TEST_CODE - String affecting test selection, usually jenkins job name.
 659     # - NODENESS - Node multiplicity of testbed, either "2n" or "3n".
 660     # - FLAVOR - Node flavor string, e.g. "clx" or "skx".
 661     # Variables set:
 662     # - TERRAFORM_MODULE_DIR - Terraform module directory.
 663     # Functions called:
 664     # - die - Print to stderr and exit.
 665     # - terraform_init - Terraform init topology.
 666     # - terraform_apply - Terraform apply topology.
 667
 668     set -exuo pipefail
 669
 670     case_text="${NODENESS}_${FLAVOR}"
 671     case "${case_text}" in
 672         "1n_aws" | "2n_aws" | "3n_aws")
 673             export TF_VAR_testbed_name="${TEST_CODE}"
 674             TERRAFORM_MODULE_DIR="terraform-aws-${NODENESS}-${FLAVOR}-c5n"
 675             terraform_init || die "Failed to call terraform init."
 676             trap "terraform_destroy" ERR EXIT || {
 677                 die "Trap attempt failed, please cleanup manually. Aborting!"
 678             }
 679             terraform_apply || die "Failed to call terraform apply."
 680             ;;
 681         "1n_c6gn" | "2n_c6gn" | "3n_c6gn")
 682             export TF_VAR_testbed_name="${TEST_CODE}"
 683             TERRAFORM_MODULE_DIR="terraform-aws-${NODENESS}-c6gn"
 684             terraform_init || die "Failed to call terraform init."
 685             trap "terraform_destroy" ERR EXIT || {
 686                 die "Trap attempt failed, please cleanup manually. Aborting!"
 687             }
 688             terraform_apply || die "Failed to call terraform apply."
 689             ;;
 690         "1n_c6in" | "2n_c6in" | "3n_c6in")
 691             export TF_VAR_testbed_name="${TEST_CODE}"
 692             TERRAFORM_MODULE_DIR="terraform-aws-${NODENESS}-c6in"
 693             terraform_init || die "Failed to call terraform init."
 694             trap "terraform_destroy" ERR EXIT || {
 695                 die "Trap attempt failed, please cleanup manually. Aborting!"
 696             }
 697             terraform_apply || die "Failed to call terraform apply."
 698             ;;
 699     esac
 700 }
 701
 702
 703 function reserve_and_cleanup_testbed () {
 704
 705     # Reserve physical testbed, perform cleanup, register trap to unreserve.
 706     # When cleanup fails, remove from topologies and keep retrying
 707     # until all topologies are removed.
 708     #
 709     # Variables read:
 710     # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on.
 711     # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
 712     # - BUILD_TAG - Any string suitable as filename, identifying
 713     #   test run executing this function. May be unset.
 714     # Variables set:
 715     # - TOPOLOGIES - Array of paths to topologies, with failed cleanups removed.
 716     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 717     # Functions called:
 718     # - die - Print to stderr and exit.
 719     # - ansible_playbook - Perform an action using ansible, see ansible.sh
 720     # Traps registered:
 721     # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
 722
 723     set -exuo pipefail
 724
 725     while true; do
 726         for topo in "${TOPOLOGIES[@]}"; do
 727             set +e
 728             scrpt="${PYTHON_SCRIPTS_DIR}/topo_reservation.py"
 729             opts=("-t" "${topo}" "-r" "${BUILD_TAG:-Unknown}")
 730             python3 "${scrpt}" "${opts[@]}"
 731             result="$?"
 732             set -e
 733             if [[ "${result}" == "0" ]]; then
 734                 # Trap unreservation before cleanup check,
 735                 # so multiple jobs showing failed cleanup improve chances
 736                 # of humans to notice and fix.
 737                 WORKING_TOPOLOGY="${topo}"
 738                 echo "Reserved: ${WORKING_TOPOLOGY}"
 739                 trap "untrap_and_unreserve_testbed" EXIT || {
 740                     message="TRAP ATTEMPT AND UNRESERVE FAILED, FIX MANUALLY."
 741                     untrap_and_unreserve_testbed "${message}" || {
 742                         die "Teardown should have died, not failed."
 743                     }
 744                     die "Trap attempt failed, unreserve succeeded. Aborting."
 745                 }
 746                 # Cleanup + calibration checks
 747                 set +e
 748                 ansible_playbook "cleanup, calibration"
 749                 result="$?"
 750                 set -e
 751                 if [[ "${result}" == "0" ]]; then
 752                     break
 753                 fi
 754                 warn "Testbed cleanup failed: ${topo}"
 755                 untrap_and_unreserve_testbed "Fail of unreserve after cleanup."
 756             fi
 757             # Else testbed is accessible but currently reserved, moving on.
 758         done
 759
 760         if [[ -n "${WORKING_TOPOLOGY-}" ]]; then
 761             # Exit the infinite while loop if we made a reservation.
 762             warn "Reservation and cleanup successful."
 763             break
 764         fi
 765
 766         if [[ "${#TOPOLOGIES[@]}" == "0" ]]; then
 767             die "Run out of operational testbeds!"
 768         fi
 769
 770         # Wait ~3minutes before next try.
 771         sleep_time="$[ ( ${RANDOM} % 20 ) + 180 ]s" || {
 772             die "Sleep time calculation failed."
 773         }
 774         echo "Sleeping ${sleep_time}"
 775         sleep "${sleep_time}" || die "Sleep failed."
 776     done
 777 }
 778
 779
 780 function run_robot () {
 781
 782     # Run robot with options based on input variables.
 783     #
 784     # Variables read:
 785     # - CSIT_DIR - Path to existing root of local CSIT git repository.
 786     # - ARCHIVE_DIR - Path to store robot result files in.
 787     # - ROBOT_ARGS, EXPANDED_TAGS - See compose_robot_arguments.sh
 788     # - GENERATED_DIR - Tests are assumed to be generated under there.
 789     # Variables set:
 790     # - ROBOT_EXIT_STATUS - Exit status of most recent robot invocation.
 791     # Functions called:
 792     # - die - Print to stderr and exit.
 793
 794     set -exuo pipefail
 795
 796     all_options=("--outputdir" "${ARCHIVE_DIR}" "${ROBOT_ARGS[@]}")
 797     all_options+=("${EXPANDED_TAGS[@]}")
 798
 799     pushd "${CSIT_DIR}" || die "Change directory operation failed."
 800     set +e
 801     robot "${all_options[@]}" "${GENERATED_DIR}/tests/"
 802     ROBOT_EXIT_STATUS="$?"
 803     set -e
 804
 805     popd || die "Change directory operation failed."
 806 }
 807
 808
 809 function select_arch_os () {
 810
 811     # Set variables affected by local CPU architecture and operating system.
 812     #
 813     # Variables set:
 814     # - VPP_VER_FILE - Name of file in CSIT dir containing vpp stable version.
 815     # - IMAGE_VER_FILE - Name of file in CSIT dir containing the image name.
 816     # - PKG_SUFFIX - Suffix of OS package file name, "rpm" or "deb."
 817
 818     set -exuo pipefail
 819
 820     source /etc/os-release || die "Get OS release failed."
 821
 822     case "${ID}" in
 823         "ubuntu"*)
 824             case "${VERSION}" in
 825                 *"LTS (Jammy Jellyfish)"*)
 826                     IMAGE_VER_FILE="VPP_DEVICE_IMAGE_UBUNTU_JAMMY"
 827                     VPP_VER_FILE="VPP_STABLE_VER_UBUNTU_JAMMY"
 828                     PKG_SUFFIX="deb"
 829                     ;;
 830                 *)
 831                     die "Unsupported Ubuntu version!"
 832                     ;;
 833             esac
 834             ;;
 835         *)
 836             die "Unsupported distro or OS!"
 837             ;;
 838     esac
 839
 840     arch=$(uname -m) || {
 841         die "Get CPU architecture failed."
 842     }
 843
 844     case "${arch}" in
 845         "aarch64")
 846             IMAGE_VER_FILE="${IMAGE_VER_FILE}_ARM"
 847             ;;
 848         *)
 849             ;;
 850     esac
 851 }
 852
 853
 854 function select_tags () {
 855
 856     # Variables read:
 857     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
 858     # - TEST_CODE - String affecting test selection, usually jenkins job name.
 859     # - DUT - CSIT test/ subdirectory, set while processing tags.
 860     # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
 861     #   Can be unset.
 862     # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
 863     # - BASH_FUNCTION_DIR - Directory with input files to process.
 864     # Variables set:
 865     # - TAGS - Array of processed tag boolean expressions.
 866     # - SELECTION_MODE - Selection criteria [test, suite, include, exclude].
 867
 868     set -exuo pipefail
 869
 870     # NIC SELECTION
 871     case "${TEST_CODE}" in
 872         *"1n-aws"* | *"1n-c6gn"* | *"1n-c6in"*)
 873             start_pattern='^  SUT:'
 874             ;;
 875         *)
 876             start_pattern='^  TG:'
 877             ;;
 878     esac
 879     end_pattern='^ \? \?[A-Za-z0-9]\+:'
 880     # Remove the sections from topology file
 881     sed_command="/${start_pattern}/,/${end_pattern}/d"
 882     # All topologies NICs
 883     available=$(sed "${sed_command}" "${TOPOLOGIES_DIR}"/* \
 884                 | grep -hoP "model: \K.*" | sort -u)
 885     # Selected topology NICs
 886     reserved=$(sed "${sed_command}" "${WORKING_TOPOLOGY}" \
 887                | grep -hoP "model: \K.*" | sort -u)
 888     # All topologies NICs - Selected topology NICs
 889     exclude_nics=($(comm -13 <(echo "${reserved}") <(echo "${available}"))) || {
 890         die "Computation of excluded NICs failed."
 891     }
 892
 893     # Select default NIC tag.
 894     case "${TEST_CODE}" in
 895         *"3n-snr"*)
 896             default_nic="nic_intel-e822cq"
 897             ;;
 898         *"3n-tsh"*)
 899             default_nic="nic_intel-x520-da2"
 900             ;;
 901         *"3n-icx"* | *"2n-icx"*)
 902             default_nic="nic_intel-e810cq"
 903             ;;
 904         *"3na-spr"*)
 905             default_nic="nic_mellanox-cx7veat"
 906             ;;
 907         *"3nb-spr"*)
 908             default_nic="nic_intel-e810cq"
 909             ;;
 910         *"2n-spr"*)
 911             default_nic="nic_intel-e810cq"
 912             ;;
 913         *"2n-clx"* | *"2n-zn2"*)
 914             default_nic="nic_intel-xxv710"
 915             ;;
 916         *"2n-tx2"* | *"3n-alt"*)
 917             default_nic="nic_intel-xl710"
 918             ;;
 919         *"1n-aws"* | *"2n-aws"* | *"3n-aws"*)
 920             default_nic="nic_amazon-nitro-50g"
 921             ;;
 922         *"1n-c6gn"* | *"2n-c6gn"* | *"3n-c6gn"*)
 923             default_nic="nic_amazon-nitro-100g"
 924             ;;
 925         *"1n-c6in"* | *"2n-c6in"* | *"3n-c6in"*)
 926             default_nic="nic_amazon-nitro-200g"
 927             ;;
 928         *)
 929             default_nic="nic_intel-x710"
 930             ;;
 931     esac
 932
 933     sed_nic_sub_cmd="sed s/\${default_nic}/${default_nic}/"
 934     awk_nics_sub_cmd=""
 935     awk_nics_sub_cmd+='gsub("xxv710","25ge2p1xxv710");'
 936     awk_nics_sub_cmd+='gsub("x710","10ge2p1x710");'
 937     awk_nics_sub_cmd+='gsub("xl710","40ge2p1xl710");'
 938     awk_nics_sub_cmd+='gsub("x520-da2","10ge2p1x520");'
 939     awk_nics_sub_cmd+='gsub("cx556a","100ge2p1cx556a");'
 940     awk_nics_sub_cmd+='gsub("cx7veat","200ge2p1cx7veat");'
 941     awk_nics_sub_cmd+='gsub("cx6dx","100ge2p1cx6dx");'
 942     awk_nics_sub_cmd+='gsub("e810cq","100ge2p1e810cq");'
 943     awk_nics_sub_cmd+='gsub("vic1227","10ge2p1vic1227");'
 944     awk_nics_sub_cmd+='gsub("vic1385","40ge2p1vic1385");'
 945     awk_nics_sub_cmd+='gsub("nitro-50g","50ge1p1ENA");'
 946     awk_nics_sub_cmd+='gsub("nitro-100g","100ge1p1ENA");'
 947     awk_nics_sub_cmd+='gsub("nitro-200g","200ge1p1ENA");'
 948     awk_nics_sub_cmd+='gsub("virtual","1ge1p82540em");'
 949     awk_nics_sub_cmd+='if ($9 =="drv_avf") drv="avf-";'
 950     awk_nics_sub_cmd+='else if ($9 =="drv_rdma_core") drv ="rdma-";'
 951     awk_nics_sub_cmd+='else if ($9 =="drv_mlx5_core") drv ="mlx5-";'
 952     awk_nics_sub_cmd+='else if ($9 =="drv_af_xdp") drv ="af-xdp-";'
 953     awk_nics_sub_cmd+='else drv="";'
 954     awk_nics_sub_cmd+='if ($1 =="-") cores="";'
 955     awk_nics_sub_cmd+='else cores=$1;'
 956     awk_nics_sub_cmd+='print "*"$7"-" drv $11"-"$5"."$3"-" cores "-" drv $11"-"$5'
 957
 958     # Tag file directory shorthand.
 959     tfd="${JOB_SPECS_DIR}"
 960     case "${TEST_CODE}" in
 961         # Select specific performance tests based on jenkins job type variable.
 962         *"device"* )
 963             readarray -t test_tag_array <<< $(grep -v "#" \
 964                 ${tfd}/vpp_device/${DUT}-${NODENESS}-${FLAVOR}.md |
 965                 awk {"$awk_nics_sub_cmd"} || echo "devicetest") || die
 966             SELECTION_MODE="--test"
 967             ;;
 968         *"hoststack-daily"* )
 969             readarray -t test_tag_array <<< $(grep -v "#" \
 970                 ${tfd}/hoststack_daily/${DUT}-${NODENESS}-${FLAVOR}.md |
 971                 awk {"$awk_nics_sub_cmd"} || echo "perftest") || die
 972             SELECTION_MODE="--test"
 973             ;;
 974         *"ndrpdr-weekly"* )
 975             readarray -t test_tag_array <<< $(grep -v "#" \
 976                 ${tfd}/ndrpdr_weekly/${DUT}-${NODENESS}-${FLAVOR}.md |
 977                 awk {"$awk_nics_sub_cmd"} || echo "perftest") || die
 978             SELECTION_MODE="--test"
 979             ;;
 980         *"mrr-daily"* )
 981             readarray -t test_tag_array <<< $(grep -v "#" \
 982                 ${tfd}/mrr_daily/${DUT}-${NODENESS}-${FLAVOR}.md |
 983                 awk {"$awk_nics_sub_cmd"} || echo "perftest") || die
 984             SELECTION_MODE="--test"
 985             ;;
 986         *"mrr-weekly"* )
 987             readarray -t test_tag_array <<< $(grep -v "#" \
 988                 ${tfd}/mrr_weekly/${DUT}-${NODENESS}-${FLAVOR}.md |
 989                 awk {"$awk_nics_sub_cmd"} || echo "perftest") || die
 990             SELECTION_MODE="--test"
 991             ;;
 992         *"report-iterative"* )
 993             test_sets=(${TEST_TAG_STRING//:/ })
 994             # Run only one test set per run
 995             report_file=${test_sets[0]}.md
 996             readarray -t test_tag_array <<< $(grep -v "#" \
 997                 ${tfd}/report_iterative/${NODENESS}-${FLAVOR}/${report_file} |
 998                 awk {"$awk_nics_sub_cmd"} || echo "perftest") || die
 999             SELECTION_MODE="--test"
1000             ;;
1001         *"report-coverage"* )
1002             test_sets=(${TEST_TAG_STRING//:/ })
1003             # Run only one test set per run
1004             report_file=${test_sets[0]}.md
1005             readarray -t test_tag_array <<< $(grep -v "#" \
1006                 ${tfd}/report_coverage/${NODENESS}-${FLAVOR}/${report_file} |
1007                 awk {"$awk_nics_sub_cmd"} || echo "perftest") || die
1008             SELECTION_MODE="--test"
1009             ;;
1010         * )
1011             if [[ -z "${TEST_TAG_STRING-}" ]]; then
1012                 # If nothing is specified, we will run pre-selected tests by
1013                 # following tags.
1014                 test_tag_array=("mrrAND${default_nic}AND1cAND64bANDethip4-ip4base"
1015                                 "mrrAND${default_nic}AND1cAND78bANDethip6-ip6base"
1016                                 "mrrAND${default_nic}AND1cAND64bANDeth-l2bdbasemaclrn"
1017                                 "mrrAND${default_nic}AND1cAND64bANDeth-l2xcbase"
1018                                 "!drv_af_xdp" "!drv_avf")
1019             else
1020                 # If trigger contains tags, split them into array.
1021                 test_tag_array=(${TEST_TAG_STRING//:/ })
1022             fi
1023             SELECTION_MODE="--include"
1024             ;;
1025     esac
1026
1027     # Blacklisting certain tags per topology.
1028     #
1029     # Reasons for blacklisting:
1030     # - ipsechw - Blacklisted on testbeds without crypto hardware accelerator.
1031     case "${TEST_CODE}" in
1032         *"1n-vbox"*)
1033             test_tag_array+=("!avf")
1034             test_tag_array+=("!vhost")
1035             test_tag_array+=("!flow")
1036             ;;
1037         *"1n_tx2"*)
1038             test_tag_array+=("!flow")
1039             ;;
1040         *"2n-clx"*)
1041             test_tag_array+=("!ipsechw")
1042             ;;
1043         *"2n-icx"*)
1044             test_tag_array+=("!ipsechw")
1045             ;;
1046         *"2n-spr"*)
1047             ;;
1048         *"2n-tx2"*)
1049             test_tag_array+=("!ipsechw")
1050             ;;
1051         *"2n-zn2"*)
1052             test_tag_array+=("!ipsechw")
1053             ;;
1054         *"3n-alt"*)
1055             test_tag_array+=("!ipsechw")
1056             ;;
1057         *"3n-icx"*)
1058             test_tag_array+=("!ipsechw")
1059             test_tag_array+=("!3_node_double_link_topoANDnic_intel-xxv710")
1060             ;;
1061         *"3n-snr"*)
1062             ;;
1063         *"3na-spr"*)
1064             ;;
1065         *"3nb-spr"*)
1066             ;;
1067         *"3n-tsh"*)
1068             test_tag_array+=("!drv_avf")
1069             test_tag_array+=("!ipsechw")
1070             ;;
1071         *"1n-aws"* | *"2n-aws"* | *"3n-aws"*)
1072             test_tag_array+=("!ipsechw")
1073             ;;
1074         *"1n-c6gn"* | *"2n-c6gn"* | *"3n-c6gn"*)
1075             test_tag_array+=("!ipsechw")
1076             ;;
1077         *"1n-c6in"* | *"2n-c6in"* | *"3n-c6in"*)
1078             test_tag_array+=("!ipsechw")
1079             ;;
1080     esac
1081
1082     # We will add excluded NICs.
1083     test_tag_array+=("${exclude_nics[@]/#/!NIC_}")
1084
1085     TAGS=()
1086     prefix=""
1087     if [[ "${TEST_CODE}" == "vpp-"* ]]; then
1088         if [[ "${TEST_CODE}" != *"device"* ]]; then
1089             # Automatic prefixing for VPP perf jobs to limit the NIC used.
1090             if [[ "${TEST_TAG_STRING-}" != *"nic_"* ]]; then
1091                 prefix="${default_nic}AND"
1092             fi
1093         fi
1094     fi
1095     set +x
1096     for tag in "${test_tag_array[@]}"; do
1097         if [[ "${tag}" == "!"* ]]; then
1098             # Exclude tags are not prefixed.
1099             TAGS+=("${tag}")
1100         elif [[ "${tag}" == " "* || "${tag}" == *"perftest"* ]]; then
1101             # Badly formed tag expressions can trigger way too much tests.
1102             set -x
1103             warn "The following tag expression hints at bad trigger: ${tag}"
1104             warn "Possible cause: Multiple triggers in a single comment."
1105             die "Aborting to avoid triggering too many tests."
1106         elif [[ "${tag}" == *"OR"* ]]; then
1107             # If OR had higher precedence than AND, it would be useful here.
1108             # Some people think it does, thus triggering way too much tests.
1109             set -x
1110             warn "The following tag expression hints at bad trigger: ${tag}"
1111             warn "Operator OR has lower precedence than AND. Use space instead."
1112             die "Aborting to avoid triggering too many tests."
1113         elif [[ "${tag}" != "" && "${tag}" != "#"* ]]; then
1114             # Empty and comment lines are skipped.
1115             # Other lines are normal tags, they are to be prefixed.
1116             TAGS+=("${prefix}${tag}")
1117         fi
1118     done
1119     set -x
1120 }
1121
1122
1123 function select_topology () {
1124
1125     # Variables read:
1126     # - NODENESS - Node multiplicity of testbed, either "2n" or "3n".
1127     # - FLAVOR - Node flavor string, e.g. "clx" or "skx".
1128     # - CSIT_DIR - Path to existing root of local CSIT git repository.
1129     # - TOPOLOGIES_DIR - Path to existing directory with available topologies.
1130     # Variables set:
1131     # - TOPOLOGIES - Array of paths to suitable topology yaml files.
1132     # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology.
1133     # Functions called:
1134     # - die - Print to stderr and exit.
1135
1136     set -exuo pipefail
1137
1138     case_text="${NODENESS}_${FLAVOR}"
1139     case "${case_text}" in
1140         "1n_vbox")
1141             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template )
1142             TOPOLOGIES_TAGS="2_node_single_link_topo"
1143             ;;
1144         "1n_skx" | "1n_tx2" | "1n_spr")
1145             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template )
1146             TOPOLOGIES_TAGS="2_node_single_link_topo"
1147             ;;
1148         "2n_skx")
1149             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_skx*.yaml )
1150             TOPOLOGIES_TAGS="2_node_*_link_topo"
1151             ;;
1152         "2n_zn2")
1153             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_zn2*.yaml )
1154             TOPOLOGIES_TAGS="2_node_*_link_topo"
1155             ;;
1156         "3n_skx")
1157             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_skx*.yaml )
1158             TOPOLOGIES_TAGS="3_node_*_link_topo"
1159             ;;
1160         "3n_icx")
1161             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_icx*.yaml )
1162             TOPOLOGIES_TAGS="3_node_*_link_topo"
1163             ;;
1164         "3na_spr")
1165             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3na_spr*.yaml )
1166             TOPOLOGIES_TAGS="3_node_*_link_topo"
1167             ;;
1168         "3nb_spr")
1169             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3nb_spr*.yaml )
1170             TOPOLOGIES_TAGS="3_node_*_link_topo"
1171             ;;
1172         "2n_clx")
1173             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_clx*.yaml )
1174             TOPOLOGIES_TAGS="2_node_*_link_topo"
1175             ;;
1176         "2n_icx")
1177             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_icx*.yaml )
1178             TOPOLOGIES_TAGS="2_node_*_link_topo"
1179             ;;
1180         "2n_spr")
1181             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_spr*.yaml )
1182             TOPOLOGIES_TAGS="2_node_*_link_topo"
1183             ;;
1184         "3n_snr")
1185             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_snr*.yaml )
1186             TOPOLOGIES_TAGS="3_node_single_link_topo"
1187             ;;
1188         "3n_tsh")
1189             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_tsh*.yaml )
1190             TOPOLOGIES_TAGS="3_node_single_link_topo"
1191             ;;
1192         "2n_tx2")
1193             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_tx2*.yaml )
1194             TOPOLOGIES_TAGS="2_node_single_link_topo"
1195             ;;
1196         "3n_alt")
1197             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_alt*.yaml )
1198             TOPOLOGIES_TAGS="3_node_single_link_topo"
1199             ;;
1200         "1n_aws")
1201             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*1n-aws*.yaml )
1202             TOPOLOGIES_TAGS="1_node_single_link_topo"
1203             ;;
1204         "2n_aws")
1205             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n-aws*.yaml )
1206             TOPOLOGIES_TAGS="2_node_single_link_topo"
1207             ;;
1208         "3n_aws")
1209             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n-aws*.yaml )
1210             TOPOLOGIES_TAGS="3_node_single_link_topo"
1211             ;;
1212         "1n_c6gn")
1213             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*1n-c6gn*.yaml )
1214             TOPOLOGIES_TAGS="1_node_single_link_topo"
1215             ;;
1216         "2n_c6gn")
1217             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n-c6gn*.yaml )
1218             TOPOLOGIES_TAGS="2_node_single_link_topo"
1219             ;;
1220         "3n_c6gn")
1221             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n-c6gn*.yaml )
1222             TOPOLOGIES_TAGS="3_node_single_link_topo"
1223             ;;
1224         "1n_c6in")
1225             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*1n-c6in*.yaml )
1226             TOPOLOGIES_TAGS="1_node_single_link_topo"
1227             ;;
1228         "2n_c6in")
1229             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n-c6in*.yaml )
1230             TOPOLOGIES_TAGS="2_node_single_link_topo"
1231             ;;
1232         "3n_c6in")
1233             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n-c6in*.yaml )
1234             TOPOLOGIES_TAGS="3_node_single_link_topo"
1235             ;;
1236         *)
1237             # No falling back to default, that should have been done
1238             # by the function which has set NODENESS and FLAVOR.
1239             die "Unknown specification: ${case_text}"
1240     esac
1241
1242     if [[ -z "${TOPOLOGIES-}" ]]; then
1243         die "No applicable topology found!"
1244     fi
1245 }
1246
1247
1248 function set_environment_variables () {
1249
1250     # Depending on testbed topology, overwrite defaults set in the
1251     # resources/libraries/python/Constants.py file
1252     #
1253     # Variables read:
1254     # - TEST_CODE - String affecting test selection, usually jenkins job name.
1255     # Variables set:
1256     # See specific cases
1257
1258     set -exuo pipefail
1259
1260     case "${TEST_CODE}" in
1261         *"1n-aws"* | *"2n-aws"* | *"3n-aws"*)
1262             export TREX_RX_DESCRIPTORS_COUNT=1024
1263             export TREX_EXTRA_CMDLINE="--mbuf-factor 19"
1264             export TREX_CORE_COUNT=6
1265             # Settings to prevent duration stretching.
1266             export PERF_TRIAL_STL_DELAY=0.1
1267             ;;
1268         *"1n-c6gn"* | *"2n-c6gn"* | *"3n-c6gn"*)
1269             export TREX_RX_DESCRIPTORS_COUNT=1024
1270             export TREX_EXTRA_CMDLINE="--mbuf-factor 19"
1271             export TREX_CORE_COUNT=6
1272             # Settings to prevent duration stretching.
1273             export PERF_TRIAL_STL_DELAY=0.1
1274             ;;
1275         *"1n-c6in"* | *"2n-c6in"* | *"3n-c6in"*)
1276             export TREX_RX_DESCRIPTORS_COUNT=1024
1277             export TREX_EXTRA_CMDLINE="--mbuf-factor 19"
1278             export TREX_CORE_COUNT=6
1279             # Settings to prevent duration stretching.
1280             export PERF_TRIAL_STL_DELAY=0.1
1281             ;;
1282         *"2n-zn2"*)
1283             # Maciek's workaround for Zen2 with lower amount of cores.
1284             export TREX_CORE_COUNT=14
1285     esac
1286 }
1287
1288
1289 function untrap_and_unreserve_testbed () {
1290
1291     # Use this as a trap function to ensure testbed does not remain reserved.
1292     # Perhaps call directly before script exit, to free testbed for other jobs.
1293     # This function is smart enough to avoid multiple unreservations (so safe).
1294     # Topo cleanup is executed (call it best practice), ignoring failures.
1295     #
1296     # Hardcoded values:
1297     # - default message to die with if testbed might remain reserved.
1298     # Arguments:
1299     # - ${1} - Message to die with if unreservation fails. Default hardcoded.
1300     # Variables read (by inner function):
1301     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
1302     # - PYTHON_SCRIPTS_DIR - Path to directory holding Python scripts.
1303     # Variables set:
1304     # - TERRAFORM_MODULE_DIR - Terraform module directory.
1305     # - WORKING_TOPOLOGY - Set to empty string on successful unreservation.
1306     # Trap unregistered:
1307     # - EXIT - Failure to untrap is reported, but ignored otherwise.
1308     # Functions called:
1309     # - die - Print to stderr and exit.
1310     # - ansible_playbook - Perform an action using ansible, see ansible.sh
1311
1312     set -xo pipefail
1313     set +eu  # We do not want to exit early in a "teardown" function.
1314     trap - EXIT || echo "Trap deactivation failed, continuing anyway."
1315     wt="${WORKING_TOPOLOGY}"  # Just to avoid too long lines.
1316     if [[ -z "${wt-}" ]]; then
1317         set -eu
1318         warn "Testbed looks unreserved already. Trap removal failed before?"
1319     else
1320         ansible_playbook "cleanup" || true
1321         python3 "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || {
1322             die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2
1323         }
1324         case "${TEST_CODE}" in
1325             *"1n-aws"* | *"2n-aws"* | *"3n-aws"*)
1326                 TERRAFORM_MODULE_DIR="terraform-aws-${NODENESS}-${FLAVOR}-c5n"
1327                 terraform_destroy || die "Failed to call terraform destroy."
1328                 ;;
1329             *"1n-c6gn"* | *"2n-c6gn"* | *"3n-c6gn"*)
1330                 TERRAFORM_MODULE_DIR="terraform-aws-${NODENESS}-${FLAVOR}"
1331                 terraform_destroy || die "Failed to call terraform destroy."
1332                 ;;
1333             *"1n-c6in"* | *"2n-c6in"* | *"3n-c6in"*)
1334                 TERRAFORM_MODULE_DIR="terraform-aws-${NODENESS}-${FLAVOR}"
1335                 terraform_destroy || die "Failed to call terraform destroy."
1336                 ;;
1337             *)
1338                 ;;
1339         esac
1340         WORKING_TOPOLOGY=""
1341         set -eu
1342     fi
1343 }
1344
1345
1346 function warn () {
1347
1348     # Print the message to standard error.
1349     #
1350     # Arguments:
1351     # - ${@} - The text of the message.
1352
1353     set -exuo pipefail
1354
1355     echo "$@" >&2
1356 }