1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 # This library defines functions used by multiple entry scripts.
17 # Keep functions ordered alphabetically, please.
19 # TODO: Add a link to bash style guide.
20 # TODO: Consider putting every die into a {} block,
21 # the code might become more readable (but longer).
24 function activate_virtualenv () {
29 # - ${1} - Non-empty path to existing directory for creating virtualenv in.
31 # - CSIT_DIR - Path to existing root of local CSIT git repository.
33 # - ENV_DIR - Path to the created virtualenv subdirectory.
35 # - PYTHONPATH - CSIT_DIR, as CSIT Python scripts usually need this.
37 # - die - Print to stderr and exit.
39 # TODO: Do we really need to have ENV_DIR available as a global variable?
41 if [[ "${1-}" == "" ]]; then
42 die "Root location of virtualenv to create is not specified."
45 rm -rf "${ENV_DIR}" || die "Failed to clean previous virtualenv."
47 pip install --upgrade virtualenv || {
48 die "Virtualenv package install failed."
50 virtualenv --system-site-packages "${ENV_DIR}" || {
51 die "Virtualenv creation failed."
54 source "${ENV_DIR}/bin/activate" || die "Virtualenv activation failed."
56 pip install -r "${CSIT_DIR}/requirements.txt" || {
57 die "CSIT requirements installation failed."
60 # Most CSIT Python scripts assume PYTHONPATH is set and exported.
61 export PYTHONPATH="${CSIT_DIR}" || die "Export failed."
65 function check_download_dir () {
69 # Fail if there are no files visible in ${DOWNLOAD_DIR}.
70 # TODO: Do we need this as a function, if it is (almost) a one-liner?
73 # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
75 # - ${DOWNLOAD_DIR} - Has to be non-empty to proceed.
77 # - die - Print to stderr and exit.
79 if [[ ! "$(ls -A "${DOWNLOAD_DIR}")" ]]; then
80 die "No artifacts downloaded!"
85 function common_dirs () {
90 # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
91 # - CSIT_DIR - Path to existing root of local CSIT git repository.
92 # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
93 # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources".
94 # - TOOLS_DIR - Path to existing resources subdirectory "tools".
95 # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".
96 # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive".
97 # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
99 # - die - Print to stderr and exit.
101 BASH_FUNCTION_DIR="$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")" || {
102 die "Some error during localizing this source directory."
104 # Current working directory could be in a different repo, e.g. VPP.
105 pushd "${BASH_FUNCTION_DIR}" || die "Pushd failed"
106 CSIT_DIR="$(readlink -e "$(git rev-parse --show-toplevel)")" || {
107 die "Readlink or git rev-parse failed."
109 popd || die "Popd failed."
110 TOPOLOGIES_DIR="$(readlink -e "${CSIT_DIR}/topologies/available")" || {
111 die "Readlink failed."
113 RESOURCES_DIR="$(readlink -e "${CSIT_DIR}/resources")" || {
114 die "Readlink failed."
116 TOOLS_DIR="$(readlink -e "${RESOURCES_DIR}/tools")" || {
117 die "Readlink failed."
119 PYTHON_SCRIPTS_DIR="$(readlink -e "${TOOLS_DIR}/scripts")" || {
120 die "Readlink failed."
123 ARCHIVE_DIR="$(readlink -f "${CSIT_DIR}/archive")" || {
124 die "Readlink failed."
126 mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed."
127 DOWNLOAD_DIR="$(readlink -f "${CSIT_DIR}/download_dir")" || {
128 die "Readlink failed."
130 mkdir -p "${DOWNLOAD_DIR}" || die "Mkdir failed."
134 function compose_pybot_arguments () {
139 # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
140 # - DUT - CSIT test/ subdirectory, set while processing tags.
141 # - TAGS - Array variable holding selected tag boolean expressions.
142 # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology.
144 # - PYBOT_ARGS - String holding part of all arguments for pybot.
145 # - EXPANDED_TAGS - Array of strings pybot arguments compiled from tags.
147 # No explicit check needed with "set -u".
148 PYBOT_ARGS=("--loglevel" "TRACE" "--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}")
149 PYBOT_ARGS+=("--suite" "tests.${DUT}.perf")
152 for tag in "${TAGS[@]}"; do
153 if [[ ${tag} == "!"* ]]; then
154 EXPANDED_TAGS+=("--exclude" "${tag#$"!"}")
156 EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}")
162 function copy_archives () {
167 # - WORKSPACE - Jenkins workspace, copy only if the value is not empty.
168 # Can be unset, then it speeds up manual testing.
169 # - ARCHIVE_DIR - Path to directory with content to be copied.
170 # Directories updated:
171 # - ${WORKSPACE}/archives/ - Created if does not exist.
172 # Content of ${ARCHIVE_DIR}/ is copied here.
174 # - die - Print to stderr and exit.
176 # We will create additional archive if workspace variable is set.
177 # This way if script is running in jenkins all will be
178 # automatically archived to logs.fd.io.
179 if [[ -n "${WORKSPACE-}" ]]; then
180 mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed."
181 cp -r "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed."
187 # Print the message to standard error end exit with error code specified
188 # by the second argument.
191 # - The default error message.
193 # - ${1} - The whole error message, be sure to quote. Optional
194 # - ${2} - the code to exit with, default: 1.
198 warn "${1:-Unspecified run-time error occurred!}"
203 function die_on_pybot_error () {
207 # Source this fragment if you want to abort on any failed test case.
210 # - PYBOT_EXIT_STATUS - Set by a pybot running fragment.
212 # - die - Print to stderr and exit.
214 if [[ "${PYBOT_EXIT_STATUS}" != "0" ]]; then
215 die "${PYBOT_EXIT_STATUS}" "Test failures are present!"
220 function get_test_code () {
225 # - ${1} - Optional, argument of entry script (or empty as unset).
226 # Test code value to override job name from environment.
228 # - JOB_NAME - String affecting test selection, default if not argument.
230 # - TEST_CODE - The test selection string from environment or argument.
231 # - NODENESS - Node multiplicity of desired testbed.
232 # - FLAVOR - Node flavor string, usually describing the processor.
234 TEST_CODE="${1-}" || die "Reading optional argument failed, somehow."
235 if [[ -z "${TEST_CODE}" ]]; then
236 TEST_CODE="${JOB_NAME-}" || die "Reading job name failed, somehow."
239 case "${TEST_CODE}" in
249 # Fallback to 3-node Haswell by default (backward compatibility)
257 function get_test_tag_string () {
262 # - GERRIT_EVENT_TYPE - Event type set by gerrit, can be unset.
263 # - GERRIT_EVENT_COMMENT_TEXT - Comment text, read for "comment-added" type.
265 # - TEST_TAG_STRING - The string following "perftest" in gerrit comment,
268 # TODO: ci-management scripts no longer need to perform this.
271 if [[ "${GERRIT_EVENT_TYPE-}" == "comment-added" ]]; then
272 # On parsing error, ${trigger} stays empty.
273 trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \
274 | grep -oE '(perftest$|perftest[[:space:]].+$)')" || true
276 # Set test tags as string.
277 TEST_TAG_STRING="${trigger#$"perftest"}"
281 function reserve_testbed () {
285 # Reserve physical testbed, perform cleanup, register trap to unreserve.
288 # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on.
289 # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script.
291 # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
293 # - die - Print to stderr and exit.
295 # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
298 for topo in "${TOPOLOGIES[@]}"; do
300 python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}"
303 if [[ "${result}" == "0" ]]; then
304 WORKING_TOPOLOGY="${topo}"
305 echo "Reserved: ${WORKING_TOPOLOGY}"
306 python "${PYTHON_SCRIPTS_DIR}/topo_cleanup.py" -t "${topo}" || {
307 die "Testbed cleanup failed."
309 trap "untrap_and_unreserve_testbed" EXIT || {
310 message="TRAP ATTEMPT AND UNRESERVE FAILED, FIX MANUALLY."
311 untrap_and_unreserve_testbed "${message}" || {
312 die "Teardown should have died, not failed."
314 die "Trap attempt failed, unreserve succeeded. Aborting."
320 if [[ -n "${WORKING_TOPOLOGY-}" ]]; then
321 # Exit the infinite while loop if we made a reservation.
325 # Wait ~3minutes before next try.
326 sleep_time="$[ ( $RANDOM % 20 ) + 180 ]s" || {
327 die "Sleep time calculation failed."
329 echo "Sleeping ${sleep_time}"
330 sleep "${sleep_time}" || die "Sleep failed."
335 function run_pybot () {
339 # Currently, VPP-1361 causes occasional test failures.
340 # If real result is more important than time, we can retry few times.
341 # TODO: We should be retrying on test case level instead.
344 # - ${1} - Optional number of pybot invocations to try to avoid failures.
347 # - CSIT_DIR - Path to existing root of local CSIT git repository.
348 # - ARCHIVE_DIR - Path to store robot result files in.
349 # - PYBOT_ARGS, EXPANDED_TAGS - See compose_pybot_arguments.sh
351 # - PYBOT_EXIT_STATUS - Exit status of most recent pybot invocation.
353 # - die - Print to stderr and exit.
355 # Set ${tries} as an integer variable, to fail on non-numeric input.
356 local -i "tries" || die "Setting type of variable failed."
357 tries="${1:-1}" || die "Argument evaluation failed."
358 all_options=("--outputdir" "${ARCHIVE_DIR}" "${PYBOT_ARGS[@]}")
359 all_options+=("${EXPANDED_TAGS[@]}")
362 if [[ "${tries}" -le 0 ]]; then
365 tries="$((${tries} - 1))"
367 pushd "${CSIT_DIR}" || die "Change directory operation failed."
369 # TODO: Make robot tests not require "$(pwd)" == "${CSIT_DIR}".
370 pybot "${all_options[@]}" "${CSIT_DIR}/tests/"
371 PYBOT_EXIT_STATUS="$?"
373 popd || die "Change directory operation failed."
374 if [[ "${PYBOT_EXIT_STATUS}" == "0" ]]; then
381 function select_tags () {
386 # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
387 # - TEST_CODE - String affecting test selection, usually jenkins job name.
388 # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
390 # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
392 # - TAGS - Array of processed tag boolean expressions.
394 # TODO: Empty exclude_nics (with failing grep) is expected,
395 # but others possible errors coule be checked explicitly.
397 # All topologies NICs
398 available=$(grep -hoPR "model: \K.*" "${TOPOLOGIES_DIR}"/* | sort -u)
399 # Selected topology NICs
400 reserved=$(grep -hoPR "model: \K.*" "${WORKING_TOPOLOGY}" | sort -u)
401 # All topologies NICs - Selected topology NICs
402 exclude_nics=($(comm -13 <(echo "${reserved}") <(echo "${available}")))
404 case "${TEST_CODE}" in
405 # Select specific performance tests based on jenkins job type variable.
407 test_tag_array=("ndrpdrAND64bAND1c"
410 *"mrr-daily"* | *"mrr-weekly"* )
411 test_tag_array=("mrrAND64bAND1c"
417 "mrrANDimixAND1cANDvhost"
418 "mrrANDimixAND2cANDvhost"
419 "mrrANDimixAND4cANDvhost"
420 "mrrANDimixAND1cANDmemif"
421 "mrrANDimixAND2cANDmemif"
422 "mrrANDimixAND4cANDmemif")
425 if [[ -z "${TEST_TAG_STRING-}" ]]; then
426 # If nothing is specified, we will run pre-selected tests by
427 # following tags. Items of array will be concatenated by OR
428 # in Robot Framework.
429 test_tag_array=("mrrANDnic_intel-x710AND1cAND64bANDip4base"
430 "mrrANDnic_intel-x710AND1cAND78bANDip6base"
431 "mrrANDnic_intel-x710AND1cAND64bANDl2bdbase"
432 "mrrANDnic_intel-x710AND1cAND64bANDl2xcbase")
434 # If trigger contains tags, split them into array.
435 test_tag_array=(${TEST_TAG_STRING//:/ })
440 # We will add excluded NICs.
441 test_tag_array+=("${exclude_nics[@]/#/!NIC_}")
445 # We will prefix with perftest to prevent running other tests
448 if [[ "${TEST_CODE}" == "vpp-"* ]]; then
449 # Automatic prefixing for VPP jobs to limit the NIC used and
450 # traffic evaluation to MRR.
451 prefix="${prefix}mrrANDnic_intel-x710AND"
453 for tag in "${test_tag_array[@]}"; do
454 if [[ ${tag} == "!"* ]]; then
455 # Exclude tags are not prefixed.
458 TAGS+=("${prefix}${tag}")
464 function select_topology () {
469 # - NODENESS - Node multiplicity of testbed, either "2n" or "3n".
470 # - FLAVOR - Node flavor string, currently either "hsw" or "skx".
471 # - CSIT_DIR - Path to existing root of local CSIT git repository.
472 # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
474 # - TOPOLOGIES - Array of paths to suitable topology yaml files.
475 # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology.
477 # - die - Print to stderr and exit.
479 case_text="${NODENESS}_${FLAVOR}"
480 case "${case_text}" in
483 "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml"
484 "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml"
485 "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml"
487 TOPOLOGIES_TAGS="3_node_*_link_topo"
491 "${TOPOLOGIES_DIR}/lf_2n_skx_testbed21.yaml"
492 #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed22.yaml"
493 #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed23.yaml"
494 "${TOPOLOGIES_DIR}/lf_2n_skx_testbed24.yaml"
496 TOPOLOGIES_TAGS="2_node_*_link_topo"
500 "${TOPOLOGIES_DIR}/lf_3n_skx_testbed31.yaml"
501 "${TOPOLOGIES_DIR}/lf_3n_skx_testbed32.yaml"
503 TOPOLOGIES_TAGS="3_node_*_link_topo"
506 # No falling back to 3n_hsw default, that should have been done
507 # by the function which has set NODENESS and FLAVOR.
508 die "Unknown specification: ${case_text}"
511 if [[ -z "${TOPOLOGIES-}" ]]; then
512 die "No applicable topology found!"
517 function untrap_and_unreserve_testbed () {
518 # Use this as a trap function to ensure testbed does not remain reserved.
519 # Perhaps call directly before script exit, to free testbed for other jobs.
520 # This function is smart enough to avoid multiple unreservations (so safe).
521 # Topo cleanup is executed (call it best practice), ignoring failures.
524 # - default message to die with if testbed might remain reserved.
526 # - ${1} - Message to die with if unreservation fails. Default hardcoded.
527 # Variables read (by inner function):
528 # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
529 # - PYTHON_SCRIPTS_DIR - Path to directory holding Python scripts.
531 # - WORKING_TOPOLOGY - Set to empty string on successful unreservation.
533 # - EXIT - Failure to untrap is reported, but ignored otherwise.
535 # - die - Print to stderr and exit.
538 set +eu # We do not want to exit early in a "teardown" function.
539 trap - EXIT || echo "Trap deactivation failed, continuing anyway."
540 wt="${WORKING_TOPOLOGY}" # Just to avoid too long lines.
541 if [[ -z "${wt-}" ]]; then
543 echo "Testbed looks unreserved already. Trap removal failed before?"
545 python "${PYTHON_SCRIPTS_DIR}/topo_cleanup.py" -t "${wt}" || true
546 python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || {
547 die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2
556 # Print the message to standard error.
559 # - ${@} - The text of the message.