X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fbash%2Ffunction%2Fcommon.sh;h=c0aa5f9f7d6b5828ada0873884ea3154f75d5a6f;hp=ec335f171f8145a67e4bfe3859616ee2a75fc32a;hb=523c6e6e24101206ff1318ca17c310dff8b3c9d2;hpb=806a069bc5494d1c43dd9711e1512e518c823acd

diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index ec335f171f..c0aa5f9f7d 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -42,7 +42,6 @@ function activate_docker_topology () {
     source "${BASH_FUNCTION_DIR}/device.sh" || {
         die "Source failed!"
     }
-
     device_image="$(< ${CSIT_DIR}/${IMAGE_VER_FILE})"
     case_text="${NODENESS}_${FLAVOR}"
     case "${case_text}" in
@@ -50,8 +49,9 @@ function activate_docker_topology () {
             # We execute reservation over csit-shim-dcr (ssh) which runs sourced
             # script's functions. Env variables are read from ssh output
             # back to localhost for further processing.
-            hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3) || die
-            ssh="ssh root@${hostname} -p 6022"
+            # Shim and Jenkins executor are in the same network on the same host
+            # Connect to docker's default gateway IP and shim's exposed port
+            ssh="ssh root@172.17.0.1 -p 6022"
             run="activate_wrapper ${NODENESS} ${FLAVOR} ${device_image}"
             # The "declare -f" output is long and boring.
             set +x
@@ -145,7 +145,7 @@ function archive_tests () {
 
     set -exuo pipefail
 
-    tar c "${GENERATED_DIR}/tests" | xz -9e > "${ARCHIVE_DIR}/tests.tar.xz" || {
+    tar c "${GENERATED_DIR}/tests" | xz -3 > "${ARCHIVE_DIR}/tests.tar.xz" || {
         die "Error creating archive of generated tests."
     }
 }
@@ -198,7 +198,8 @@ function common_dirs () {
     # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources".
     # - TOOLS_DIR - Path to existing resources subdirectory "tools".
     # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".
-    # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive".
+    # - ARCHIVE_DIR - Path to created CSIT subdirectory "archives".
+    #   The name is chosen to match what ci-management expects.
     # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
     # - GENERATED_DIR - Path to created CSIT subdirectory "generated".
     # Directories created if not present:
@@ -240,7 +241,7 @@ function common_dirs () {
         die "Readlink failed."
     }
 
-    ARCHIVE_DIR=$(readlink -f "${CSIT_DIR}/archive") || {
+    ARCHIVE_DIR=$(readlink -f "${CSIT_DIR}/archives") || {
         die "Readlink failed."
     }
     mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed."
@@ -295,31 +296,6 @@ function compose_pybot_arguments () {
 }
 
 
-function copy_archives () {
-
-    # Create additional archive if workspace variable is set.
-    # This way if script is running in jenkins all will be
-    # automatically archived to logs.fd.io.
-    #
-    # Variables read:
-    # - WORKSPACE - Jenkins workspace, copy only if the value is not empty.
-    #   Can be unset, then it speeds up manual testing.
-    # - ARCHIVE_DIR - Path to directory with content to be copied.
-    # Directories updated:
-    # - ${WORKSPACE}/archives/ - Created if does not exist.
-    #   Content of ${ARCHIVE_DIR}/ is copied here.
-    # Functions called:
-    # - die - Print to stderr and exit.
-
-    set -exuo pipefail
-
-    if [[ -n "${WORKSPACE-}" ]]; then
-        mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed."
-        cp -rf "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed."
-    fi
-}
-
-
 function deactivate_docker_topology () {
 
     # Deactivate virtual vpp-device topology by removing containers.
@@ -333,8 +309,7 @@ function deactivate_docker_topology () {
     case_text="${NODENESS}_${FLAVOR}"
     case "${case_text}" in
         "1n_skx" | "1n_tx2")
-            hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3) || die
-            ssh="ssh root@${hostname} -p 6022"
+            ssh="ssh root@172.17.0.1 -p 6022"
             env_vars=$(env | grep CSIT_ | tr '\n' ' ' ) || die
             # The "declare -f" output is long and boring.
             set +x
@@ -461,6 +436,10 @@ function get_test_code () {
             NODENESS="2n"
             FLAVOR="skx"
             ;;
+        *"2n-zn2"*)
+            NODENESS="2n"
+            FLAVOR="zn2"
+            ;;
         *"3n-skx"*)
             NODENESS="3n"
             FLAVOR="skx"
@@ -516,14 +495,33 @@ function get_test_tag_string () {
                 die "Unknown specification: ${TEST_CODE}"
         esac
         # Ignore lines not containing the trigger word.
-        comment=$(fgrep "${trigger}" <<< "${GERRIT_EVENT_COMMENT_TEXT}") || true
+        comment=$(fgrep "${trigger}" <<< "${GERRIT_EVENT_COMMENT_TEXT}" || true)
         # The vpp-csit triggers trail stuff we are not interested in.
         # Removing them and trigger word: https://unix.stackexchange.com/a/13472
         # (except relying on \s whitespace, \S non-whitespace and . both).
         # The last string is concatenated, only the middle part is expanded.
         cmd=("grep" "-oP" '\S*'"${trigger}"'\S*\s\K.+$') || die "Unset trigger?"
         # On parsing error, TEST_TAG_STRING probably stays empty.
-        TEST_TAG_STRING=$("${cmd[@]}" <<< "${comment}") || true
+        TEST_TAG_STRING=$("${cmd[@]}" <<< "${comment}" || true)
+        if [[ -z "${TEST_TAG_STRING-}" ]]; then
+            # Probably we got a base64 encoded comment.
+            comment=$(base64 --decode <<< "${GERRIT_EVENT_COMMENT_TEXT}" || true)
+            comment=$(fgrep "${trigger}" <<< "${comment}" || true)
+            TEST_TAG_STRING=$("${cmd[@]}" <<< "${comment}" || true)
+        fi
+        if [[ -n "${TEST_TAG_STRING-}" ]]; then
+            test_tag_array=(${TEST_TAG_STRING})
+            if [[ "${test_tag_array[0]}" == "icl" ]]; then
+                export GRAPH_NODE_VARIANT="icl"
+                TEST_TAG_STRING="${test_tag_array[@]:1}" || true
+            elif [[ "${test_tag_array[0]}" == "skx" ]]; then
+                export GRAPH_NODE_VARIANT="skx"
+                TEST_TAG_STRING="${test_tag_array[@]:1}" || true
+            elif [[ "${test_tag_array[0]}" == "hsw" ]]; then
+                export GRAPH_NODE_VARIANT="hsw"
+                TEST_TAG_STRING="${test_tag_array[@]:1}" || true
+            fi
+        fi
     fi
 }
 
@@ -546,6 +544,47 @@ function installed () {
 }
 
 
+function move_archives () {
+
+    # Move archive directory to top of workspace, if not already there.
+    #
+    # ARCHIVE_DIR is positioned relative to CSIT_DIR,
+    # but in some jobs CSIT_DIR is not same as WORKSPACE
+    # (e.g. under VPP_DIR). To simplify ci-management settings,
+    # we want to move the data to the top. We do not want simple copy,
+    # as ci-management is eager with recursive search.
+    #
+    # As some scripts may call this function multiple times,
+    # the actual implementation use copying and deletion,
+    # so the workspace gets "union" of contents (except overwrites on conflict).
+    # The consequence is empty ARCHIVE_DIR remaining after this call.
+    #
+    # As the source directory is emptied,
+    # the check for dirs being different is essential.
+    #
+    # Variables read:
+    # - WORKSPACE - Jenkins workspace, move only if the value is not empty.
+    #   Can be unset, then it speeds up manual testing.
+    # - ARCHIVE_DIR - Path to directory with content to be moved.
+    # Directories updated:
+    # - ${WORKSPACE}/archives/ - Created if does not exist.
+    #   Content of ${ARCHIVE_DIR}/ is moved.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    if [[ -n "${WORKSPACE-}" ]]; then
+        target=$(readlink -f "${WORKSPACE}/archives")
+        if [[ "${target}" != "${ARCHIVE_DIR}" ]]; then
+            mkdir -p "${target}" || die "Archives dir create failed."
+            cp -rf "${ARCHIVE_DIR}"/* "${target}" || die "Copy failed."
+            rm -rf "${ARCHIVE_DIR}"/* || die "Delete failed."
+        fi
+    fi
+}
+
+
 function reserve_and_cleanup_testbed () {
 
     # Reserve physical testbed, perform cleanup, register trap to unreserve.
@@ -744,7 +783,7 @@ function select_tags () {
         *"3n-tsh"*)
             default_nic="nic_intel-x520-da2"
             ;;
-        *"3n-skx"* | *"2n-skx"* | *"2n-clx"*)
+        *"3n-skx"* | *"2n-skx"* | *"2n-clx"* | *"2n-zn2"*)
             default_nic="nic_intel-xxv710"
             ;;
         *"3n-hsw"* | *"mrr-daily-master")
@@ -771,17 +810,17 @@ function select_tags () {
         *"ndrpdr-weekly"* )
             readarray -t test_tag_array <<< $(sed 's/ //g' \
                 ${tfd}/mlr_weekly/${DUT}-${NODENESS}-${FLAVOR}.md |
-                eval ${sed_nics_sub_cmd}) || die
+                eval ${sed_nics_sub_cmd} || echo "perftest") || die
             ;;
         *"mrr-daily"* )
             readarray -t test_tag_array <<< $(sed 's/ //g' \
                 ${tfd}/mrr_daily/${DUT}-${NODENESS}-${FLAVOR}.md |
-                eval ${sed_nics_sub_cmd}) || die
+                eval ${sed_nics_sub_cmd} || echo "perftest") || die
             ;;
         *"mrr-weekly"* )
             readarray -t test_tag_array <<< $(sed 's/ //g' \
                 ${tfd}/mrr_weekly/${DUT}-${NODENESS}-${FLAVOR}.md |
-                eval ${sed_nics_sub_cmd}) || die
+                eval ${sed_nics_sub_cmd} || echo "perftest") || die
             ;;
         *"report-iterative"* )
             test_sets=(${TEST_TAG_STRING//:/ })
@@ -789,7 +828,7 @@ function select_tags () {
             report_file=${test_sets[0]}.md
             readarray -t test_tag_array <<< $(sed 's/ //g' \
                 ${tfd}/report_iterative/${NODENESS}-${FLAVOR}/${report_file} |
-                eval ${sed_nics_sub_cmd}) || die
+                eval ${sed_nics_sub_cmd} || echo "perftest") || die
             ;;
         *"report-coverage"* )
             test_sets=(${TEST_TAG_STRING//:/ })
@@ -797,7 +836,7 @@ function select_tags () {
             report_file=${test_sets[0]}.md
             readarray -t test_tag_array <<< $(sed 's/ //g' \
                 ${tfd}/report_coverage/${NODENESS}-${FLAVOR}/${report_file} |
-                eval ${sed_nics_sub_cmd}) || die
+                eval ${sed_nics_sub_cmd} || echo "perftest") || die
             ;;
         * )
             if [[ -z "${TEST_TAG_STRING-}" ]]; then
@@ -822,7 +861,7 @@ function select_tags () {
     # TODO: Add missing reasons here (if general) or where used (if specific).
     case "${TEST_CODE}" in
         *"2n-skx"*)
-            test_tag_array+=("!ipsechw")
+            test_tag_array+=("!ipsec")
             ;;
         *"3n-skx"*)
             test_tag_array+=("!ipsechw")
@@ -830,7 +869,10 @@ function select_tags () {
             test_tag_array+=("!3_node_double_link_topoANDnic_intel-xxv710")
             ;;
         *"2n-clx"*)
-            test_tag_array+=("!ipsechw")
+            test_tag_array+=("!ipsec")
+            ;;
+        *"2n-zn2"*)
+            test_tag_array+=("!ipsec")
             ;;
         *"2n-dnv"*)
             test_tag_array+=("!ipsechw")
@@ -853,10 +895,6 @@ function select_tags () {
             test_tag_array+=("!ipsechw")
             ;;
         *"3n-hsw"*)
-            # TODO: Introduce NOIOMMU version of AVF tests.
-            # TODO: Make (both) AVF tests work on Haswell,
-            # or document why (some of) it is not possible.
-            # https://github.com/FDio/vpp/blob/master/src/plugins/avf/README.md
             test_tag_array+=("!drv_avf")
             # All cards have access to QAT. But only one card (xl710)
             # resides in same NUMA as QAT. Other cards must go over QPI
@@ -898,6 +936,13 @@ function select_tags () {
             warn "The following tag expression hints at bad trigger: ${tag}"
             warn "Possible cause: Multiple triggers in a single comment."
             die "Aborting to avoid triggering too many tests."
+        elif [[ "${tag}" == *"OR"* ]]; then
+            # If OR had higher precedence than AND, it would be useful here.
+            # Some people think it does, thus triggering way too much tests.
+            set -x
+            warn "The following tag expression hints at bad trigger: ${tag}"
+            warn "Operator OR has lower precedence than AND. Use space instead."
+            die "Aborting to avoid triggering too many tests."
         elif [[ "${tag}" != "" && "${tag}" != "#"* ]]; then
             # Empty and comment lines are skipped.
             # Other lines are normal tags, they are to be prefixed.
@@ -939,6 +984,10 @@ function select_topology () {
             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_skx*.yaml )
             TOPOLOGIES_TAGS="2_node_*_link_topo"
             ;;
+        "2n_zn2")
+            TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_zn2*.yaml )
+            TOPOLOGIES_TAGS="2_node_*_link_topo"
+            ;;
         "3n_skx")
             TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_skx*.yaml )
             TOPOLOGIES_TAGS="3_node_*_link_topo"