CSIT-1260 Create initial version of bootstrap including VF reservation 35/14435/60
authorPeter Mikus <pmikus@cisco.com>
Thu, 23 Aug 2018 09:37:16 +0000 (09:37 +0000)
committerPeter Mikus <pmikus@cisco.com>
Wed, 24 Oct 2018 14:27:02 +0000 (14:27 +0000)
Change-Id: I153d378849ea76444915b7fc219237db1b7bca1a
Signed-off-by: Peter Mikus <pmikus@cisco.com>
VPP_DEVICE_IMAGE [new file with mode: 0644]
resources/libraries/bash/entry/bootstrap_verify_perf.sh
resources/libraries/bash/entry/bootstrap_vpp_device.sh [new file with mode: 0755]
resources/libraries/bash/entry/per_patch_perf.sh
resources/libraries/bash/function/common.sh
resources/libraries/bash/function/device.sh [new file with mode: 0644]
topologies/available/vpp_device.template [new file with mode: 0644]

diff --git a/VPP_DEVICE_IMAGE b/VPP_DEVICE_IMAGE
new file mode 100644 (file)
index 0000000..3f47e59
--- /dev/null
@@ -0,0 +1 @@
+snergster/csit-sut:latest
index 74edc4c..1a0d638 100644 (file)
@@ -34,8 +34,8 @@ source "${BASH_FUNCTION_DIR}/common.sh" || {
 }
 source "${BASH_FUNCTION_DIR}/gather.sh" || die "Source failed."
 common_dirs || die
-get_test_tag_string || die
 get_test_code "${1-}" || die
+get_test_tag_string || die
 select_topology || die
 gather_build || die
 check_download_dir || die
diff --git a/resources/libraries/bash/entry/bootstrap_vpp_device.sh b/resources/libraries/bash/entry/bootstrap_vpp_device.sh
new file mode 100755 (executable)
index 0000000..6d389d6
--- /dev/null
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/gather.sh" || die "Source failed."
+common_dirs || die
+get_test_code "${1-}" || die
+get_test_tag_string || die
+select_topology || die
+gather_build || die
+check_download_dir || die
+activate_virtualenv "${CSIT_DIR}" || die
+activate_docker_topology || die
+select_vpp_device_tags || die
+compose_pybot_arguments || die
+run_pybot || die
+copy_archives || die
+die_on_pybot_error || die
index abb1001..73d4f69 100644 (file)
@@ -46,10 +46,10 @@ prepare_build_parent || die
 build_vpp_ubuntu_amd64 "PARENT" || die
 prepare_test || die
 ## Replace previous 4 lines with this to speed up testing.
-#download_builds "https://jenkins.fd.io/sandbox/job/vpp-csit-verify-perf-master-2n-skx/2/artifact/*zip*/archive.zip" || die
+#download_builds "REPLACE_WITH_URL" || die
 initialize_csit_dirs || die
-get_test_tag_string || die
 get_test_code "${1-}" || die
+get_test_tag_string || die
 set_perpatch_dut || die
 select_topology || die
 activate_virtualenv "${VPP_DIR}" || die
index 69c7093..db3d865 100644 (file)
@@ -21,6 +21,71 @@ set -exuo pipefail
 #   the code might become more readable (but longer).
 
 
+function activate_docker_topology () {
+    # Create virtual vpp-device topology. Output of the function is topology
+    # file describing created environment saved to a file.
+    #
+    # Variables read:
+    # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
+    # - TOPOLOGIES - Available topologies.
+    # - NODENESS - Node multiplicity of desired testbed.
+    # - FLAVOR - Node flavor string, usually describing the processor.
+    # Variables set:
+    # - WORKING_TOPOLOGY - Path to topology file.
+
+    set -exuo pipefail
+
+    source "${BASH_FUNCTION_DIR}/device.sh" || {
+        die "Source failed!"
+    }
+
+    device_image="$(< ${CSIT_DIR}/VPP_DEVICE_IMAGE)"
+    case_text="${NODENESS}_${FLAVOR}"
+    case "${case_text}" in
+        "1n_skx")
+            # We execute reservation over csit-shim-dcr (ssh) which runs sourced
+            # script's functions. Env variables are read from ssh output
+            # back to localhost for further processing.
+            hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3)
+            ssh="ssh root@${hostname} -p 6022"
+            run="activate_wrapper ${NODENESS} ${FLAVOR} ${device_image}"
+            env_vars=$(${ssh} "$(declare -f); ${run}") || {
+                die "Topology reservation via shim-dcr failed!"
+            }
+            set -a
+            source <(echo "$env_vars" | grep -v /usr/bin/docker) || {
+                die "Source failed!"
+            }
+            set +a
+            ;;
+        "1n_vbox")
+            # We execute reservation on localhost. Sourced script automatially
+            # sets environment variables for further processing.
+            activate_wrapper "${NODENESS}" "${FLAVOR}" "${device_image}" || die
+            ;;
+        *)
+            die "Unknown specification: ${case_text}!"
+    esac
+
+    trap 'deactivate_docker_topology' EXIT || {
+         die "Trap attempt failed, please cleanup manually. Aborting!"
+    }
+
+    # Replace all variables in template with those in environment.
+    source <(echo 'cat <<EOF >topo.yml'; cat ${TOPOLOGIES[0]}; echo EOF;) || {
+        die "Topology file create failed!"
+    }
+
+    WORKING_TOPOLOGY="/tmp/topology.yaml"
+    mv topo.yml "${WORKING_TOPOLOGY}" || {
+        die "Topology move failed!"
+    }
+    cat ${WORKING_TOPOLOGY} | grep -v password || {
+        die "Topology read failed!"
+    }
+}
+
+
 function activate_virtualenv () {
 
     set -exuo pipefail
@@ -154,13 +219,28 @@ function compose_pybot_arguments () {
     # - DUT - CSIT test/ subdirectory, set while processing tags.
     # - TAGS - Array variable holding selected tag boolean expressions.
     # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology.
+    # - TEST_CODE - The test selection string from environment or argument.
     # Variables set:
     # - PYBOT_ARGS - String holding part of all arguments for pybot.
     # - EXPANDED_TAGS - Array of strings pybot arguments compiled from tags.
 
     # No explicit check needed with "set -u".
-    PYBOT_ARGS=("--loglevel" "TRACE" "--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}")
-    PYBOT_ARGS+=("--suite" "tests.${DUT}.perf")
+    PYBOT_ARGS=("--loglevel" "TRACE")
+    PYBOT_ARGS+=("--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}")
+
+    case "${TEST_CODE}" in
+        *"device"*)
+            PYBOT_ARGS+=("--suite" "tests.${DUT}.device")
+            ;;
+        *"func"*)
+            PYBOT_ARGS+=("--suite" "tests.${DUT}.func")
+            ;;
+        *"perf"*)
+            PYBOT_ARGS+=("--suite" "tests.${DUT}.perf")
+            ;;
+        *)
+            die "Unknown specification: ${TEST_CODE}"
+    esac
 
     EXPANDED_TAGS=()
     for tag in "${TAGS[@]}"; do
@@ -197,6 +277,38 @@ function copy_archives () {
 }
 
 
+function deactivate_docker_topology () {
+    # Deactivate virtual vpp-device topology by removing containers.
+    #
+    # Variables read:
+    # - NODENESS - Node multiplicity of desired testbed.
+    # - FLAVOR - Node flavor string, usually describing the processor.
+
+    set -exuo pipefail
+
+    case_text="${NODENESS}_${FLAVOR}"
+    case "${case_text}" in
+        "1n_skx")
+            hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3)
+            ssh="ssh root@${hostname} -p 6022"
+            env_vars="$(env | grep CSIT_ | tr '\n' ' ' )"
+            ${ssh} "$(declare -f); deactivate_wrapper ${env_vars}" || {
+                die "Topology cleanup via shim-dcr failed!"
+            }
+            ;;
+        "1n_vbox")
+            enter_mutex || die
+            clean_environment || {
+                die "Topology cleanup locally failed!"
+            }
+            exit_mutex || die
+            ;;
+        *)
+            die "Unknown specification: ${case_text}!"
+    esac
+}
+
+
 function die () {
     # Print the message to standard error end exit with error code specified
     # by the second argument.
@@ -251,6 +363,14 @@ function get_test_code () {
     fi
 
     case "${TEST_CODE}" in
+        *"1n-vbox"*)
+            NODENESS="1n"
+            FLAVOR="vbox"
+            ;;
+        *"1n-skx"*)
+            NODENESS="1n"
+            FLAVOR="skx"
+            ;;
         *"2n-skx"*)
             NODENESS="2n"
             FLAVOR="skx"
@@ -275,6 +395,7 @@ function get_test_tag_string () {
     # Variables read:
     # - GERRIT_EVENT_TYPE - Event type set by gerrit, can be unset.
     # - GERRIT_EVENT_COMMENT_TEXT - Comment text, read for "comment-added" type.
+    # - TEST_CODE - The test selection string from environment or argument.
     # Variables set:
     # - TEST_TAG_STRING - The string following "perftest" in gerrit comment,
     #   or empty.
@@ -283,12 +404,27 @@ function get_test_tag_string () {
 
     trigger=""
     if [[ "${GERRIT_EVENT_TYPE-}" == "comment-added" ]]; then
-        # On parsing error, ${trigger} stays empty.
-        trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \
-            | grep -oE '(perftest$|perftest[[:space:]].+$)')" || true
+        case "${TEST_CODE}" in
+            *"device"*)
+                # On parsing error, ${trigger} stays empty.
+                trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \
+                    | grep -oE '(devicetest$|devicetest[[:space:]].+$)')" \
+                    || true
+                # Set test tags as string.
+                TEST_TAG_STRING="${trigger#$"devicetest"}"
+                ;;
+            *"perf"*)
+                # On parsing error, ${trigger} stays empty.
+                trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \
+                    | grep -oE '(perftest$|perftest[[:space:]].+$)')" \
+                    || true
+                # Set test tags as string.
+                TEST_TAG_STRING="${trigger#$"perftest"}"
+                ;;
+            *)
+                die "Unknown specification: ${TEST_CODE}"
+        esac
     fi
-    # Set test tags as string.
-    TEST_TAG_STRING="${trigger#$"perftest"}"
 }
 
 
@@ -565,6 +701,52 @@ function select_tags () {
 }
 
 
+function select_vpp_device_tags () {
+
+    set -exuo pipefail
+
+    # Variables read:
+    # - TEST_CODE - String affecting test selection, usually jenkins job name.
+    # - TEST_TAG_STRING - String selecting tags, from gerrit comment.
+    #   Can be unset.
+    # Variables set:
+    # - TAGS - Array of processed tag boolean expressions.
+
+    case "${TEST_CODE}" in
+        # Select specific performance tests based on jenkins job type variable.
+        * )
+            if [[ -z "${TEST_TAG_STRING-}" ]]; then
+                # If nothing is specified, we will run pre-selected tests by
+                # following tags. Items of array will be concatenated by OR
+                # in Robot Framework.
+                test_tag_array=()
+            else
+                # If trigger contains tags, split them into array.
+                test_tag_array=(${TEST_TAG_STRING//:/ })
+            fi
+            ;;
+    esac
+
+    TAGS=()
+
+    # We will prefix with perftest to prevent running other tests
+    # (e.g. Functional).
+    prefix="devicetestAND"
+    if [[ "${TEST_CODE}" == "vpp-"* ]]; then
+        # Automatic prefixing for VPP jobs to limit testing.
+        prefix="${prefix}"
+    fi
+    for tag in "${test_tag_array[@]}"; do
+        if [[ ${tag} == "!"* ]]; then
+            # Exclude tags are not prefixed.
+            TAGS+=("${tag}")
+        else
+            TAGS+=("${prefix}${tag}")
+        fi
+    done
+}
+
+
 function select_topology () {
 
     set -exuo pipefail
@@ -573,7 +755,7 @@ function select_topology () {
     # - NODENESS - Node multiplicity of testbed, either "2n" or "3n".
     # - FLAVOR - Node flavor string, currently either "hsw" or "skx".
     # - CSIT_DIR - Path to existing root of local CSIT git repository.
-    # - TOPOLOGIES_DIR - Path to existing directory with available tpologies.
+    # - TOPOLOGIES_DIR - Path to existing directory with available topologies.
     # Variables set:
     # - TOPOLOGIES - Array of paths to suitable topology yaml files.
     # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology.
@@ -582,13 +764,17 @@ function select_topology () {
 
     case_text="${NODENESS}_${FLAVOR}"
     case "${case_text}" in
-        "3n_hsw")
+        "1n_vbox")
             TOPOLOGIES=(
-                        "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml"
-                        "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml"
-                        "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml"
+                        "${TOPOLOGIES_DIR}/vpp_device.template"
                        )
-            TOPOLOGIES_TAGS="3_node_single_link_topo"
+            TOPOLOGIES_TAGS="2_node_single_link_topo"
+            ;;
+        "1n_skx")
+            TOPOLOGIES=(
+                        "${TOPOLOGIES_DIR}/vpp_device.template"
+                       )
+            TOPOLOGIES_TAGS="2_node_single_link_topo"
             ;;
         "2n_skx")
             TOPOLOGIES=(
@@ -606,6 +792,14 @@ function select_topology () {
                        )
             TOPOLOGIES_TAGS="3_node_*_link_topo"
             ;;
+        "3n_hsw")
+            TOPOLOGIES=(
+                        "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml"
+                        "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml"
+                        "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml"
+                       )
+            TOPOLOGIES_TAGS="3_node_single_link_topo"
+            ;;
         *)
             # No falling back to 3n_hsw default, that should have been done
             # by the function which has set NODENESS and FLAVOR.
diff --git a/resources/libraries/bash/function/device.sh b/resources/libraries/bash/function/device.sh
new file mode 100644 (file)
index 0000000..e4ed319
--- /dev/null
@@ -0,0 +1,570 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+# This library defines functions used by multiple entry scripts.
+# Keep functions ordered alphabetically, please.
+
+function activate_wrapper () {
+    # Acts as wrapper for activate docker topology.
+    #
+    # Variables read:
+    # - ${1} - Node multiplicity of desired testbed.
+    # - ${2} - Node flavor string, usually describing the processor.
+    # - ${3} - CSIT-SUT-DCR image name and version.
+
+    set -exuo pipefail
+
+    enter_mutex || die
+    get_available_interfaces "${1}" "${2}" || die
+    start_topology_containers "${3}" || die
+    bind_interfaces_to_containers || die
+    set_env_variables || die
+    print_env_variables || die
+    exit_mutex || die
+}
+
+
+function bind_interfaces_to_containers () {
+    # Bind linux network interface to container and create symlink for PCI
+    # address in container.
+    #
+    # Variables read:
+    # - DCR_UUIDS - Docker Container UUIDs.
+    # - DCR_CPIDS - Docker Container PIDs (namespaces).
+    # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
+    # - PCI_ADDR - PCI address of network device.
+    # - TG_NETDEVS - List of network devices allocated to TG container.
+    # Variables set:
+    # - NETDEV - Linux network interface.
+
+    set -exuo pipefail
+
+    for NETDEV in "${TG_NETDEVS[@]}"; do
+        get_pci_addr || die
+        link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
+            die "Reading symlink for PCI address failed!"
+        }
+        cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
+
+        sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
+            die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
+        }
+        docker exec "${DCR_UUIDS[tg]}" ${cmd} || {
+            die "Linking PCI address in container failed!"
+        }
+    done
+    for NETDEV in "${DUT1_NETDEVS[@]}"; do
+        get_pci_addr || die
+        link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
+            die "Reading symlink for PCI address failed!"
+        }
+        cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
+
+        sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
+            die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
+        }
+        docker exec "${DCR_UUIDS[dut1]}" ${cmd} ||  {
+            die "Linking PCI address in container failed!"
+        }
+    done
+}
+
+
+function bind_interfaces_to_driver () {
+    # Bind network interface specified by parameter to driver specified by
+    # parameter.
+    #
+    # Variables read:
+    # - ADDR - PCI address of network interface.
+    # - DRIVER - Kernel driver.
+
+    pci_path="/sys/bus/pci/devices/${ADDR}"
+    drv_path="/sys/bus/pci/drivers/${DRIVER}"
+    vd="$(cat ${pci_path}/vendor ${pci_path}/device)" || {
+        die "Failed to retrieve interface details!"
+    }
+    set +e
+    echo ${vd} | sudo tee ${drv_path}/new_id
+    set -e
+    echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || {
+        die "Failed to unbind interface ${ADDR}!"
+    }
+    echo ${ADDR} | sudo tee ${drv_path}/bind || {
+        die "Failed to bind interface ${ADDR}!"
+    }
+}
+
+
+function clean_environment () {
+    # Cleanup environment by removing topology containers and binding
+    # interfaces back to original driver.
+    #
+    # Variables read:
+    # - DCR_UUIDS - Docker Container UUIDs.
+    # - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container.
+    # - TG_PCIDEVS - List of PCI addresses of devices of TG container.
+    # Variables set:
+    # - ADDR - PCI address of network interface.
+    # - DRIVER - Kernel driver.
+
+    set -exuo pipefail
+
+    # Kill docker containers.
+    docker rm --force "${DCR_UUIDS[@]}" || die "Cleanup containers failed!"
+
+    # Rebind interfaces back to kernel drivers.
+    for ADDR in ${TG_PCIDEVS[@]}; do
+        DRIVER="${TG_DRIVERS[0]}"
+        bind_interfaces_to_driver || die
+    done
+    for ADDR in ${DUT1_PCIDEVS[@]}; do
+        DRIVER="${DUT1_DRIVERS[0]}"
+        bind_interfaces_to_driver || die
+    done
+}
+
+
+function clean_environment_on_exit () {
+    # Cleanup environment by removing topology containers and binding
+    # interfaces back to original driver only if exit code is not 0.
+    # This function acts as workaround as 'set -eu' does not trigger ERR trap.
+
+    if [ $? -ne 0 ]; then
+        clean_environment || die
+    fi
+}
+
+
+function deactivate_wrapper () {
+    # Acts as wrapper for deactivate docker topology.
+    #
+    # Variables read:
+    # - ${@} - CSIT environment variables.
+
+    set -exuo pipefail
+
+    enter_mutex || die
+    read_env_variables "${@}" || die
+    clean_environment || die
+    exit_mutex || die
+}
+
+
+function die () {
+    # Print the message to standard error end exit with error code specified
+    # by the second argument.
+    #
+    # Hardcoded values:
+    # - The default error message.
+    # Arguments:
+    # - ${1} - The whole error message, be sure to quote. Optional
+    # - ${2} - the code to exit with, default: 1.
+
+    set -x
+    set +eu
+    warn "${1:-Unspecified run-time error occurred!}"
+    exit "${2:-1}"
+}
+
+
+function enter_mutex () {
+    # Enter mutual exclusion for protecting execution from starvation and
+    # deadlock.
+
+    set -exuo pipefail
+
+    mutex_timeout=3600
+    mutex_file="/tmp/mutex_file"
+
+    # Create mutex.
+    exec {lock_fd}>${mutex_file} || {
+        die "Mutex enter failed!"
+    }
+    flock --timeout "${mutex_timeout}" "${lock_fd}" || {
+        die "Calling flock() failed!"
+    }
+    # ----------------------
+    # Enter mutex succeeded.
+    warn "Mutex enter succeeded for PID $$."
+}
+
+
+function exit_mutex () {
+    # Exit mutual exclusion.
+
+    set -exuo pipefail
+
+    # ---------------------
+    # Remove mutex so we are not blocking others anymore.
+    flock -u "${lock_fd}" || {
+        die "Mutex destroy failed!"
+    }
+    warn "Mutex leave succeeded for PID $$."
+}
+
+
+function get_available_interfaces () {
+    # Find and get available Virtual functions.
+    #
+    # Arguments:
+    # - ${1} - Node flavor string, usually describing the processor and node
+    # multiplicity of desired testbed, separated by underscore.
+    # Variables set:
+    # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
+    # - DUT1_PCIDEVS - List of PCI addresses allocated to DUT1 container.
+    # - DUT1_NETMACS - List of MAC addresses allocated to DUT1 container.
+    # - DUT1_DRIVERS - List of interface drivers to DUT1 container.
+    # - TG_NETDEVS - List of network devices allocated to TG container.
+    # - TG_PCIDEVS - List of PCI addresses allocated to TG container.
+    # - TG_NETMACS - List of MAC addresses allocated to TG container.
+    # - TG_DRIVERS - List of interface drivers to TG container.
+
+    set -exuo pipefail
+
+    # Following code is specifing VFs ID based on nodeness and flavor.
+    # As there is great variability in hardware configuration outside LF,
+    # from bootstrap architecure point of view these are considered as flavors.
+    # Anyone can override flavor for its own machine and add condition here.
+    # See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
+    case_text="${1}_${2}"
+    case "${case_text}" in
+        "1n_skx")
+            # Add Intel Corporation XL710/X710 Virtual Function to the
+            # whitelist.
+            pci_id="0x154c"
+            tg_netdev=(enp24)
+            dut1_netdev=(enp59)
+            ;;
+        "1n_vbox")
+            # Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
+            # whitelist.
+            pci_id="0x100f"
+            tg_netdev=(eth1 eth2)
+            dut1_netdev=(eth3 eth4)
+            ;;
+        *)
+            die "Unknown specification: ${case_text}!"
+    esac
+
+    net_path="/sys/bus/pci/devices/*/net/*"
+
+    # TG side of connections.
+    TG_NETDEVS=()
+    TG_PCIDEVS=()
+    TG_NETMACS=()
+    TG_DRIVERS=()
+    # DUT1 side of connections.
+    DUT1_NETDEVS=()
+    DUT1_PCIDEVS=()
+    DUT1_NETMACS=()
+    DUT1_DRIVERS=()
+
+    # Following code is filtering available VFs represented by network device
+    # name. Only allowed VFs PCI IDs are used.
+    for netdev in \
+        $(find ${net_path} -type d -name . -o -prune -exec basename '{}' ';');
+    do
+        if grep -q "${pci_id}" "/sys/class/net/${netdev}/device/device"; then
+            # We will filter to TG/DUT1 side of connection (this can be in
+            # future overriden by more advanced conditions for mapping).
+            for sub in ${tg_netdev[@]}; do
+                if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
+                    tg_side+=(${netdev})
+                fi
+            done
+            for sub in ${dut1_netdev[@]}; do
+                if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
+                    dut1_side+=(${netdev})
+                fi
+            done
+        fi
+    done
+
+    for netdev in "${tg_side[@]::2}"; do
+        TG_NETDEVS+=(${netdev})
+    done
+    for netdev in "${dut1_side[@]::2}"; do
+        DUT1_NETDEVS+=(${netdev})
+    done
+
+    for NETDEV in "${TG_NETDEVS[@]}"; do
+        get_pci_addr
+        get_mac_addr
+        get_krn_driver
+        TG_PCIDEVS+=(${PCI_ADDR})
+        TG_NETMACS+=(${MAC_ADDR})
+        TG_DRIVERS+=(${KRN_DRIVER})
+    done
+    for NETDEV in "${DUT1_NETDEVS[@]}"; do
+        get_pci_addr
+        get_mac_addr
+        get_krn_driver
+        DUT1_PCIDEVS+=(${PCI_ADDR})
+        DUT1_NETMACS+=(${MAC_ADDR})
+        DUT1_DRIVERS+=(${KRN_DRIVER})
+    done
+
+    # We need at least two interfaces for TG/DUT1 for building topology.
+    if [ "${#TG_NETDEVS[@]}" -ne 2 ] || [ "${#DUT1_NETDEVS[@]}" -ne 2 ]; then
+        die "Not enough linux network interfaces found!"
+    fi
+    if [ "${#TG_PCIDEVS[@]}" -ne 2 ] || [ "${#DUT1_PCIDEVS[@]}" -ne 2 ]; then
+        die "Not enough pci interfaces found!"
+    fi
+}
+
+
+function get_krn_driver () {
+    # Get kernel driver from linux network device name.
+    #
+    # Variables read:
+    # - PCI_ADDR - PCI address of network device.
+    # Variables set:
+    # - KRN_DRIVER - Kernel driver of network device.
+
+    set -exuo pipefail
+
+    pci_path="/sys/bus/pci/devices/${PCI_ADDR}"
+    KRN_DRIVER="$(basename $(readlink -f ${pci_path}/driver))" || {
+        die "Failed to get kernel driver of PCI interface!"
+    }
+}
+
+
+function get_mac_addr () {
+    # Get MAC address from linux network device name.
+    #
+    # Variables read:
+    # - NETDEV - Linux network device name.
+    # Variables set:
+    # - MAC_ADDR - MAC address of network device.
+
+    set -exuo pipefail
+
+    if [ -d /sys/class/net/${NETDEV}/device ]; then
+        MAC_ADDR="$(</sys/class/net/${NETDEV}/address)" || {
+            die "Failed to get MAC address of linux network interface!"
+        }
+    fi
+}
+
+
+function get_pci_addr () {
+    # Get PCI address in <domain>:<bus:<device>.<func> format from linux network
+    # device name.
+    #
+    # Variables read:
+    # - NETDEV - Linux network device name.
+    # Variables set:
+    # - PCI_ADDR - PCI address of network device.
+
+    set -exuo pipefail
+
+    if [ -d /sys/class/net/${NETDEV}/device ]; then
+        PCI_ADDR=$(basename $(readlink /sys/class/net/${NETDEV}/device)) || {
+            die "Failed to get PCI address of linux network interface!"
+        }
+    fi
+    if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then
+        die "PCI device ${NETDEV} doesn't exist!"
+    fi
+}
+
+
+function installed () {
+
+    set -exuo pipefail
+
+    # Check if the given utility is installed. Fail if not installed.
+    #
+    # Arguments:
+    # - ${1} - Utility to check.
+    # Returns:
+    # - 0 - If command is installed.
+    # - 1 - If command is not installed.
+
+    command -v "${1}"
+}
+
+
+function print_env_variables () {
+    # Get environment variables prefixed by CSIT_.
+
+    set -exuo pipefail
+
+    env | grep CSIT_
+}
+
+
+function read_env_variables () {
+    # Read environment variables from parameters.
+    #
+    # Arguments:
+    # - ${@} - Variables passed as an argument.
+
+    set -exuo pipefail
+
+    for param in "$@"; do
+        export "${param}"
+    done
+    declare -gA DCR_UUIDS
+    DCR_UUIDS+=([tg]="${CSIT_TG_UUID}")
+    DCR_UUIDS+=([dut1]="${CSIT_DUT1_UUID}")
+    TG_PCIDEVS=("${CSIT_TG_INTERFACES_PORT1_PCI}")
+    TG_DRIVERS=("${CSIT_TG_INTERFACES_PORT1_DRV}")
+    TG_PCIDEVS+=("${CSIT_TG_INTERFACES_PORT2_PCI}")
+    TG_DRIVERS+=("${CSIT_TG_INTERFACES_PORT2_DRV}")
+    DUT1_PCIDEVS=("${CSIT_DUT1_INTERFACES_PORT1_PCI}")
+    DUT1_DRIVERS=("${CSIT_DUT1_INTERFACES_PORT1_DRV}")
+    DUT1_PCIDEVS+=("${CSIT_DUT1_INTERFACES_PORT2_PCI}")
+    DUT1_DRIVERS+=("${CSIT_DUT1_INTERFACES_PORT2_DRV}")
+}
+
+
+function set_env_variables () {
+    # Set environment variables.
+    #
+    # Variables read:
+    # - DCR_UUIDS - Docker Container UUIDs.
+    # - DCR_PORTS - Docker Container's SSH ports.
+    # - DUT1_NETMACS - List of network devices MAC addresses of DUT1 container.
+    # - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container.
+    # - DUT1_DRIVERS - List of interface drivers to DUT1 container.
+    # - TG_NETMACS - List of network devices MAC addresses of TG container.
+    # - TG_PCIDEVS - List of PCI addresses of devices of TG container.
+    # - TG_DRIVERS - List of interface drivers to TG container.
+
+    set -exuo pipefail
+
+    set -a
+    CSIT_TG_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
+        die "Reading hostname IP address failed!"
+    }
+    CSIT_TG_PORT="${DCR_PORTS[tg]#*:}"
+    CSIT_TG_UUID="${DCR_UUIDS[tg]}"
+    CSIT_TG_ARCH="$(uname -i)" || {
+        die "Reading machine architecture failed!"
+    }
+    CSIT_DUT1_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
+        die "Reading hostname IP address failed!"
+    }
+    CSIT_DUT1_PORT="${DCR_PORTS[dut1]#*:}"
+    CSIT_DUT1_UUID="${DCR_UUIDS[dut1]}"
+    CSIT_DUT1_ARCH="$(uname -i)" || {
+        die "Reading machine architecture failed!"
+    }
+    CSIT_TG_INTERFACES_PORT1_MAC="${TG_NETMACS[0]}"
+    CSIT_TG_INTERFACES_PORT1_PCI="${TG_PCIDEVS[0]}"
+    CSIT_TG_INTERFACES_PORT1_DRV="${TG_DRIVERS[0]}"
+    CSIT_TG_INTERFACES_PORT2_MAC="${TG_NETMACS[1]}"
+    CSIT_TG_INTERFACES_PORT2_PCI="${TG_PCIDEVS[1]}"
+    CSIT_TG_INTERFACES_PORT2_DRV="${TG_DRIVERS[1]}"
+    CSIT_DUT1_INTERFACES_PORT1_MAC="${DUT1_NETMACS[0]}"
+    CSIT_DUT1_INTERFACES_PORT1_PCI="${DUT1_PCIDEVS[0]}"
+    CSIT_DUT1_INTERFACES_PORT1_DRV="${DUT1_DRIVERS[0]}"
+    CSIT_DUT1_INTERFACES_PORT2_MAC="${DUT1_NETMACS[1]}"
+    CSIT_DUT1_INTERFACES_PORT2_PCI="${DUT1_PCIDEVS[1]}"
+    CSIT_DUT1_INTERFACES_PORT2_DRV="${DUT1_DRIVERS[1]}"
+    set +a
+}
+
+
+function start_topology_containers () {
+    # Starts csit-sut-dcr docker containers for TG/DUT1.
+    #
+    # Variables read:
+    # - CSIT_DIR - Path to existing root of local CSIT git repository.
+    # Variables set:
+    # - DCR_UUIDS - Docker Container UUIDs.
+    # - DCR_PORTS - Docker Container SSH TCP ports.
+    # - DCR_CPIDS - Docker Container PIDs (namespaces).
+
+    set -exuo pipefail
+
+    if ! installed docker; then
+        die "Docker not present. Please install before continue!"
+    fi
+
+    # If the IMAGE is not already loaded then docker run will pull the IMAGE,
+    # and all image dependencies, before it starts the container.
+    dcr_image="${1}"
+    # Run the container in the background and print the new container ID.
+    dcr_stc_params="--detach=true "
+    # Give extended privileges to this container. A "privileged" container is
+    # given access to all devices and able to run nested containers.
+    dcr_stc_params+="--privileged "
+    # Publish all exposed ports to random ports on the host interfaces.
+    dcr_stc_params+="--publish-all "
+    # Automatically remove the container when it exits.
+    dcr_stc_params+="--rm "
+    # Size of /dev/shm.
+    dcr_stc_params+="--shm-size 512M "
+    # Override access to PCI bus by attaching a filesystem mount to the
+    # container.
+    dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices "
+    # Mount vfio to be able to bind to see binded interfaces. We cannot use
+    # --device=/dev/vfio as this does not see newly binded interfaces.
+    dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
+
+    # Docker Container UUIDs.
+    declare -gA DCR_UUIDS
+    # Docker Container SSH TCP ports.
+    declare -gA DCR_PORTS
+    # Docker Container PIDs (namespaces).
+    declare -gA DCR_CPIDS
+
+    # Run TG and DUT1. As initial version we do support only 2-node.
+    params=(${dcr_stc_params} --name csit-tg-$(uuidgen) ${dcr_image})
+    DCR_UUIDS+=([tg]="$(docker run "${params[@]}")") || {
+        die "Failed to start TG docker container!"
+    }
+    params=(${dcr_stc_params} --name csit-dut1-$(uuidgen) ${dcr_image})
+    DCR_UUIDS+=([dut1]="$(docker run "${params[@]}")") || {
+        die "Failed to start DUT1 docker container!"
+    }
+
+    trap 'clean_environment_on_exit' EXIT || {
+        die "Trap attempt failed, please cleanup manually. Aborting!"
+    }
+
+    # Get Containers TCP ports.
+    params=(${DCR_UUIDS[tg]})
+    DCR_PORTS+=([tg]="$(docker port "${params[@]}")") || {
+        die "Failed to get port of TG docker container!"
+    }
+    params=(${DCR_UUIDS[dut1]})
+    DCR_PORTS+=([dut1]="$(docker port "${params[@]}")") || {
+        die "Failed to get port of DUT1 docker container!"
+    }
+
+    # Get Containers PIDs.
+    params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[tg]})
+    DCR_CPIDS+=([tg]="$(docker inspect "${params[@]}")") || {
+        die "Failed to get PID of TG docker container!"
+    }
+    params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[dut1]})
+    DCR_CPIDS+=([dut1]="$(docker inspect "${params[@]}")") || {
+        die "Failed to get PID of DUT1 docker container!"
+    }
+}
+
+function warn () {
+    # Print the message to standard error.
+    #
+    # Arguments:
+    # - ${@} - The text of the message.
+
+    echo "$@" >&2
+}
diff --git a/topologies/available/vpp_device.template b/topologies/available/vpp_device.template
new file mode 100644 (file)
index 0000000..c855a71
--- /dev/null
@@ -0,0 +1,45 @@
+---
+metadata:
+  version: 0.1
+  schema:
+    - resources/topology_schemas/2_node_topology.sch.yaml
+    - resources/topology_schemas/topology.sch.yaml
+  tags: [dcr, 2-node]
+
+nodes:
+  TG:
+    type: TG
+    host: "${CSIT_TG_HOST}"
+    arch: "${CSIT_TG_ARCH}"
+    port: ${CSIT_TG_PORT}
+    username: root
+    password: Csit1234
+    interfaces:
+      port1:
+        mac_address: "${CSIT_TG_INTERFACES_PORT1_MAC}"
+        pci_address: "${CSIT_TG_INTERFACES_PORT1_PCI}"
+        link: link1
+        driver: "${CSIT_TG_INTERFACES_PORT1_DRV}"
+      port2:
+        mac_address: "${CSIT_TG_INTERFACES_PORT2_MAC}"
+        pci_address: "${CSIT_TG_INTERFACES_PORT2_PCI}"
+        link: link2
+        driver: "${CSIT_TG_INTERFACES_PORT2_DRV}"
+  DUT1:
+    type: DUT
+    host: "${CSIT_DUT1_HOST}"
+    arch: "${CSIT_DUT1_ARCH}"
+    port: ${CSIT_DUT1_PORT}
+    username: root
+    password: Csit1234
+    interfaces:
+      port1:
+        mac_address: "${CSIT_DUT1_INTERFACES_PORT1_MAC}"
+        pci_address: "${CSIT_DUT1_INTERFACES_PORT1_PCI}"
+        link: link1
+        driver: "${CSIT_DUT1_INTERFACES_PORT1_DRV}"
+      port2:
+        mac_address: "${CSIT_DUT1_INTERFACES_PORT2_MAC}"
+        pci_address: "${CSIT_DUT1_INTERFACES_PORT2_PCI}"
+        link: link2
+        driver: "${CSIT_DUT1_INTERFACES_PORT2_DRV}"

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.