-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
set -exuo pipefail
# This library defines functions used by multiple entry scripts.
+# Deliberately not depending on common.sh to allow standalone usage.
# Keep functions ordered alphabetically, please.
function activate_wrapper () {
+
# Acts as wrapper for activate docker topology.
#
# Variables read:
function bind_interfaces_to_containers () {
+
# Bind linux network interface to container and create symlink for PCI
# address in container.
#
function bind_interfaces_to_driver () {
+
# Bind network interface specified by parameter to driver specified by
# parameter.
#
# - ADDR - PCI address of network interface.
# - DRIVER - Kernel driver.
+ set -exuo pipefail
+
pci_path="/sys/bus/pci/devices/${ADDR}"
drv_path="/sys/bus/pci/drivers/${DRIVER}"
- vd="$(cat ${pci_path}/vendor ${pci_path}/device)" || {
+ vd=$(cat ${pci_path}/vendor ${pci_path}/device) || {
die "Failed to retrieve interface details!"
}
set +e
function clean_environment () {
- # Cleanup environment by removing topology containers and binding
- # interfaces back to original driver.
+
+ # Cleanup environment by removing topology containers and shared volumes
+ # and binding interfaces back to original driver.
#
# Variables read:
# - DCR_UUIDS - Docker Container UUIDs.
# Kill docker containers.
docker rm --force "${DCR_UUIDS[@]}" || die "Cleanup containers failed!"
+ # Check if there are some leftover containers and remove all. Command will
+ # not fail in case there are no containers to remove.
+ docker rm --force $(docker ps -q --filter name=${DCR_UUIDS[dut1]}) || {
+ warn "Failed to remove hanged containers or nothing to remove!"
+ }
+
# Rebind interfaces back to kernel drivers.
for ADDR in ${TG_PCIDEVS[@]}; do
DRIVER="${TG_DRIVERS[0]}"
function clean_environment_on_exit () {
+
# Cleanup environment by removing topology containers and binding
# interfaces back to original driver only if exit code is not 0.
# This function acts as workaround as 'set -eu' does not trigger ERR trap.
+ set -exuo pipefail
+
if [ $? -ne 0 ]; then
clean_environment || die
fi
function deactivate_wrapper () {
+
# Acts as wrapper for deactivate docker topology.
#
# Variables read:
function die () {
+
# Print the message to standard error end exit with error code specified
# by the second argument.
#
+ # Duplicate of common.sh function, as this file is also used standalone.
+ #
# Hardcoded values:
# - The default error message.
# Arguments:
function enter_mutex () {
+
# Enter mutual exclusion for protecting execution from starvation and
# deadlock.
function exit_mutex () {
+
# Exit mutual exclusion.
set -exuo pipefail
function get_available_interfaces () {
+
# Find and get available Virtual functions.
#
# Arguments:
- # - ${1} - Node flavor string, usually describing the processor and node
- # multiplicity of desired testbed, separated by underscore.
+ # - ${1} - Nodeness, as set by common.sh get_test_code.
+ # - ${2} - Flavor, as set by common.sh get_test_code.
# Variables set:
# - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
# - DUT1_PCIDEVS - List of PCI addresses allocated to DUT1 container.
tg_netdev=(enp24)
dut1_netdev=(enp59)
;;
- "1n_vbox")
+ "1n_tx2")
+ # Add Intel Corporation XL710/X710 Virtual Function to the
+ # whitelist.
+ pci_id="0x154c"
+ tg_netdev=(enp5s2 enp5s3 enp5s4 enp5s5
+ enp5s6 enp5s7 enp5s8 enp5s9)
+ tg_netdev+=(enp8s2 enp8s3 enp8s4 enp8s5
+ enp8s6 enp8s7 enp8s8 enp8s9)
+ tg_netdev+=(enp8s10 enp8s11 enp8s12 enp8s13
+ enp8s14 enp8s15 enp8s16 enp8s17)
+ dut1_netdev=(enp133s2 enp133s3 enp133s4 enp133s5
+ enp133s6 enp133s7 enp133s8 enp133s9)
+ dut1_netdev+=(enp133s10 enp133s11 enp133s12 enp133s13
+ enp133s14 enp133s15 enp133s16 enp133s17)
+ dut1_netdev+=(enp5s10 enp5s11 enp5s12 enp5s13
+ enp5s14 enp5s15 enp5s16 enp5s17)
+ ;;
+ "1n_vbox")
# Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
# whitelist.
pci_id="0x100f"
- tg_netdev=(eth1 eth2)
- dut1_netdev=(eth3 eth4)
+ tg_netdev=(enpTGa enpTGb)
+ dut1_netdev=(enpSUTa enpSUTb)
;;
*)
die "Unknown specification: ${case_text}!"
esac
- net_path="/sys/bus/pci/devices/*/net/*"
+ device_count=2
# TG side of connections.
TG_NETDEVS=()
DUT1_NETMACS=()
DUT1_DRIVERS=()
- # Following code is filtering available VFs represented by network device
- # name. Only allowed VFs PCI IDs are used.
- for netdev in \
- $(find ${net_path} -type d -name . -o -prune -exec basename '{}' ';');
+ # Find the first ${device_count} number of available TG Linux network
+ # VF device names. Only allowed VF PCI IDs are filtered.
+ for netdev in ${tg_netdev[@]}
do
- if grep -q "${pci_id}" "/sys/class/net/${netdev}/device/device"; then
- # We will filter to TG/DUT1 side of connection (this can be in
- # future overriden by more advanced conditions for mapping).
- for sub in ${tg_netdev[@]}; do
- if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
- tg_side+=(${netdev})
- fi
- done
- for sub in ${dut1_netdev[@]}; do
- if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
- dut1_side+=(${netdev})
- fi
- done
+ for netdev_path in $(grep -l "${pci_id}" \
+ /sys/class/net/${netdev}*/device/device \
+ 2> /dev/null)
+ do
+ if [[ ${#TG_NETDEVS[@]} -lt ${device_count} ]]; then
+ tg_netdev_name=$(dirname ${netdev_path})
+ tg_netdev_name=$(dirname ${tg_netdev_name})
+ TG_NETDEVS+=($(basename ${tg_netdev_name}))
+ else
+ break
+ fi
+ done
+ if [[ ${#TG_NETDEVS[@]} -eq ${device_count} ]]; then
+ break
fi
done
- for netdev in "${tg_side[@]::2}"; do
- TG_NETDEVS+=(${netdev})
- done
- for netdev in "${dut1_side[@]::2}"; do
- DUT1_NETDEVS+=(${netdev})
+ i=0
+ for netdev in "${TG_NETDEVS[@]}"; do
+ # Find the index of selected tg netdev among tg_netdevs
+ # e.g. enp8s5f7 is a vf of netdev enp8s5 with index 11
+ # and the corresponding dut1 netdev is enp133s13.
+ while [[ "${netdev}" != "${tg_netdev[$i]}"* ]]; do
+ ((i++))
+ done
+ # Rename tg netdev to dut1 netdev
+ # e.g. enp8s5f7 -> enp133s13f7
+ DUT1_NETDEVS+=(${netdev/${tg_netdev[$i]}/${dut1_netdev[$i]}})
+ # Don't need to reset i, all netdevs are sorted.
done
for NETDEV in "${TG_NETDEVS[@]}"; do
function get_krn_driver () {
+
# Get kernel driver from linux network device name.
#
# Variables read:
function get_mac_addr () {
+
# Get MAC address from linux network device name.
#
# Variables read:
function get_pci_addr () {
+
# Get PCI address in <domain>:<bus:<device>.<func> format from linux network
# device name.
#
function installed () {
- set -exuo pipefail
-
# Check if the given utility is installed. Fail if not installed.
#
+ # Duplicate of common.sh function, as this file is also used standalone.
+ #
# Arguments:
# - ${1} - Utility to check.
# Returns:
# - 0 - If command is installed.
# - 1 - If command is not installed.
+ set -exuo pipefail
+
command -v "${1}"
}
function print_env_variables () {
+
# Get environment variables prefixed by CSIT_.
set -exuo pipefail
- env | grep CSIT_
+ env | grep CSIT_ || true
}
function read_env_variables () {
+
# Read environment variables from parameters.
#
# Arguments:
# - ${@} - Variables passed as an argument.
+ # Variables read, set or exported: Multiple,
+ # see the code for the current list.
+ # TODO: Do we need to list them and their meanings?
set -exuo pipefail
function set_env_variables () {
+
# Set environment variables.
#
# Variables read:
# - TG_NETMACS - List of network devices MAC addresses of TG container.
# - TG_PCIDEVS - List of PCI addresses of devices of TG container.
# - TG_DRIVERS - List of interface drivers to TG container.
+ # Variables set: TODO.
set -exuo pipefail
function start_topology_containers () {
+
# Starts csit-sut-dcr docker containers for TG/DUT1.
#
# Variables read:
# Mount vfio to be able to bind to see binded interfaces. We cannot use
# --device=/dev/vfio as this does not see newly binded interfaces.
dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
+ # Mount nested_vm image to be able to run VM tests.
+ dcr_stc_params+="--volume /var/lib/vm/vhost-nested.img:/var/lib/vm/vhost-nested.img "
+ # Mount docker.sock to be able to use docker deamon of the host.
+ dcr_stc_params+="--volume /var/run/docker.sock:/var/run/docker.sock "
+ # Mount /opt/boot/ where VM kernel and initrd are located.
+ dcr_stc_params+="--volume /opt/boot/:/opt/boot/ "
+ # Mount host hugepages for VMs.
+ dcr_stc_params+="--volume /dev/hugepages/:/dev/hugepages/ "
# Docker Container UUIDs.
declare -gA DCR_UUIDS
# Run TG and DUT1. As initial version we do support only 2-node.
params=(${dcr_stc_params} --name csit-tg-$(uuidgen) ${dcr_image})
- DCR_UUIDS+=([tg]="$(docker run "${params[@]}")") || {
+ DCR_UUIDS+=([tg]=$(docker run "${params[@]}")) || {
die "Failed to start TG docker container!"
}
params=(${dcr_stc_params} --name csit-dut1-$(uuidgen) ${dcr_image})
- DCR_UUIDS+=([dut1]="$(docker run "${params[@]}")") || {
+ DCR_UUIDS+=([dut1]=$(docker run "${params[@]}")) || {
die "Failed to start DUT1 docker container!"
}
# Get Containers TCP ports.
params=(${DCR_UUIDS[tg]})
- DCR_PORTS+=([tg]="$(docker port "${params[@]}")") || {
+ DCR_PORTS+=([tg]=$(docker port "${params[@]}")) || {
die "Failed to get port of TG docker container!"
}
params=(${DCR_UUIDS[dut1]})
- DCR_PORTS+=([dut1]="$(docker port "${params[@]}")") || {
+ DCR_PORTS+=([dut1]=$(docker port "${params[@]}")) || {
die "Failed to get port of DUT1 docker container!"
}
# Get Containers PIDs.
params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[tg]})
- DCR_CPIDS+=([tg]="$(docker inspect "${params[@]}")") || {
+ DCR_CPIDS+=([tg]=$(docker inspect "${params[@]}")) || {
die "Failed to get PID of TG docker container!"
}
params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[dut1]})
- DCR_CPIDS+=([dut1]="$(docker inspect "${params[@]}")") || {
+ DCR_CPIDS+=([dut1]=$(docker inspect "${params[@]}")) || {
die "Failed to get PID of DUT1 docker container!"
}
}
function warn () {
# Print the message to standard error.
#
+ # Duplicate of common.sh function, as this file is also used standalone.
+ #
# Arguments:
# - ${@} - The text of the message.
+ set -exuo pipefail
+
echo "$@" >&2
}