1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 # This library defines functions used by multiple entry scripts.
17 # Deliberately not depending on common.sh to allow standalone usage.
18 # Keep functions ordered alphabetically, please.
20 function activate_wrapper () {
22 # Acts as wrapper for activate docker topology.
25 # - ${1} - Node multiplicity of desired testbed.
26 # - ${2} - Node flavor string, usually describing the processor.
27 # - ${3} - CSIT-SUT-DCR image name and version.
32 get_available_interfaces "${1}" "${2}" || die
33 start_topology_containers "${3}" || die
34 bind_interfaces_to_containers || die
35 set_env_variables || die
36 print_env_variables || die
41 function bind_interfaces_to_containers () {
43 # Bind linux network interface to container and create symlink for PCI
44 # address in container.
47 # - DCR_UUIDS - Docker Container UUIDs.
48 # - DCR_CPIDS - Docker Container PIDs (namespaces).
49 # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
50 # - PCI_ADDR - PCI address of network device.
51 # - TG_NETDEVS - List of network devices allocated to TG container.
53 # - NETDEV - Linux network interface.
57 for NETDEV in "${TG_NETDEVS[@]}"; do
59 link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
60 die "Reading symlink for PCI address failed!"
62 cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
64 sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
65 die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
67 docker exec "${DCR_UUIDS[tg]}" ${cmd} || {
68 die "Linking PCI address in container failed!"
71 for NETDEV in "${DUT1_NETDEVS[@]}"; do
73 link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
74 die "Reading symlink for PCI address failed!"
76 cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
78 sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
79 die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
81 docker exec "${DCR_UUIDS[dut1]}" ${cmd} || {
82 die "Linking PCI address in container failed!"
88 function bind_interfaces_to_driver () {
90 # Bind network interface specified by parameter to driver specified by
94 # - ADDR - PCI address of network interface.
95 # - DRIVER - Kernel driver.
99 pci_path="/sys/bus/pci/devices/${ADDR}"
100 drv_path="/sys/bus/pci/drivers/${DRIVER}"
101 vd=$(cat ${pci_path}/vendor ${pci_path}/device) || {
102 die "Failed to retrieve interface details!"
105 echo ${vd} | sudo tee ${drv_path}/new_id
107 echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || {
108 die "Failed to unbind interface ${ADDR}!"
110 echo ${ADDR} | sudo tee ${drv_path}/bind || {
111 die "Failed to bind interface ${ADDR}!"
116 function clean_environment () {
118 # Cleanup environment by removing topology containers and shared volumes
119 # and binding interfaces back to original driver.
122 # - DCR_UUIDS - Docker Container UUIDs.
123 # - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container.
124 # - TG_PCIDEVS - List of PCI addresses of devices of TG container.
126 # - ADDR - PCI address of network interface.
127 # - DRIVER - Kernel driver.
131 # Kill docker containers.
132 docker rm --force "${DCR_UUIDS[@]}" || die "Cleanup containers failed!"
134 # Check if there are some leftover containers and remove all. Command will
135 # not fail in case there are no containers to remove.
136 docker rm --force $(docker ps -q --filter name=${DCR_UUIDS[dut1]}) || {
137 warn "Failed to remove hanged containers or nothing to remove!"
140 # Rebind interfaces back to kernel drivers.
141 for ADDR in ${TG_PCIDEVS[@]}; do
142 DRIVER="${TG_DRIVERS[0]}"
143 bind_interfaces_to_driver || die
145 for ADDR in ${DUT1_PCIDEVS[@]}; do
146 DRIVER="${DUT1_DRIVERS[0]}"
147 bind_interfaces_to_driver || die
152 function clean_environment_on_exit () {
154 # Cleanup environment by removing topology containers and binding
155 # interfaces back to original driver only if exit code is not 0.
156 # This function acts as workaround as 'set -eu' does not trigger ERR trap.
160 if [ $? -ne 0 ]; then
161 clean_environment || die
166 function deactivate_wrapper () {
168 # Acts as wrapper for deactivate docker topology.
171 # - ${@} - CSIT environment variables.
176 read_env_variables "${@}" || die
177 clean_environment || die
184 # Print the message to standard error end exit with error code specified
185 # by the second argument.
187 # Duplicate of common.sh function, as this file is also used standalone.
190 # - The default error message.
192 # - ${1} - The whole error message, be sure to quote. Optional
193 # - ${2} - the code to exit with, default: 1.
197 warn "${1:-Unspecified run-time error occurred!}"
202 function enter_mutex () {
204 # Enter mutual exclusion for protecting execution from starvation and
210 mutex_file="/tmp/mutex_file"
213 exec {lock_fd}>${mutex_file} || {
214 die "Mutex enter failed!"
216 flock --timeout "${mutex_timeout}" "${lock_fd}" || {
217 die "Calling flock() failed!"
219 # ----------------------
220 # Enter mutex succeeded.
221 warn "Mutex enter succeeded for PID $$."
225 function exit_mutex () {
227 # Exit mutual exclusion.
231 # ---------------------
232 # Remove mutex so we are not blocking others anymore.
233 flock -u "${lock_fd}" || {
234 die "Mutex destroy failed!"
236 warn "Mutex leave succeeded for PID $$."
240 function get_available_interfaces () {
242 # Find and get available Virtual functions.
245 # - ${1} - Nodeness, as set by common.sh get_test_code.
246 # - ${2} - Flavor, as set by common.sh get_test_code.
248 # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
249 # - DUT1_PCIDEVS - List of PCI addresses allocated to DUT1 container.
250 # - DUT1_NETMACS - List of MAC addresses allocated to DUT1 container.
251 # - DUT1_DRIVERS - List of interface drivers to DUT1 container.
252 # - TG_NETDEVS - List of network devices allocated to TG container.
253 # - TG_PCIDEVS - List of PCI addresses allocated to TG container.
254 # - TG_NETMACS - List of MAC addresses allocated to TG container.
255 # - TG_DRIVERS - List of interface drivers to TG container.
259 # Following code is specifing VFs ID based on nodeness and flavor.
260 # As there is great variability in hardware configuration outside LF,
261 # from bootstrap architecure point of view these are considered as flavors.
262 # Anyone can override flavor for its own machine and add condition here.
263 # See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
264 case_text="${1}_${2}"
265 case "${case_text}" in
267 # Add Intel Corporation XL710/X710 Virtual Function to the
274 # Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
277 tg_netdev=(enpTGa enpTGb)
278 dut1_netdev=(enpSUTa enpSUTb)
281 die "Unknown specification: ${case_text}!"
284 net_path="/sys/bus/pci/devices/*/net/*"
286 # TG side of connections.
291 # DUT1 side of connections.
297 # Following code is filtering available VFs represented by network device
298 # name. Only allowed VFs PCI IDs are used.
300 $(find ${net_path} -type d -name . -o -prune -exec basename '{}' ';');
302 if grep -q "${pci_id}" "/sys/class/net/${netdev}/device/device"; then
303 # We will filter to TG/DUT1 side of connection (this can be in
304 # future overriden by more advanced conditions for mapping).
305 for sub in ${tg_netdev[@]}; do
306 if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
310 for sub in ${dut1_netdev[@]}; do
311 if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
312 dut1_side+=(${netdev})
318 case "${case_text}" in
320 # Pick up first two DUT1 interfaces binded to i40evf.
321 for netdev in "${dut1_side[@]::2}"; do
322 DUT1_NETDEVS+=(${netdev})
324 # Corresponding TG interfaces will be same ID.SUB_ID, but on
325 # opposite linked device.
326 for netdev in "${DUT1_NETDEVS[@]}"; do
327 TG_NETDEVS+=(${netdev/$dut1_netdev/$tg_netdev})
331 for netdev in "${tg_side[@]::2}"; do
332 TG_NETDEVS+=(${netdev})
334 for netdev in "${dut1_side[@]::2}"; do
335 DUT1_NETDEVS+=(${netdev})
339 for NETDEV in "${TG_NETDEVS[@]}"; do
343 TG_PCIDEVS+=(${PCI_ADDR})
344 TG_NETMACS+=(${MAC_ADDR})
345 TG_DRIVERS+=(${KRN_DRIVER})
347 for NETDEV in "${DUT1_NETDEVS[@]}"; do
351 DUT1_PCIDEVS+=(${PCI_ADDR})
352 DUT1_NETMACS+=(${MAC_ADDR})
353 DUT1_DRIVERS+=(${KRN_DRIVER})
356 # We need at least two interfaces for TG/DUT1 for building topology.
357 if [ "${#TG_NETDEVS[@]}" -ne 2 ] || [ "${#DUT1_NETDEVS[@]}" -ne 2 ]; then
358 die "Not enough linux network interfaces found!"
360 if [ "${#TG_PCIDEVS[@]}" -ne 2 ] || [ "${#DUT1_PCIDEVS[@]}" -ne 2 ]; then
361 die "Not enough pci interfaces found!"
366 function get_krn_driver () {
368 # Get kernel driver from linux network device name.
371 # - PCI_ADDR - PCI address of network device.
373 # - KRN_DRIVER - Kernel driver of network device.
377 pci_path="/sys/bus/pci/devices/${PCI_ADDR}"
378 KRN_DRIVER="$(basename $(readlink -f ${pci_path}/driver))" || {
379 die "Failed to get kernel driver of PCI interface!"
384 function get_mac_addr () {
386 # Get MAC address from linux network device name.
389 # - NETDEV - Linux network device name.
391 # - MAC_ADDR - MAC address of network device.
395 if [ -d /sys/class/net/${NETDEV}/device ]; then
396 MAC_ADDR="$(</sys/class/net/${NETDEV}/address)" || {
397 die "Failed to get MAC address of linux network interface!"
403 function get_pci_addr () {
405 # Get PCI address in <domain>:<bus:<device>.<func> format from linux network
409 # - NETDEV - Linux network device name.
411 # - PCI_ADDR - PCI address of network device.
415 if [ -d /sys/class/net/${NETDEV}/device ]; then
416 PCI_ADDR=$(basename $(readlink /sys/class/net/${NETDEV}/device)) || {
417 die "Failed to get PCI address of linux network interface!"
420 if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then
421 die "PCI device ${NETDEV} doesn't exist!"
426 function installed () {
428 # Check if the given utility is installed. Fail if not installed.
430 # Duplicate of common.sh function, as this file is also used standalone.
433 # - ${1} - Utility to check.
435 # - 0 - If command is installed.
436 # - 1 - If command is not installed.
444 function print_env_variables () {
446 # Get environment variables prefixed by CSIT_.
450 env | grep CSIT_ || true
454 function read_env_variables () {
456 # Read environment variables from parameters.
459 # - ${@} - Variables passed as an argument.
460 # Variables read, set or exported: Multiple,
461 # see the code for the current list.
462 # TODO: Do we need to list them and their meanings?
466 for param in "$@"; do
469 declare -gA DCR_UUIDS
470 DCR_UUIDS+=([tg]="${CSIT_TG_UUID}")
471 DCR_UUIDS+=([dut1]="${CSIT_DUT1_UUID}")
472 TG_PCIDEVS=("${CSIT_TG_INTERFACES_PORT1_PCI}")
473 TG_DRIVERS=("${CSIT_TG_INTERFACES_PORT1_DRV}")
474 TG_PCIDEVS+=("${CSIT_TG_INTERFACES_PORT2_PCI}")
475 TG_DRIVERS+=("${CSIT_TG_INTERFACES_PORT2_DRV}")
476 DUT1_PCIDEVS=("${CSIT_DUT1_INTERFACES_PORT1_PCI}")
477 DUT1_DRIVERS=("${CSIT_DUT1_INTERFACES_PORT1_DRV}")
478 DUT1_PCIDEVS+=("${CSIT_DUT1_INTERFACES_PORT2_PCI}")
479 DUT1_DRIVERS+=("${CSIT_DUT1_INTERFACES_PORT2_DRV}")
483 function set_env_variables () {
485 # Set environment variables.
488 # - DCR_UUIDS - Docker Container UUIDs.
489 # - DCR_PORTS - Docker Container's SSH ports.
490 # - DUT1_NETMACS - List of network devices MAC addresses of DUT1 container.
491 # - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container.
492 # - DUT1_DRIVERS - List of interface drivers to DUT1 container.
493 # - TG_NETMACS - List of network devices MAC addresses of TG container.
494 # - TG_PCIDEVS - List of PCI addresses of devices of TG container.
495 # - TG_DRIVERS - List of interface drivers to TG container.
496 # Variables set: TODO.
501 CSIT_TG_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
502 die "Reading hostname IP address failed!"
504 CSIT_TG_PORT="${DCR_PORTS[tg]#*:}"
505 CSIT_TG_UUID="${DCR_UUIDS[tg]}"
506 CSIT_TG_ARCH="$(uname -i)" || {
507 die "Reading machine architecture failed!"
509 CSIT_DUT1_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
510 die "Reading hostname IP address failed!"
512 CSIT_DUT1_PORT="${DCR_PORTS[dut1]#*:}"
513 CSIT_DUT1_UUID="${DCR_UUIDS[dut1]}"
514 CSIT_DUT1_ARCH="$(uname -i)" || {
515 die "Reading machine architecture failed!"
517 CSIT_TG_INTERFACES_PORT1_MAC="${TG_NETMACS[0]}"
518 CSIT_TG_INTERFACES_PORT1_PCI="${TG_PCIDEVS[0]}"
519 CSIT_TG_INTERFACES_PORT1_DRV="${TG_DRIVERS[0]}"
520 CSIT_TG_INTERFACES_PORT2_MAC="${TG_NETMACS[1]}"
521 CSIT_TG_INTERFACES_PORT2_PCI="${TG_PCIDEVS[1]}"
522 CSIT_TG_INTERFACES_PORT2_DRV="${TG_DRIVERS[1]}"
523 CSIT_DUT1_INTERFACES_PORT1_MAC="${DUT1_NETMACS[0]}"
524 CSIT_DUT1_INTERFACES_PORT1_PCI="${DUT1_PCIDEVS[0]}"
525 CSIT_DUT1_INTERFACES_PORT1_DRV="${DUT1_DRIVERS[0]}"
526 CSIT_DUT1_INTERFACES_PORT2_MAC="${DUT1_NETMACS[1]}"
527 CSIT_DUT1_INTERFACES_PORT2_PCI="${DUT1_PCIDEVS[1]}"
528 CSIT_DUT1_INTERFACES_PORT2_DRV="${DUT1_DRIVERS[1]}"
533 function start_topology_containers () {
535 # Starts csit-sut-dcr docker containers for TG/DUT1.
538 # - CSIT_DIR - Path to existing root of local CSIT git repository.
540 # - DCR_UUIDS - Docker Container UUIDs.
541 # - DCR_PORTS - Docker Container SSH TCP ports.
542 # - DCR_CPIDS - Docker Container PIDs (namespaces).
546 if ! installed docker; then
547 die "Docker not present. Please install before continue!"
550 # If the IMAGE is not already loaded then docker run will pull the IMAGE,
551 # and all image dependencies, before it starts the container.
553 # Run the container in the background and print the new container ID.
554 dcr_stc_params="--detach=true "
555 # Give extended privileges to this container. A "privileged" container is
556 # given access to all devices and able to run nested containers.
557 dcr_stc_params+="--privileged "
558 # Publish all exposed ports to random ports on the host interfaces.
559 dcr_stc_params+="--publish-all "
560 # Automatically remove the container when it exits.
561 dcr_stc_params+="--rm "
563 dcr_stc_params+="--shm-size 512M "
564 # Override access to PCI bus by attaching a filesystem mount to the
566 dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices "
567 # Mount vfio to be able to bind to see binded interfaces. We cannot use
568 # --device=/dev/vfio as this does not see newly binded interfaces.
569 dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
570 # Mount nested_vm image to be able to run VM tests.
571 dcr_stc_params+="--volume /var/lib/vm/vhost-nested.img:/var/lib/vm/vhost-nested.img "
572 # Mount docker.sock to be able to use docker deamon of the host.
573 dcr_stc_params+="--volume /var/run/docker.sock:/var/run/docker.sock "
575 # Docker Container UUIDs.
576 declare -gA DCR_UUIDS
577 # Docker Container SSH TCP ports.
578 declare -gA DCR_PORTS
579 # Docker Container PIDs (namespaces).
580 declare -gA DCR_CPIDS
582 # Run TG and DUT1. As initial version we do support only 2-node.
583 params=(${dcr_stc_params} --name csit-tg-$(uuidgen) ${dcr_image})
584 DCR_UUIDS+=([tg]=$(docker run "${params[@]}")) || {
585 die "Failed to start TG docker container!"
587 params=(${dcr_stc_params} --name csit-dut1-$(uuidgen) ${dcr_image})
588 DCR_UUIDS+=([dut1]=$(docker run "${params[@]}")) || {
589 die "Failed to start DUT1 docker container!"
592 trap 'clean_environment_on_exit' EXIT || {
593 die "Trap attempt failed, please cleanup manually. Aborting!"
596 # Get Containers TCP ports.
597 params=(${DCR_UUIDS[tg]})
598 DCR_PORTS+=([tg]=$(docker port "${params[@]}")) || {
599 die "Failed to get port of TG docker container!"
601 params=(${DCR_UUIDS[dut1]})
602 DCR_PORTS+=([dut1]=$(docker port "${params[@]}")) || {
603 die "Failed to get port of DUT1 docker container!"
606 # Get Containers PIDs.
607 params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[tg]})
608 DCR_CPIDS+=([tg]=$(docker inspect "${params[@]}")) || {
609 die "Failed to get PID of TG docker container!"
611 params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[dut1]})
612 DCR_CPIDS+=([dut1]=$(docker inspect "${params[@]}")) || {
613 die "Failed to get PID of DUT1 docker container!"
618 # Print the message to standard error.
620 # Duplicate of common.sh function, as this file is also used standalone.
623 # - ${@} - The text of the message.