1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 # This library defines functions used by multiple entry scripts.
17 # Keep functions ordered alphabetically, please.
19 function activate_wrapper () {
20 # Acts as wrapper for activate docker topology.
23 # - ${1} - Node multiplicity of desired testbed.
24 # - ${2} - Node flavor string, usually describing the processor.
25 # - ${3} - CSIT-SUT-DCR image name and version.
30 get_available_interfaces "${1}" "${2}" || die
31 start_topology_containers "${3}" || die
32 bind_interfaces_to_containers || die
33 set_env_variables || die
34 print_env_variables || die
39 function bind_interfaces_to_containers () {
40 # Bind linux network interface to container and create symlink for PCI
41 # address in container.
44 # - DCR_UUIDS - Docker Container UUIDs.
45 # - DCR_CPIDS - Docker Container PIDs (namespaces).
46 # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
47 # - PCI_ADDR - PCI address of network device.
48 # - TG_NETDEVS - List of network devices allocated to TG container.
50 # - NETDEV - Linux network interface.
54 for NETDEV in "${TG_NETDEVS[@]}"; do
56 link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
57 die "Reading symlink for PCI address failed!"
59 cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
61 sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
62 die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
64 docker exec "${DCR_UUIDS[tg]}" ${cmd} || {
65 die "Linking PCI address in container failed!"
68 for NETDEV in "${DUT1_NETDEVS[@]}"; do
70 link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
71 die "Reading symlink for PCI address failed!"
73 cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
75 sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
76 die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
78 docker exec "${DCR_UUIDS[dut1]}" ${cmd} || {
79 die "Linking PCI address in container failed!"
85 function bind_interfaces_to_driver () {
86 # Bind network interface specified by parameter to driver specified by
90 # - ADDR - PCI address of network interface.
91 # - DRIVER - Kernel driver.
93 pci_path="/sys/bus/pci/devices/${ADDR}"
94 drv_path="/sys/bus/pci/drivers/${DRIVER}"
95 vd="$(cat ${pci_path}/vendor ${pci_path}/device)" || {
96 die "Failed to retrieve interface details!"
99 echo ${vd} | sudo tee ${drv_path}/new_id
101 echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || {
102 die "Failed to unbind interface ${ADDR}!"
104 echo ${ADDR} | sudo tee ${drv_path}/bind || {
105 die "Failed to bind interface ${ADDR}!"
110 function clean_environment () {
111 # Cleanup environment by removing topology containers and shared volumes
112 # and binding interfaces back to original driver.
115 # - DCR_UUIDS - Docker Container UUIDs.
116 # - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container.
117 # - TG_PCIDEVS - List of PCI addresses of devices of TG container.
119 # - ADDR - PCI address of network interface.
120 # - DRIVER - Kernel driver.
124 # Kill docker containers.
125 docker rm --force "${DCR_UUIDS[@]}" || die "Cleanup containers failed!"
127 # Check if some container is using volume and remove all the hanged
128 # containers before removing volume. Command will not fail in case there
129 # are no containers to remove.
130 docker rm --force $(docker ps -q --filter volume=${DCR_VOLUMES[dut1]}) || {
131 warn "Failed to remove hanged containers or nothing to remove!"
134 # Remove DUT1 volume.
135 docker volume rm --force "${DCR_VOLUMES[dut1]}" || {
136 die "Failed to remove DUT1 volume!"
139 # Rebind interfaces back to kernel drivers.
140 for ADDR in ${TG_PCIDEVS[@]}; do
141 DRIVER="${TG_DRIVERS[0]}"
142 bind_interfaces_to_driver || die
144 for ADDR in ${DUT1_PCIDEVS[@]}; do
145 DRIVER="${DUT1_DRIVERS[0]}"
146 bind_interfaces_to_driver || die
151 function clean_environment_on_exit () {
152 # Cleanup environment by removing topology containers and binding
153 # interfaces back to original driver only if exit code is not 0.
154 # This function acts as workaround as 'set -eu' does not trigger ERR trap.
156 if [ $? -ne 0 ]; then
157 clean_environment || die
162 function deactivate_wrapper () {
163 # Acts as wrapper for deactivate docker topology.
166 # - ${@} - CSIT environment variables.
171 read_env_variables "${@}" || die
172 clean_environment || die
178 # Print the message to standard error end exit with error code specified
179 # by the second argument.
182 # - The default error message.
184 # - ${1} - The whole error message, be sure to quote. Optional
185 # - ${2} - the code to exit with, default: 1.
189 warn "${1:-Unspecified run-time error occurred!}"
194 function enter_mutex () {
195 # Enter mutual exclusion for protecting execution from starvation and
201 mutex_file="/tmp/mutex_file"
204 exec {lock_fd}>${mutex_file} || {
205 die "Mutex enter failed!"
207 flock --timeout "${mutex_timeout}" "${lock_fd}" || {
208 die "Calling flock() failed!"
210 # ----------------------
211 # Enter mutex succeeded.
212 warn "Mutex enter succeeded for PID $$."
216 function exit_mutex () {
217 # Exit mutual exclusion.
221 # ---------------------
222 # Remove mutex so we are not blocking others anymore.
223 flock -u "${lock_fd}" || {
224 die "Mutex destroy failed!"
226 warn "Mutex leave succeeded for PID $$."
230 function get_available_interfaces () {
231 # Find and get available Virtual functions.
234 # - ${1} - Node flavor string, usually describing the processor and node
235 # multiplicity of desired testbed, separated by underscore.
237 # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
238 # - DUT1_PCIDEVS - List of PCI addresses allocated to DUT1 container.
239 # - DUT1_NETMACS - List of MAC addresses allocated to DUT1 container.
240 # - DUT1_DRIVERS - List of interface drivers to DUT1 container.
241 # - TG_NETDEVS - List of network devices allocated to TG container.
242 # - TG_PCIDEVS - List of PCI addresses allocated to TG container.
243 # - TG_NETMACS - List of MAC addresses allocated to TG container.
244 # - TG_DRIVERS - List of interface drivers to TG container.
248 # Following code is specifing VFs ID based on nodeness and flavor.
249 # As there is great variability in hardware configuration outside LF,
250 # from bootstrap architecure point of view these are considered as flavors.
251 # Anyone can override flavor for its own machine and add condition here.
252 # See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
253 case_text="${1}_${2}"
254 case "${case_text}" in
256 # Add Intel Corporation XL710/X710 Virtual Function to the
263 # Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
266 tg_netdev=(eth1 eth2)
267 dut1_netdev=(eth3 eth4)
270 die "Unknown specification: ${case_text}!"
273 net_path="/sys/bus/pci/devices/*/net/*"
275 # TG side of connections.
280 # DUT1 side of connections.
286 # Following code is filtering available VFs represented by network device
287 # name. Only allowed VFs PCI IDs are used.
289 $(find ${net_path} -type d -name . -o -prune -exec basename '{}' ';');
291 if grep -q "${pci_id}" "/sys/class/net/${netdev}/device/device"; then
292 # We will filter to TG/DUT1 side of connection (this can be in
293 # future overriden by more advanced conditions for mapping).
294 for sub in ${tg_netdev[@]}; do
295 if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
299 for sub in ${dut1_netdev[@]}; do
300 if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
301 dut1_side+=(${netdev})
307 for netdev in "${tg_side[@]::2}"; do
308 TG_NETDEVS+=(${netdev})
310 for netdev in "${dut1_side[@]::2}"; do
311 DUT1_NETDEVS+=(${netdev})
314 for NETDEV in "${TG_NETDEVS[@]}"; do
318 TG_PCIDEVS+=(${PCI_ADDR})
319 TG_NETMACS+=(${MAC_ADDR})
320 TG_DRIVERS+=(${KRN_DRIVER})
322 for NETDEV in "${DUT1_NETDEVS[@]}"; do
326 DUT1_PCIDEVS+=(${PCI_ADDR})
327 DUT1_NETMACS+=(${MAC_ADDR})
328 DUT1_DRIVERS+=(${KRN_DRIVER})
331 # We need at least two interfaces for TG/DUT1 for building topology.
332 if [ "${#TG_NETDEVS[@]}" -ne 2 ] || [ "${#DUT1_NETDEVS[@]}" -ne 2 ]; then
333 die "Not enough linux network interfaces found!"
335 if [ "${#TG_PCIDEVS[@]}" -ne 2 ] || [ "${#DUT1_PCIDEVS[@]}" -ne 2 ]; then
336 die "Not enough pci interfaces found!"
341 function get_krn_driver () {
342 # Get kernel driver from linux network device name.
345 # - PCI_ADDR - PCI address of network device.
347 # - KRN_DRIVER - Kernel driver of network device.
351 pci_path="/sys/bus/pci/devices/${PCI_ADDR}"
352 KRN_DRIVER="$(basename $(readlink -f ${pci_path}/driver))" || {
353 die "Failed to get kernel driver of PCI interface!"
358 function get_mac_addr () {
359 # Get MAC address from linux network device name.
362 # - NETDEV - Linux network device name.
364 # - MAC_ADDR - MAC address of network device.
368 if [ -d /sys/class/net/${NETDEV}/device ]; then
369 MAC_ADDR="$(</sys/class/net/${NETDEV}/address)" || {
370 die "Failed to get MAC address of linux network interface!"
376 function get_pci_addr () {
377 # Get PCI address in <domain>:<bus:<device>.<func> format from linux network
381 # - NETDEV - Linux network device name.
383 # - PCI_ADDR - PCI address of network device.
387 if [ -d /sys/class/net/${NETDEV}/device ]; then
388 PCI_ADDR=$(basename $(readlink /sys/class/net/${NETDEV}/device)) || {
389 die "Failed to get PCI address of linux network interface!"
392 if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then
393 die "PCI device ${NETDEV} doesn't exist!"
398 function installed () {
402 # Check if the given utility is installed. Fail if not installed.
405 # - ${1} - Utility to check.
407 # - 0 - If command is installed.
408 # - 1 - If command is not installed.
414 function print_env_variables () {
415 # Get environment variables prefixed by CSIT_.
423 function read_env_variables () {
424 # Read environment variables from parameters.
427 # - ${@} - Variables passed as an argument.
431 for param in "$@"; do
434 declare -gA DCR_UUIDS
435 declare -gA DCR_VOLUMES
436 DCR_UUIDS+=([tg]="${CSIT_TG_UUID}")
437 DCR_UUIDS+=([dut1]="${CSIT_DUT1_UUID}")
438 DCR_VOLUMES+=([dut1]="${CSIT_DUT1_VOL}")
439 TG_PCIDEVS=("${CSIT_TG_INTERFACES_PORT1_PCI}")
440 TG_DRIVERS=("${CSIT_TG_INTERFACES_PORT1_DRV}")
441 TG_PCIDEVS+=("${CSIT_TG_INTERFACES_PORT2_PCI}")
442 TG_DRIVERS+=("${CSIT_TG_INTERFACES_PORT2_DRV}")
443 DUT1_PCIDEVS=("${CSIT_DUT1_INTERFACES_PORT1_PCI}")
444 DUT1_DRIVERS=("${CSIT_DUT1_INTERFACES_PORT1_DRV}")
445 DUT1_PCIDEVS+=("${CSIT_DUT1_INTERFACES_PORT2_PCI}")
446 DUT1_DRIVERS+=("${CSIT_DUT1_INTERFACES_PORT2_DRV}")
450 function set_env_variables () {
451 # Set environment variables.
454 # - DCR_UUIDS - Docker Container UUIDs.
455 # - DCR_PORTS - Docker Container's SSH ports.
456 # - DUT1_NETMACS - List of network devices MAC addresses of DUT1 container.
457 # - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container.
458 # - DUT1_DRIVERS - List of interface drivers to DUT1 container.
459 # - TG_NETMACS - List of network devices MAC addresses of TG container.
460 # - TG_PCIDEVS - List of PCI addresses of devices of TG container.
461 # - TG_DRIVERS - List of interface drivers to TG container.
466 CSIT_TG_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
467 die "Reading hostname IP address failed!"
469 CSIT_TG_PORT="${DCR_PORTS[tg]#*:}"
470 CSIT_TG_UUID="${DCR_UUIDS[tg]}"
471 CSIT_TG_ARCH="$(uname -i)" || {
472 die "Reading machine architecture failed!"
474 CSIT_DUT1_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
475 die "Reading hostname IP address failed!"
477 CSIT_DUT1_PORT="${DCR_PORTS[dut1]#*:}"
478 CSIT_DUT1_UUID="${DCR_UUIDS[dut1]}"
479 CSIT_DUT1_ARCH="$(uname -i)" || {
480 die "Reading machine architecture failed!"
482 CSIT_DUT1_VOL="${DCR_VOLUMES[dut1]}"
483 CSIT_TG_INTERFACES_PORT1_MAC="${TG_NETMACS[0]}"
484 CSIT_TG_INTERFACES_PORT1_PCI="${TG_PCIDEVS[0]}"
485 CSIT_TG_INTERFACES_PORT1_DRV="${TG_DRIVERS[0]}"
486 CSIT_TG_INTERFACES_PORT2_MAC="${TG_NETMACS[1]}"
487 CSIT_TG_INTERFACES_PORT2_PCI="${TG_PCIDEVS[1]}"
488 CSIT_TG_INTERFACES_PORT2_DRV="${TG_DRIVERS[1]}"
489 CSIT_DUT1_INTERFACES_PORT1_MAC="${DUT1_NETMACS[0]}"
490 CSIT_DUT1_INTERFACES_PORT1_PCI="${DUT1_PCIDEVS[0]}"
491 CSIT_DUT1_INTERFACES_PORT1_DRV="${DUT1_DRIVERS[0]}"
492 CSIT_DUT1_INTERFACES_PORT2_MAC="${DUT1_NETMACS[1]}"
493 CSIT_DUT1_INTERFACES_PORT2_PCI="${DUT1_PCIDEVS[1]}"
494 CSIT_DUT1_INTERFACES_PORT2_DRV="${DUT1_DRIVERS[1]}"
499 function start_topology_containers () {
500 # Starts csit-sut-dcr docker containers for TG/DUT1.
503 # - CSIT_DIR - Path to existing root of local CSIT git repository.
505 # - DCR_UUIDS - Docker Container UUIDs.
506 # - DCR_PORTS - Docker Container SSH TCP ports.
507 # - DCR_CPIDS - Docker Container PIDs (namespaces).
511 if ! installed docker; then
512 die "Docker not present. Please install before continue!"
515 # If the IMAGE is not already loaded then docker run will pull the IMAGE,
516 # and all image dependencies, before it starts the container.
518 # Run the container in the background and print the new container ID.
519 dcr_stc_params="--detach=true "
520 # Give extended privileges to this container. A "privileged" container is
521 # given access to all devices and able to run nested containers.
522 dcr_stc_params+="--privileged "
523 # Publish all exposed ports to random ports on the host interfaces.
524 dcr_stc_params+="--publish-all "
525 # Automatically remove the container when it exits.
526 dcr_stc_params+="--rm "
528 dcr_stc_params+="--shm-size 512M "
529 # Override access to PCI bus by attaching a filesystem mount to the
531 dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices "
532 # Mount vfio to be able to bind to see binded interfaces. We cannot use
533 # --device=/dev/vfio as this does not see newly binded interfaces.
534 dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
535 # Mount nested_vm image to be able to run VM tests.
536 dcr_stc_params+="--volume /var/lib/vm/vhost-nested.img:/var/lib/vm/vhost-nested.img "
537 # Mount docker.sock to be able to use docker deamon of the host.
538 dcr_stc_params+="--volume /var/run/docker.sock:/var/run/docker.sock "
540 # Docker Container UUIDs.
541 declare -gA DCR_UUIDS
542 # Docker Container SSH TCP ports.
543 declare -gA DCR_PORTS
544 # Docker Container PIDs (namespaces).
545 declare -gA DCR_CPIDS
546 # Docker Container volumes with no relationship to the host.
547 declare -gA DCR_VOLUMES
549 # Create DUT1 /tmp volume to be able to install VPP in "nested" container.
550 params=(--name DUT1_VOL_$(uuidgen))
551 DCR_VOLUMES+=([dut1]="$(docker volume create "${params[@]}")") || {
552 die "Failed to create DUT1 /tmp volume!"
555 # Mount DUT1_VOL as /tmp directory on DUT1 container
556 dcr_stc_params_dut1="--volume ${DCR_VOLUMES[dut1]}:/tmp "
558 # Run TG and DUT1. As initial version we do support only 2-node.
559 params=(${dcr_stc_params} --name csit-tg-$(uuidgen) ${dcr_image})
560 DCR_UUIDS+=([tg]="$(docker run "${params[@]}")") || {
561 die "Failed to start TG docker container!"
563 params=(${dcr_stc_params} ${dcr_stc_params_dut1}
564 --name csit-dut1-$(uuidgen) ${dcr_image})
565 DCR_UUIDS+=([dut1]="$(docker run "${params[@]}")") || {
566 die "Failed to start DUT1 docker container!"
569 trap 'clean_environment_on_exit' EXIT || {
570 die "Trap attempt failed, please cleanup manually. Aborting!"
573 # Get Containers TCP ports.
574 params=(${DCR_UUIDS[tg]})
575 DCR_PORTS+=([tg]="$(docker port "${params[@]}")") || {
576 die "Failed to get port of TG docker container!"
578 params=(${DCR_UUIDS[dut1]})
579 DCR_PORTS+=([dut1]="$(docker port "${params[@]}")") || {
580 die "Failed to get port of DUT1 docker container!"
583 # Get Containers PIDs.
584 params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[tg]})
585 DCR_CPIDS+=([tg]="$(docker inspect "${params[@]}")") || {
586 die "Failed to get PID of TG docker container!"
588 params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[dut1]})
589 DCR_CPIDS+=([dut1]="$(docker inspect "${params[@]}")") || {
590 die "Failed to get PID of DUT1 docker container!"
595 # Print the message to standard error.
598 # - ${@} - The text of the message.