1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
16 # This library defines functions used by multiple entry scripts.
17 # Deliberately not depending on common.sh to allow standalone usage.
18 # Keep functions ordered alphabetically, please.
20 function activate_wrapper () {
22 # Acts as wrapper for activate docker topology.
25 # - ${1} - Node multiplicity of desired testbed.
26 # - ${2} - Node flavor string, usually describing the processor.
27 # - ${3} - CSIT-SUT-DCR image name and version.
32 get_available_interfaces "${1}" "${2}" || die
33 start_topology_containers "${3}" || die
34 bind_interfaces_to_containers || die
35 set_env_variables || die
36 print_env_variables || die
41 function bind_interfaces_to_containers () {
43 # Bind linux network interface to container and create symlink for PCI
44 # address in container.
47 # - DCR_UUIDS - Docker Container UUIDs.
48 # - DCR_CPIDS - Docker Container PIDs (namespaces).
49 # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
50 # - PCI_ADDR - PCI address of network device.
51 # - TG_NETDEVS - List of network devices allocated to TG container.
53 # - NETDEV - Linux network interface.
57 for NETDEV in "${TG_NETDEVS[@]}"; do
59 link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
60 die "Reading symlink for PCI address failed!"
62 cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
64 sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
65 die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
67 docker exec "${DCR_UUIDS[tg]}" ${cmd} || {
68 die "Linking PCI address in container failed!"
71 for NETDEV in "${DUT1_NETDEVS[@]}"; do
73 link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
74 die "Reading symlink for PCI address failed!"
76 cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
78 sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
79 die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
81 docker exec "${DCR_UUIDS[dut1]}" ${cmd} || {
82 die "Linking PCI address in container failed!"
88 function bind_interfaces_to_driver () {
90 # Bind network interface specified by parameter to driver specified by
94 # - ADDR - PCI address of network interface.
95 # - DRIVER - Kernel driver.
99 pci_path="/sys/bus/pci/devices/${ADDR}"
100 drv_path="/sys/bus/pci/drivers/${DRIVER}"
101 if [ -d "${pci_path}/driver" ]; then
102 echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || {
103 die "Failed to unbind interface ${ADDR}!"
106 echo ${ADDR} | sudo tee ${drv_path}/bind || {
107 die "Failed to bind interface ${ADDR}!"
112 function clean_environment () {
114 # Cleanup environment by removing topology containers and shared volumes
115 # and binding interfaces back to original driver.
118 # - DCR_UUIDS - Docker Container UUIDs.
119 # - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container.
120 # - TG_PCIDEVS - List of PCI addresses of devices of TG container.
122 # - ADDR - PCI address of network interface.
123 # - DRIVER - Kernel driver.
127 # Kill docker containers.
128 docker rm --force "${DCR_UUIDS[@]}" || die "Cleanup containers failed!"
130 # Check if there are some leftover containers and remove all. Command will
131 # not fail in case there are no containers to remove.
132 docker rm --force $(docker ps -q --filter name=${DCR_UUIDS[dut1]}) || {
133 warn "Failed to remove hanged containers or nothing to remove!"
136 # Rebind interfaces back to kernel drivers.
137 for ADDR in ${TG_PCIDEVS[@]}; do
138 DRIVER="${TG_DRIVERS[0]}"
139 bind_interfaces_to_driver || die
141 for ADDR in ${DUT1_PCIDEVS[@]}; do
142 DRIVER="${DUT1_DRIVERS[0]}"
143 bind_interfaces_to_driver || die
148 function clean_environment_on_exit () {
150 # Cleanup environment by removing topology containers and binding
151 # interfaces back to original driver only if exit code is not 0.
152 # This function acts as workaround as 'set -eu' does not trigger ERR trap.
156 if [ $? -ne 0 ]; then
157 clean_environment || die
162 function deactivate_wrapper () {
164 # Acts as wrapper for deactivate docker topology.
167 # - ${@} - CSIT environment variables.
172 read_env_variables "${@}" || die
173 clean_environment || die
180 # Print the message to standard error end exit with error code specified
181 # by the second argument.
183 # Duplicate of common.sh function, as this file is also used standalone.
186 # - The default error message.
188 # - ${1} - The whole error message, be sure to quote. Optional
189 # - ${2} - the code to exit with, default: 1.
193 warn "${1:-Unspecified run-time error occurred!}"
198 function enter_mutex () {
200 # Enter mutual exclusion for protecting execution from starvation and
206 mutex_file="/tmp/mutex_file"
209 exec {lock_fd}>${mutex_file} || {
210 die "Mutex enter failed!"
212 flock --timeout "${mutex_timeout}" "${lock_fd}" || {
213 die "Calling flock() failed!"
215 # ----------------------
216 # Enter mutex succeeded.
217 warn "Mutex enter succeeded for PID $$."
221 function exit_mutex () {
223 # Exit mutual exclusion.
227 # ---------------------
228 # Remove mutex so we are not blocking others anymore.
229 flock -u "${lock_fd}" || {
230 die "Mutex destroy failed!"
232 warn "Mutex leave succeeded for PID $$."
236 function get_available_interfaces () {
238 # Find and get available Virtual functions.
241 # - ${1} - Nodeness, as set by common.sh get_test_code.
242 # - ${2} - Flavor, as set by common.sh get_test_code.
244 # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
245 # - DUT1_PCIDEVS - List of PCI addresses allocated to DUT1 container.
246 # - DUT1_NETMACS - List of MAC addresses allocated to DUT1 container.
247 # - DUT1_DRIVERS - List of interface drivers to DUT1 container.
248 # - DUT1_VLANS - List of interface vlans to TG container.
249 # - DUT1_MODEL - List of interface models to TG container.
250 # - TG_NETDEVS - List of network devices allocated to TG container.
251 # - TG_PCIDEVS - List of PCI addresses allocated to TG container.
252 # - TG_NETMACS - List of MAC addresses allocated to TG container.
253 # - TG_DRIVERS - List of interface drivers to TG container.
254 # - TG_VLANS - List of interface vlans to TG container.
255 # - TG_MODEL - List of interface models to TG container.
259 # Following code is specifying VFs ID based on nodeness and flavor.
260 # As there is great variability in hardware configuration outside LF,
261 # from bootstrap architecture point of view these are considered as flavors.
262 # Anyone can override flavor for its own machine and add condition here.
263 # See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
264 case_text="${1}_${2}"
265 case "${case_text}" in
267 # Add Intel Corporation XL710/X710 Virtual Function to the
269 # Add Intel Corporation E810 Virtual Function to the
271 pci_id="0x154c\|0x1889"
272 tg_netdev=(ens1 enp134)
273 dut1_netdev=(ens5 enp175)
277 # Add Intel Corporation XL710/X710 Virtual Function to the
285 # Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
288 tg_netdev=(enp0s8 enp0s9)
289 dut1_netdev=(enp0s16 enp0s17)
293 die "Unknown specification: ${case_text}!"
296 # TG side of connections.
303 # DUT1 side of connections.
311 # Find the first ${device_count} number of available TG Linux network
312 # VF device names. Only allowed VF PCI IDs are filtered.
313 for netdev in ${tg_netdev[@]}
316 for netdev_path in $(grep -l "${pci_id}" \
317 /sys/class/net/${netdev}*/device/device \
320 if [[ ${ports} -lt ${ports_per_nic} ]]; then
321 tg_netdev_name=$(dirname ${netdev_path})
322 tg_netdev_name=$(dirname ${tg_netdev_name})
323 TG_NETDEVS+=($(basename ${tg_netdev_name}))
329 ports_per_device=$((${ports_per_nic}*${#tg_netdev[@]}))
330 if [[ ${#TG_NETDEVS[@]} -eq ${ports_per_device} ]]; then
336 for netdev in "${TG_NETDEVS[@]}"; do
337 # Find the index of selected tg netdev among tg_netdevs
338 # e.g. enp8s5f7 is a vf of netdev enp8s5 with index 11
339 # and the corresponding dut1 netdev is enp133s13.
340 while [[ "${netdev}" != "${tg_netdev[$i]}"* ]]; do
343 # Rename tg netdev to dut1 netdev
344 # e.g. enp8s5f7 -> enp133s13f7
345 DUT1_NETDEVS+=(${netdev/${tg_netdev[$i]}/${dut1_netdev[$i]}})
346 # Don't need to reset i, all netdevs are sorted.
349 for NETDEV in "${TG_NETDEVS[@]}"; do
355 TG_PCIDEVS+=(${PCI_ADDR})
356 TG_NETMACS+=(${MAC_ADDR})
357 TG_DRIVERS+=(${KRN_DRIVER})
358 TG_VLANS+=(${VLAN_ID})
359 TG_MODELS+=(${MODEL})
361 for NETDEV in "${DUT1_NETDEVS[@]}"; do
367 DUT1_PCIDEVS+=(${PCI_ADDR})
368 DUT1_NETMACS+=(${MAC_ADDR})
369 DUT1_DRIVERS+=(${KRN_DRIVER})
370 DUT1_VLANS+=(${VLAN_ID})
371 DUT1_MODELS+=(${MODEL})
374 # We need at least two interfaces for TG/DUT1 for building topology.
375 if [ "${#TG_NETDEVS[@]}" -lt 2 ] || [ "${#DUT1_NETDEVS[@]}" -lt 2 ]; then
376 die "Not enough linux network interfaces found!"
381 function get_krn_driver () {
383 # Get kernel driver from linux network device name.
386 # - PCI_ADDR - PCI address of network device.
388 # - KRN_DRIVER - Kernel driver of network device.
392 pci_path="/sys/bus/pci/devices/${PCI_ADDR}"
393 KRN_DRIVER="$(basename $(readlink -f ${pci_path}/driver))" || {
394 die "Failed to get kernel driver of PCI interface!"
399 function get_mac_addr () {
401 # Get MAC address from linux network device name.
404 # - NETDEV - Linux network device name.
406 # - MAC_ADDR - MAC address of network device.
410 if [ -d /sys/class/net/${NETDEV}/device ]; then
411 MAC_ADDR="$(</sys/class/net/${NETDEV}/address)" || {
412 die "Failed to get MAC address of linux network interface!"
418 function get_csit_model () {
420 # Get CSIT model name from linux network device name.
423 # - NETDEV - Linux network device name.
425 # - MODEL - CSIT model name of network device.
429 if [ -d /sys/class/net/${NETDEV}/device ]; then
430 ID="$(</sys/class/net/${NETDEV}/device/device)" || {
431 die "Failed to get device id of linux network interface!"
447 function get_pci_addr () {
449 # Get PCI address in <domain>:<bus:<device>.<func> format from linux network
453 # - NETDEV - Linux network device name.
455 # - PCI_ADDR - PCI address of network device.
459 if [ -d /sys/class/net/${NETDEV}/device ]; then
460 PCI_ADDR=$(basename $(readlink /sys/class/net/${NETDEV}/device)) || {
461 die "Failed to get PCI address of linux network interface!"
464 if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then
465 die "PCI device ${NETDEV} doesn't exist!"
470 function get_vlan_filter () {
472 # Get VLAN stripping filter from PF searched by mac adress.
475 # - MAC_ADDR - MAC address of VF.
477 # - VLAN_ID - VLAN ids.
481 # Sed regular expression pattern.
482 exp="s/^.*vlan ([[:digit:]]+).*$/\1/"
483 VLAN_ID=$(ip link | grep vlan | grep ${MAC_ADDR} | sed -re "${exp}") || true
484 VLAN_ID="${VLAN_ID:-0}"
488 function installed () {
490 # Check if the given utility is installed. Fail if not installed.
492 # Duplicate of common.sh function, as this file is also used standalone.
495 # - ${1} - Utility to check.
497 # - 0 - If command is installed.
498 # - 1 - If command is not installed.
506 function parse_env_variables () {
508 # Parse environment variables.
510 # Variables read, set or exported: Multiple,
511 # see the code for the current list.
515 IFS=@ read -a TG_NETMACS <<< "${CSIT_TG_INTERFACES_PORT_MAC}"
516 IFS=@ read -a TG_PCIDEVS <<< "${CSIT_TG_INTERFACES_PORT_PCI}"
517 IFS=@ read -a TG_DRIVERS <<< "${CSIT_TG_INTERFACES_PORT_DRV}"
518 IFS=@ read -a TG_VLANS <<< "${CSIT_TG_INTERFACES_PORT_VLAN}"
519 IFS=@ read -a TG_MODELS <<< "${CSIT_TG_INTERFACES_PORT_MODEL}"
520 IFS=@ read -a DUT1_NETMACS <<< "${CSIT_DUT1_INTERFACES_PORT_MAC}"
521 IFS=@ read -a DUT1_PCIDEVS <<< "${CSIT_DUT1_INTERFACES_PORT_PCI}"
522 IFS=@ read -a DUT1_DRIVERS <<< "${CSIT_DUT1_INTERFACES_PORT_DRV}"
523 IFS=@ read -a DUT1_VLANS <<< "${CSIT_DUT1_INTERFACES_PORT_VLAN}"
524 IFS=@ read -a DUT1_MODELS <<< "${CSIT_DUT1_INTERFACES_PORT_MODEL}"
526 for port in $(seq "${#TG_NETMACS[*]}"); do
527 CSIT_TG_INTERFACES+=$(cat << EOF
529 mac_address: "${TG_NETMACS[$((port-1))]}"
530 pci_address: "${TG_PCIDEVS[$((port-1))]}"
531 link: "link$((port-1))"
532 model: ${TG_MODELS[$((port-1))]}
533 driver: "${TG_DRIVERS[$((port-1))]}"
534 vlan: ${TG_VLANS[$((port-1))]}
537 CSIT_TG_INTERFACES+=$'\n'
539 for port in $(seq "${#DUT1_NETMACS[*]}"); do
540 CSIT_DUT1_INTERFACES+=$(cat << EOF
542 mac_address: "${DUT1_NETMACS[$((port-1))]}"
543 pci_address: "${DUT1_PCIDEVS[$((port-1))]}"
544 link: "link$((port-1))"
545 model: ${DUT1_MODELS[$((port-1))]}
546 driver: "${DUT1_DRIVERS[$((port-1))]}"
547 vlan: ${DUT1_VLANS[$((port-1))]}
550 CSIT_DUT1_INTERFACES+=$'\n'
555 function print_env_variables () {
557 # Get environment variables prefixed by CSIT_.
561 env | grep CSIT_ || true
565 function read_env_variables () {
567 # Read environment variables from parameters.
570 # - ${@} - Variables passed as an argument.
571 # Variables read, set or exported: Multiple,
572 # see the code for the current list.
576 for param in "$@"; do
579 declare -gA DCR_UUIDS
580 DCR_UUIDS+=([tg]="${CSIT_TG_UUID}")
581 DCR_UUIDS+=([dut1]="${CSIT_DUT1_UUID}")
583 IFS=@ read -a TG_NETMACS <<< "${CSIT_TG_INTERFACES_PORT_MAC}"
584 IFS=@ read -a TG_PCIDEVS <<< "${CSIT_TG_INTERFACES_PORT_PCI}"
585 IFS=@ read -a TG_DRIVERS <<< "${CSIT_TG_INTERFACES_PORT_DRV}"
586 IFS=@ read -a TG_VLANS <<< "${CSIT_TG_INTERFACES_PORT_VLAN}"
587 IFS=@ read -a TG_MODELS <<< "${CSIT_TG_INTERFACES_PORT_MODEL}"
588 IFS=@ read -a DUT1_NETMACS <<< "${CSIT_DUT1_INTERFACES_PORT_MAC}"
589 IFS=@ read -a DUT1_PCIDEVS <<< "${CSIT_DUT1_INTERFACES_PORT_PCI}"
590 IFS=@ read -a DUT1_DRIVERS <<< "${CSIT_DUT1_INTERFACES_PORT_DRV}"
591 IFS=@ read -a DUT1_VLANS <<< "${CSIT_DUT1_INTERFACES_PORT_VLAN}"
592 IFS=@ read -a DUT1_MODELS <<< "${CSIT_DUT1_INTERFACES_PORT_MODEL}"
596 function set_env_variables () {
598 # Set environment variables.
601 # - DCR_UUIDS - Docker Container UUIDs.
602 # - DCR_PORTS - Docker Container's SSH ports.
603 # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
604 # - DUT1_PCIDEVS - List of PCI addresses allocated to DUT1 container.
605 # - DUT1_NETMACS - List of MAC addresses allocated to DUT1 container.
606 # - DUT1_DRIVERS - List of interface drivers to DUT1 container.
607 # - DUT1_VLANS - List of interface vlans to TG container.
608 # - DUT1_MODEL - List of interface models to TG container.
609 # - TG_NETDEVS - List of network devices allocated to TG container.
610 # - TG_PCIDEVS - List of PCI addresses allocated to TG container.
611 # - TG_NETMACS - List of MAC addresses allocated to TG container.
612 # - TG_DRIVERS - List of interface drivers to TG container.
613 # - TG_VLANS - List of interface vlans to TG container.
614 # - TG_MODEL - List of interface models to TG container.
619 CSIT_TG_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
620 die "Reading hostname IP address failed!"
622 CSIT_TG_PORT="${DCR_PORTS[tg]#*:}"
623 CSIT_TG_UUID="${DCR_UUIDS[tg]}"
624 CSIT_TG_ARCH="$(uname -i)" || {
625 die "Reading machine architecture failed!"
627 CSIT_DUT1_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
628 die "Reading hostname IP address failed!"
630 CSIT_DUT1_PORT="${DCR_PORTS[dut1]#*:}"
631 CSIT_DUT1_UUID="${DCR_UUIDS[dut1]}"
632 CSIT_DUT1_ARCH="$(uname -i)" || {
633 die "Reading machine architecture failed!"
637 CSIT_TG_INTERFACES_PORT_MAC="${TG_NETMACS[*]}"
638 CSIT_TG_INTERFACES_PORT_PCI="${TG_PCIDEVS[*]}"
639 CSIT_TG_INTERFACES_PORT_DRV="${TG_DRIVERS[*]}"
640 CSIT_TG_INTERFACES_PORT_VLAN="${TG_VLANS[*]}"
641 CSIT_TG_INTERFACES_PORT_MODEL="${TG_MODELS[*]}"
642 CSIT_DUT1_INTERFACES_PORT_MAC="${DUT1_NETMACS[*]}"
643 CSIT_DUT1_INTERFACES_PORT_PCI="${DUT1_PCIDEVS[*]}"
644 CSIT_DUT1_INTERFACES_PORT_DRV="${DUT1_DRIVERS[*]}"
645 CSIT_DUT1_INTERFACES_PORT_VLAN="${DUT1_VLANS[*]}"
646 CSIT_DUT1_INTERFACES_PORT_MODEL="${DUT1_MODELS[*]}"
652 function start_topology_containers () {
654 # Starts csit-sut-dcr docker containers for TG/DUT1.
657 # - CSIT_DIR - Path to existing root of local CSIT git repository.
659 # - DCR_UUIDS - Docker Container UUIDs.
660 # - DCR_PORTS - Docker Container SSH TCP ports.
661 # - DCR_CPIDS - Docker Container PIDs (namespaces).
665 if ! installed docker; then
666 die "Docker not present. Please install before continue!"
669 # If the IMAGE is not already loaded then docker run will pull the IMAGE,
670 # and all image dependencies, before it starts the container.
672 # Run the container in the background and print the new container ID.
673 dcr_stc_params="--detach=true "
674 # Give extended privileges to this container. A "privileged" container is
675 # given access to all devices and able to run nested containers.
676 dcr_stc_params+="--privileged "
677 # Publish all exposed ports to random ports on the host interfaces.
678 dcr_stc_params+="--publish-all "
679 # Automatically remove the container when it exits.
680 dcr_stc_params+="--rm "
682 dcr_stc_params+="--shm-size 2G "
683 # Override access to PCI bus by attaching a filesystem mount to the
685 dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices "
686 # Mount vfio to be able to bind to see bound interfaces. We cannot use
687 # --device=/dev/vfio as this does not see newly bound interfaces.
688 dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
689 # Disable manipulation with hugepages by VPP.
690 dcr_stc_params+="--volume /dev/null:/etc/sysctl.d/80-vpp.conf "
691 # Mount docker.sock to be able to use docker deamon of the host.
692 dcr_stc_params+="--volume /var/run/docker.sock:/var/run/docker.sock "
693 # Mount /opt/boot/ where VM kernel and initrd are located.
694 dcr_stc_params+="--volume /opt/boot/:/opt/boot/ "
695 # Mount host hugepages for VMs.
696 dcr_stc_params+="--volume /dev/hugepages/:/dev/hugepages/ "
698 dcr_stc_params+="--sysctl net.ipv6.conf.all.disable_ipv6=1 "
699 dcr_stc_params+="--sysctl net.ipv6.conf.default.disable_ipv6=1 "
701 # Docker Container UUIDs.
702 declare -gA DCR_UUIDS
703 # Docker Container SSH TCP ports.
704 declare -gA DCR_PORTS
705 # Docker Container PIDs (namespaces).
706 declare -gA DCR_CPIDS
708 # Run TG and DUT1. As initial version we do support only 2-node.
709 params=(${dcr_stc_params} --name csit-tg-$(uuidgen) ${dcr_image})
710 DCR_UUIDS+=([tg]=$(docker run "${params[@]}")) || {
711 die "Failed to start TG docker container!"
713 params=(${dcr_stc_params} --name csit-dut1-$(uuidgen) ${dcr_image})
714 DCR_UUIDS+=([dut1]=$(docker run "${params[@]}")) || {
715 die "Failed to start DUT1 docker container!"
718 trap 'clean_environment_on_exit' EXIT || {
719 die "Trap attempt failed, please cleanup manually. Aborting!"
722 # Get Containers TCP ports.
723 params=(${DCR_UUIDS[tg]})
724 DCR_PORTS+=([tg]=$(docker port "${params[@]}")) || {
725 die "Failed to get port of TG docker container!"
727 params=(${DCR_UUIDS[dut1]})
728 DCR_PORTS+=([dut1]=$(docker port "${params[@]}")) || {
729 die "Failed to get port of DUT1 docker container!"
732 # Get Containers PIDs.
733 params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[tg]})
734 DCR_CPIDS+=([tg]=$(docker inspect "${params[@]}")) || {
735 die "Failed to get PID of TG docker container!"
737 params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[dut1]})
738 DCR_CPIDS+=([dut1]=$(docker inspect "${params[@]}")) || {
739 die "Failed to get PID of DUT1 docker container!"
744 # Print the message to standard error.
746 # Duplicate of common.sh function, as this file is also used standalone.
749 # - ${@} - The text of the message.