X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fbash%2Ffunction%2Fdevice.sh;h=1819c457bba5a5633ee835e828cc9f9d5efb187b;hp=228a73b78cecb015e4d5e54ac7cc6b57f43972e2;hb=HEAD;hpb=55629cb3e0406fd7b054ab620202167b2d954140

diff --git a/resources/libraries/bash/function/device.sh b/resources/libraries/bash/function/device.sh
index 228a73b78c..4d39cd2de6 100644
--- a/resources/libraries/bash/function/device.sh
+++ b/resources/libraries/bash/function/device.sh
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -30,6 +30,7 @@ function activate_wrapper () {
 
     enter_mutex || die
     get_available_interfaces "${1}" "${2}" || die
+    bind_dut_interfaces_to_vpp_driver || die
     start_topology_containers "${3}" || die
     bind_interfaces_to_containers || die
     set_env_variables || die
@@ -38,6 +39,29 @@ function activate_wrapper () {
 }
 
 
+function bind_dut_interfaces_to_vpp_driver () {
+
+    # Bind DUT network interfaces to the driver that vpp will use
+    #
+    # Variables read:
+    # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
+    # Variables set:
+    # - NETDEV - Linux network interface.
+    # - DRIVER - Kernel driver to bind the interface to.
+    # - KRN_DRIVER - The original kernel driver of the network interface.
+
+    for NETDEV in "${DUT1_NETDEVS[@]}"; do
+        get_pci_addr || die
+        get_krn_driver || die
+        if [[ ${KRN_DRIVER} == "iavf" ]]; then
+            DRIVER="vfio-pci"
+            ADDR=${PCI_ADDR}
+            bind_interfaces_to_driver || die
+        fi
+    done
+}
+
+
 function bind_interfaces_to_containers () {
 
     # Bind linux network interface to container and create symlink for PCI
@@ -51,36 +75,42 @@ function bind_interfaces_to_containers () {
     # - TG_NETDEVS - List of network devices allocated to TG container.
     # Variables set:
     # - NETDEV - Linux network interface.
+    # - KRN_DRIVER - Kernel driver of network device.
 
     set -exuo pipefail
 
-    for NETDEV in "${TG_NETDEVS[@]}"; do
-        get_pci_addr || die
+    for PCI_ADDR in "${TG_PCIDEVS[@]}"; do
+        get_netdev_name || die
         link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
             die "Reading symlink for PCI address failed!"
         }
         cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
 
-        sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
-            die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
-        }
         docker exec "${DCR_UUIDS[tg]}" ${cmd} || {
             die "Linking PCI address in container failed!"
         }
+
+        sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
+            die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
+        }
     done
-    for NETDEV in "${DUT1_NETDEVS[@]}"; do
-        get_pci_addr || die
+    for PCI_ADDR in "${DUT1_PCIDEVS[@]}"; do
         link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
             die "Reading symlink for PCI address failed!"
         }
         cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
 
-        sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
-            die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
-        }
         docker exec "${DCR_UUIDS[dut1]}" ${cmd} || {
             die "Linking PCI address in container failed!"
         }
+
+        get_krn_driver
+        if [[ ${KRN_DRIVER} != "vfio-pci" ]]; then
+            get_netdev_name || die
+            sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
+                die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
+            }
+        fi
     done
 }
 
@@ -99,13 +129,22 @@ function bind_interfaces_to_driver () {
     pci_path="/sys/bus/pci/devices/${ADDR}"
     drv_path="/sys/bus/pci/drivers/${DRIVER}"
     if [ -d "${pci_path}/driver" ]; then
-        echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || {
+        echo ${ADDR} | sudo tee ${pci_path}/driver/unbind > /dev/null || {
             die "Failed to unbind interface ${ADDR}!"
         }
     fi
-    echo ${ADDR} | sudo tee ${drv_path}/bind || {
+
+    echo ${DRIVER} | sudo tee /sys/bus/pci/devices/${ADDR}/driver_override \
+        > /dev/null || {
+        die "Failed to override driver to ${DRIVER} for ${ADDR}!"
+    }
+
+    echo ${ADDR} | sudo tee ${drv_path}/bind > /dev/null || {
         die "Failed to bind interface ${ADDR}!"
     }
+
+    echo | sudo tee /sys/bus/pci/devices/${ADDR}/driver_override > /dev/null \
+        || die "Failed to reset driver override for ${ADDR}!"
 }
 
 
@@ -134,13 +173,17 @@ function clean_environment () {
     }
 
     # Rebind interfaces back to kernel drivers.
+    i=0
     for ADDR in ${TG_PCIDEVS[@]}; do
-        DRIVER="${TG_DRIVERS[0]}"
+        DRIVER="${TG_DRIVERS[${i}]}"
         bind_interfaces_to_driver || die
+        ((i++))
     done
+    i=0
     for ADDR in ${DUT1_PCIDEVS[@]}; do
-        DRIVER="${DUT1_DRIVERS[0]}"
+        DRIVER="${DUT1_DRIVERS[${i}]}"
         bind_interfaces_to_driver || die
+        ((i++))
     done
 }
 
@@ -273,14 +316,24 @@ function get_available_interfaces () {
             dut1_netdev=(ens5 enp175)
             ports_per_nic=2
             ;;
-       "1n_tx2")
+       "1n_alt")
             # Add Intel Corporation XL710/X710 Virtual Function to the
             # whitelist.
-            pci_id="0x154c"
-            tg_netdev=(enp5)
-            dut1_netdev=(enp145)
+            # Add MT2892 Family [ConnectX-6 Dx] Virtual Function to the
+            # whitelist.
+            pci_id="0x154c\|0x101e"
+            tg_netdev=(enp1s0f0 enp1s0f1 enP1p1s0f0)
+            dut1_netdev=(enP3p2s0f0 enP3p2s0f1 enP1p1s0f1)
             ports_per_nic=2
             ;;
+        "1n_spr")
+            # Add Intel Corporation E810 Virtual Function to the
+            # whitelist.
+            pci_id="0x1889"
+            tg_netdev=(enp42s0 enp44s0)
+            dut1_netdev=(enp63s0 enp61s0)
+            ports_per_nic=1
+            ;;
        "1n_vbox")
             # Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
             # whitelist.
@@ -415,6 +468,25 @@ function get_mac_addr () {
 }
 
 
+function get_netdev_name () {
+
+    # Get Linux network device name.
+    #
+    # Variables read:
+    # - PCI_ADDR - PCI address of the device.
+    # Variables set:
+    # - NETDEV - Linux network device name.
+
+    set -exuo pipefail
+
+    if [ -d /sys/bus/pci/devices/${PCI_ADDR}/net ]; then
+        NETDEV="$(basename /sys/bus/pci/devices/${PCI_ADDR}/net/*)" || {
+            die "Failed to get Linux interface name of ${PCI_ADDR}"
+        }
+    fi
+}
+
+
 function get_csit_model () {
 
     # Get CSIT model name from linux network device name.
@@ -437,7 +509,10 @@ function get_csit_model () {
             "0x1572"|"0x154c")
                 MODEL="Intel-X710"
                 ;;
-            "*")
+            "0x101e")
+                MODEL="Mellanox-CX6DX"
+                ;;
+            *)
                 MODEL="virtual"
         esac
     fi
@@ -460,13 +535,33 @@ function get_pci_addr () {
         PCI_ADDR=$(basename $(readlink /sys/class/net/${NETDEV}/device)) || {
             die "Failed to get PCI address of linux network interface!"
         }
-    fi
-    if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then
-        die "PCI device ${NETDEV} doesn't exist!"
+        if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then
+            die "PCI device ${PCI_ADDR} doesn't exist!"
+        fi
+    else
+        die "Can't get device info of interface ${NETDEV}!"
     fi
 }
 
 
+function get_vfio_group () {
+
+    # Get the VFIO group of a pci device.
+    #
+    # Variables read:
+    # - PCI_ADDR - PCI address of a device.
+    # Variables set:
+    # - VFIO_GROUP - The VFIO group of the PCI device.
+
+    if [[ -d /sys/bus/pci/devices/${PCI_ADDR}/iommu_group ]]; then
+        VFIO_GROUP="$(basename\
+            $(readlink /sys/bus/pci/devices/${PCI_ADDR}/iommu_group)\
+        )" || {
+            die "PCI device ${PCI_ADDR} does not have an iommu group!"
+        }
+    fi
+}
+
 function get_vlan_filter () {
 
     # Get VLAN stripping filter from PF searched by mac adress.
@@ -619,7 +714,7 @@ function set_env_variables () {
     CSIT_TG_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
         die "Reading hostname IP address failed!"
     }
-    CSIT_TG_PORT="${DCR_PORTS[tg]#*:}"
+    CSIT_TG_PORT="${DCR_PORTS[tg]##*:}"
     CSIT_TG_UUID="${DCR_UUIDS[tg]}"
     CSIT_TG_ARCH="$(uname -i)" || {
         die "Reading machine architecture failed!"
@@ -627,7 +722,7 @@ function set_env_variables () {
     CSIT_DUT1_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
         die "Reading hostname IP address failed!"
     }
-    CSIT_DUT1_PORT="${DCR_PORTS[dut1]#*:}"
+    CSIT_DUT1_PORT="${DCR_PORTS[dut1]##*:}"
     CSIT_DUT1_UUID="${DCR_UUIDS[dut1]}"
     CSIT_DUT1_ARCH="$(uname -i)" || {
         die "Reading machine architecture failed!"
@@ -683,9 +778,19 @@ function start_topology_containers () {
     # Override access to PCI bus by attaching a filesystem mount to the
     # container.
     dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices "
-    # Mount vfio to be able to bind to see bound interfaces. We cannot use
-    # --device=/dev/vfio as this does not see newly bound interfaces.
-    dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
+    # Mount vfio devices to be able to use VFs inside the container.
+    vfio_bound="false"
+    for PCI_ADDR in ${DUT1_PCIDEVS[@]}; do
+        get_krn_driver
+        if [[ ${KRN_DRIVER} == "vfio-pci" ]]; then
+            get_vfio_group
+            dcr_stc_params+="--device /dev/vfio/${VFIO_GROUP} "
+            vfio_bound="true"
+        fi
+    done
+    if ! ${vfio_bound}; then
+        dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
+    fi
     # Disable manipulation with hugepages by VPP.
     dcr_stc_params+="--volume /dev/null:/etc/sysctl.d/80-vpp.conf "
     # Mount docker.sock to be able to use docker deamon of the host.