-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
enter_mutex || die
get_available_interfaces "${1}" "${2}" || die
+ bind_dut_interfaces_to_vpp_driver || die
start_topology_containers "${3}" || die
bind_interfaces_to_containers || die
set_env_variables || die
}
+function bind_dut_interfaces_to_vpp_driver () {
+
+ # Bind DUT network interfaces to the driver that vpp will use
+ #
+ # Variables read:
+ # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
+ # Variables set:
+ # - NETDEV - Linux network interface.
+ # - DRIVER - Kernel driver to bind the interface to.
+ # - KRN_DRIVER - The original kernel driver of the network interface.
+
+ for NETDEV in "${DUT1_NETDEVS[@]}"; do
+ get_pci_addr || die
+ get_krn_driver || die
+ if [[ ${KRN_DRIVER} == "iavf" ]]; then
+ DRIVER="vfio-pci"
+ ADDR=${PCI_ADDR}
+ bind_interfaces_to_driver || die
+ fi
+ done
+}
+
+
function bind_interfaces_to_containers () {
# Bind linux network interface to container and create symlink for PCI
# - TG_NETDEVS - List of network devices allocated to TG container.
# Variables set:
# - NETDEV - Linux network interface.
+ # - KRN_DRIVER - Kernel driver of network device.
set -exuo pipefail
- for NETDEV in "${TG_NETDEVS[@]}"; do
- get_pci_addr || die
+ for PCI_ADDR in "${TG_PCIDEVS[@]}"; do
+ get_netdev_name || die
link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
die "Reading symlink for PCI address failed!"
}
cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
- sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
- die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
- }
docker exec "${DCR_UUIDS[tg]}" ${cmd} || {
die "Linking PCI address in container failed!"
}
+
+ sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
+ die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
+ }
done
- for NETDEV in "${DUT1_NETDEVS[@]}"; do
- get_pci_addr || die
+ for PCI_ADDR in "${DUT1_PCIDEVS[@]}"; do
link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
die "Reading symlink for PCI address failed!"
}
cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
- sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
- die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
- }
docker exec "${DCR_UUIDS[dut1]}" ${cmd} || {
die "Linking PCI address in container failed!"
}
+
+ get_krn_driver
+ if [[ ${KRN_DRIVER} != "vfio-pci" ]]; then
+ get_netdev_name || die
+ sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
+ die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
+ }
+ fi
done
}
pci_path="/sys/bus/pci/devices/${ADDR}"
drv_path="/sys/bus/pci/drivers/${DRIVER}"
if [ -d "${pci_path}/driver" ]; then
- echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || {
+ echo ${ADDR} | sudo tee ${pci_path}/driver/unbind > /dev/null || {
die "Failed to unbind interface ${ADDR}!"
}
fi
- echo ${ADDR} | sudo tee ${drv_path}/bind || {
+
+ echo ${DRIVER} | sudo tee /sys/bus/pci/devices/${ADDR}/driver_override \
+ > /dev/null || {
+ die "Failed to override driver to ${DRIVER} for ${ADDR}!"
+ }
+
+ echo ${ADDR} | sudo tee ${drv_path}/bind > /dev/null || {
die "Failed to bind interface ${ADDR}!"
}
+
+ echo | sudo tee /sys/bus/pci/devices/${ADDR}/driver_override > /dev/null \
+ || die "Failed to reset driver override for ${ADDR}!"
}
}
# Rebind interfaces back to kernel drivers.
+ i=0
for ADDR in ${TG_PCIDEVS[@]}; do
- DRIVER="${TG_DRIVERS[0]}"
+ DRIVER="${TG_DRIVERS[${i}]}"
bind_interfaces_to_driver || die
+ ((i++))
done
+ i=0
for ADDR in ${DUT1_PCIDEVS[@]}; do
- DRIVER="${DUT1_DRIVERS[0]}"
+ DRIVER="${DUT1_DRIVERS[${i}]}"
bind_interfaces_to_driver || die
+ ((i++))
done
}
dut1_netdev=(ens5 enp175)
ports_per_nic=2
;;
- "1n_tx2")
+ "1n_alt")
# Add Intel Corporation XL710/X710 Virtual Function to the
# whitelist.
- pci_id="0x154c"
- tg_netdev=(enp5)
- dut1_netdev=(enp145)
+ # Add MT2892 Family [ConnectX-6 Dx] Virtual Function to the
+ # whitelist.
+ pci_id="0x154c\|0x101e"
+ tg_netdev=(enp1s0f0 enp1s0f1 enP1p1s0f0)
+ dut1_netdev=(enP3p2s0f0 enP3p2s0f1 enP1p1s0f1)
ports_per_nic=2
;;
+ "1n_spr")
+ # Add Intel Corporation E810 Virtual Function to the
+ # whitelist.
+ pci_id="0x1889"
+ tg_netdev=(enp42s0 enp44s0)
+ dut1_netdev=(enp63s0 enp61s0)
+ ports_per_nic=1
+ ;;
"1n_vbox")
# Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
# whitelist.
}
+function get_netdev_name () {
+
+ # Get Linux network device name.
+ #
+ # Variables read:
+ # - PCI_ADDR - PCI address of the device.
+ # Variables set:
+ # - NETDEV - Linux network device name.
+
+ set -exuo pipefail
+
+ if [ -d /sys/bus/pci/devices/${PCI_ADDR}/net ]; then
+ NETDEV="$(basename /sys/bus/pci/devices/${PCI_ADDR}/net/*)" || {
+ die "Failed to get Linux interface name of ${PCI_ADDR}"
+ }
+ fi
+}
+
+
function get_csit_model () {
# Get CSIT model name from linux network device name.
"0x1572"|"0x154c")
MODEL="Intel-X710"
;;
- "*")
+ "0x101e")
+ MODEL="Mellanox-CX6DX"
+ ;;
+ *)
MODEL="virtual"
esac
fi
PCI_ADDR=$(basename $(readlink /sys/class/net/${NETDEV}/device)) || {
die "Failed to get PCI address of linux network interface!"
}
- fi
- if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then
- die "PCI device ${NETDEV} doesn't exist!"
+ if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then
+ die "PCI device ${PCI_ADDR} doesn't exist!"
+ fi
+ else
+ die "Can't get device info of interface ${NETDEV}!"
fi
}
+function get_vfio_group () {
+
+ # Get the VFIO group of a pci device.
+ #
+ # Variables read:
+ # - PCI_ADDR - PCI address of a device.
+ # Variables set:
+ # - VFIO_GROUP - The VFIO group of the PCI device.
+
+ if [[ -d /sys/bus/pci/devices/${PCI_ADDR}/iommu_group ]]; then
+ VFIO_GROUP="$(basename\
+ $(readlink /sys/bus/pci/devices/${PCI_ADDR}/iommu_group)\
+ )" || {
+ die "PCI device ${PCI_ADDR} does not have an iommu group!"
+ }
+ fi
+}
+
function get_vlan_filter () {
# Get VLAN stripping filter from PF searched by mac adress.
CSIT_TG_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
die "Reading hostname IP address failed!"
}
- CSIT_TG_PORT="${DCR_PORTS[tg]#*:}"
+ CSIT_TG_PORT="${DCR_PORTS[tg]##*:}"
CSIT_TG_UUID="${DCR_UUIDS[tg]}"
CSIT_TG_ARCH="$(uname -i)" || {
die "Reading machine architecture failed!"
CSIT_DUT1_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
die "Reading hostname IP address failed!"
}
- CSIT_DUT1_PORT="${DCR_PORTS[dut1]#*:}"
+ CSIT_DUT1_PORT="${DCR_PORTS[dut1]##*:}"
CSIT_DUT1_UUID="${DCR_UUIDS[dut1]}"
CSIT_DUT1_ARCH="$(uname -i)" || {
die "Reading machine architecture failed!"
# Override access to PCI bus by attaching a filesystem mount to the
# container.
dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices "
- # Mount vfio to be able to bind to see bound interfaces. We cannot use
- # --device=/dev/vfio as this does not see newly bound interfaces.
- dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
+ # Mount vfio devices to be able to use VFs inside the container.
+ vfio_bound="false"
+ for PCI_ADDR in ${DUT1_PCIDEVS[@]}; do
+ get_krn_driver
+ if [[ ${KRN_DRIVER} == "vfio-pci" ]]; then
+ get_vfio_group
+ dcr_stc_params+="--device /dev/vfio/${VFIO_GROUP} "
+ vfio_bound="true"
+ fi
+ done
+ if ! ${vfio_bound}; then
+ dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
+ fi
# Disable manipulation with hugepages by VPP.
dcr_stc_params+="--volume /dev/null:/etc/sysctl.d/80-vpp.conf "
# Mount docker.sock to be able to use docker deamon of the host.
Code Review / csit.git / blobdiff