table_flag = False
force_flag = False
args = []
+loaded_modules = []
try:
raw_input
return subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=stderr, **kwargs).communicate()[0]
+kernel_ver = check_output(['uname', '-r'], universal_newlines = True).strip()
+
+
def find_module(mod):
'''find the .ko file for kernel module named mod.
Searches the $RTE_SDK/$RTE_TARGET directory, the kernel
# check using depmod
try:
depmod_out = check_output(["modinfo", "-n", mod], \
- stderr=subprocess.STDOUT, universal_newlines = True).lower()
- if "error" not in depmod_out:
+ stderr=subprocess.STDOUT, universal_newlines = True)
+ if "error" not in depmod_out.lower():
path = depmod_out.strip()
if exists(path):
return path
pass
# check for a copy based off current path
- tools_dir = dirname(abspath(sys.argv[0]))
- if (tools_dir.endswith("tools")):
- base_dir = dirname(tools_dir)
- find_out = check_output(["find", base_dir, "-name", mod + ".ko"], universal_newlines = True)
- if len(find_out) > 0: #something matched
- path = find_out.splitlines()[0]
- if exists(path):
- return path
+ drivers_dir = '/lib/modules/%s/kernel/drivers' % kernel_ver
+ find_out = check_output(["find", drivers_dir, "-name", mod + ".ko\*"], universal_newlines = True)
+ if find_out: #something matched
+ path = find_out.splitlines()[0]
+ if exists(path):
+ return path
+
+def get_loaded_modules():
+ if not loaded_modules:
+ with open("/proc/modules") as fd:
+ loaded_mods = fd.readlines()
+ for line in loaded_mods:
+ loaded_modules.append(line.split()[0])
+ return loaded_modules
def check_modules():
'''Checks that igb_uio is loaded'''
global dpdk_drivers
- with open("/proc/modules") as fd:
- loaded_mods = fd.readlines()
-
# list of supported modules
mods = [{"Name" : driver, "Found" : False} for driver in dpdk_drivers]
# first check if module is loaded
- for line in loaded_mods:
+ for line in get_loaded_modules():
for mod in mods:
if line.startswith(mod["Name"]):
mod["Found"] = True
return None
return res[0][7]
+def is_module_used(module):
+ refcnt_file = '/sys/module/%s/refcnt' % module
+ if not os.path.exists(refcnt_file):
+ return False
+ with open(refcnt_file) as f:
+ ref_cnt = int(f.read().strip())
+ if ref_cnt:
+ return True
+ return False
+
def get_pid_using_pci(pci_list):
if not isinstance(pci_list, list):
pci_list = [pci_list]
(dev[b"Slot"], dev[b"Device_str"], dev[b"Interface"]))
return
- if not force and dev.get('Driver_str') in dpdk_drivers and get_igb_uio_usage():
+ # Mellanox NICs do not need unbind
+ if not force and dev['Driver_str'] in dpdk_and_kernel:
+ print('Mellanox NICs do not need unbinding.')
+ if not confirm('Confirm unbind (y/N)'):
+ print('Not unbinding.')
+ sys.exit(-1)
+
+ if not force and dev['Driver_str'] in dpdk_drivers and is_module_used(dev['Driver_str']):
pid = get_pid_using_pci(dev_id)
if pid:
cmdline = read_pid_cmdline(pid)
print('Interface %s is in use by process:\n%s' % (dev_id, cmdline))
if not confirm('Unbinding might hang the process. Confirm unbind (y/N)'):
print('Not unbinding.')
- return
+ sys.exit(-1)
# prevent us disconnecting ourselves
if dev["Active"] and not force:
print('netstat indicates that interface %s is active.' % dev_id)
if not confirm('Confirm unbind (y/N)'):
print('Not unbinding.')
- return
+ sys.exit(-1)
# write to /sys to unbind
filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"]
print("netstat indicates that interface %s is active" % dev_id)
if not confirm("Confirm bind (y/N)"):
print('Not binding.')
- return
+ sys.exit(-1)
# unbind any existing drivers we don't want
if has_driver(dev_id):
if dev["Driver_str"] == driver:
print("%s already bound to driver %s, skipping\n" % (dev_id, driver))
- return
+ return True
else:
saved_driver = dev["Driver_str"]
- if not force and get_igb_uio_usage():
+ if not force and is_module_used(saved_driver):
pid = get_pid_using_pci(dev_id)
if pid:
cmdline = read_pid_cmdline(pid)
print('Interface %s is in use by process:\n%s' % (dev_id, cmdline))
if not confirm('Binding to other driver might hang the process. Confirm unbind (y/N)'):
print('Not binding.')
- return
+ sys.exit(-1)
unbind_one(dev_id, force)
dev["Driver_str"] = "" # clear driver string
try:
f.write(dev_id)
f.close()
+ return True
except:
# for some reason, closing dev_id after adding a new PCI ID to new_id
# results in IOError. however, if the device was successfully bound,
# we don't care for any errors and can safely ignore IOError
tmp = get_pci_device_details(dev_id)
if "Driver_str" in tmp and tmp["Driver_str"] == driver:
- return
+ return True
print("Error: bind failed for %s - Cannot bind to driver %s" % (dev_id, driver))
if saved_driver is not None: # restore any previous driver
bind_one(dev_id, saved_driver, force)
dev_list = list(map(dev_id_from_dev_name, dev_list))
for d in dev_list:
- bind_one(d, driver, force)
+ res = bind_one(d, driver, force)
+ if not res:
+ sys.exit(-1)
# when binding devices to a generic driver (i.e. one that doesn't have a
# PCI ID table), some devices that are not bound to any other driver could
# 32 : no errors - mlx share object should be loaded
MLX_EXIT_CODE = 32
-out = subprocess.check_output(['lsmod'])
-DPDK_DRIVER = 'vfio-pci' if out.find('vfio_pci') != -1 else 'igb_uio'
+class VFIOBindErr(Exception): pass
class ConfigCreator(object):
mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
print('Saved to %s.' % filename)
return config_str
+# only load igb_uio if it's available
+def load_igb_uio():
+ if 'igb_uio' in dpdk_nic_bind.get_loaded_modules():
+ return True
+ km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
+ if os.path.exists(km):
+ return os.system('insmod %s' % km) == 0
+
+# try to compile igb_uio if it's missing
+def compile_and_load_igb_uio():
+ loaded_mods = dpdk_nic_bind.get_loaded_modules()
+ if 'igb_uio' in loaded_mods:
+ return
+ if 'uio' not in loaded_mods:
+ res = os.system('modprobe uio')
+ if res:
+ print('Failed inserting uio module, please check if it is installed')
+ sys.exit(-1)
+ km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
+ if not os.path.exists(km):
+ print("ERROR: We don't have precompiled igb_uio.ko module for your kernel version")
+ print('Will try compiling automatically.')
+ try:
+ subprocess.check_output('make', cwd = './ko/src', stderr = subprocess.STDOUT)
+ subprocess.check_output(['make', 'install'], cwd = './ko/src', stderr = subprocess.STDOUT)
+ print('\nSuccess.')
+ except Exception as e:
+ print('\nAutomatic compilation failed: (%s)' % e)
+ print('You can try compiling yourself, using the following commands:')
+ print(' $cd ko/src')
+ print(' $make')
+ print(' $make install')
+ print(' $cd -')
+ print('Then, try to run TRex again.')
+ print('Note: you might need additional Linux packages for that:')
+ print(' * yum based (Fedora, CentOS, RedHat):')
+ print(' sudo yum install kernel-devel-`uname -r`')
+ print(' sudo yum group install "Development tools"')
+ print(' * apt based (Ubuntu):')
+ print(' sudo apt install linux-headers-`uname -r`')
+ print(' sudo apt install build-essential')
+ sys.exit(-1)
+ res = os.system('insmod %s' % km)
+ if res:
+ print('Failed inserting igb_uio module')
+ sys.exit(-1)
class map_driver(object):
args=None;
raise DpdkSetup('Error: port_limit should not be higher than number of interfaces in config file: %s\n' % fcfg)
- def do_bind_one (self,key,mellanox):
- if mellanox:
- drv="mlx5_core"
- else:
- drv=DPDK_DRIVER
-
- cmd='%s dpdk_nic_bind.py --bind=%s %s ' % (sys.executable, drv,key)
+ def do_bind_all(self, drv, pci, force = False):
+ assert type(pci) is list
+ cmd = '{ptn} dpdk_nic_bind.py --bind={drv} {pci} {frc}'.format(
+ ptn = sys.executable,
+ drv = drv,
+ pci = ' '.join(pci),
+ frc = '--force' if force else '')
print(cmd)
- res=os.system(cmd);
- if res!=0:
- raise DpdkSetup('')
+ return os.system(cmd)
+
+ # pros: no need to compile .ko per Kernel version
+ # cons: need special config/hw (not always works)
+ def try_bind_to_vfio_pci(self, to_bind_list):
+ krnl_params_file = '/proc/cmdline'
+ if not os.path.exists(krnl_params_file):
+ raise VFIOBindErr('Could not find file with Kernel boot parameters: %s' % krnl_params_file)
+ with open(krnl_params_file) as f:
+ krnl_params = f.read()
+ if 'iommu=' in krnl_params:
+ if 'vfio_pci' not in dpdk_nic_bind.get_loaded_modules():
+ ret = os.system('modprobe vfio_pci')
+ if ret:
+ raise VFIOBindErr('Could not load vfio_pci')
+ ret = self.do_bind_all('vfio-pci', to_bind_list)
+ if ret:
+ raise VFIOBindErr('Binding to vfio_pci failed')
+ raise VFIOBindErr('vfio-pci is not an option here')
def pci_name_to_full_name (self,pci_name):
err=" %s does not have Vendor_str " %key;
raise DpdkSetup(err)
- if self.m_devices[key]['Vendor_str'].find("Mellanox")>-1 :
- Mellanox_cnt=Mellanox_cnt+1
+ if 'Mellanox' in self.m_devices[key]['Vendor_str']:
+ Mellanox_cnt += 1
if not map_driver.parent_args.dump_interfaces:
- if ((Mellanox_cnt>0) and (Mellanox_cnt!= len(if_list))):
+ if (Mellanox_cnt > 0) and (Mellanox_cnt != len(if_list)):
err=" All driver should be from one vendor. you have at least one driver from Mellanox but not all ";
raise DpdkSetup(err)
-
-
- if not map_driver.parent_args.dump_interfaces:
- if Mellanox_cnt>0 :
+ if Mellanox_cnt > 0:
self.set_only_mellanox_nics()
if self.get_only_mellanox_nics():
if only_check_all_mlx:
- if Mellanox_cnt >0:
+ if Mellanox_cnt > 0:
exit(MLX_EXIT_CODE);
else:
exit(0);
- for key in if_list:
- if key not in self.m_devices:
- err=" %s does not exist " %key;
- raise DpdkSetup(err)
-
- if 'Driver_str' in self.m_devices[key]:
- if self.m_devices[key]['Driver_str'] not in (dpdk_nic_bind.dpdk_drivers+dpdk_nic_bind.dpdk_and_kernel) :
- self.do_bind_one (key,(Mellanox_cnt>0))
- pass;
- else:
- self.do_bind_one (key,(Mellanox_cnt>0))
- pass;
-
if if_list and map_driver.args.parent and self.m_cfg_dict[0].get('enable_zmq_pub', True):
publisher_port = self.m_cfg_dict[0].get('zmq_pub_port', 4500)
pid = dpdk_nic_bind.get_tcp_port_usage(publisher_port)
print("Could not start scapy_daemon_server, which is needed by GUI to create packets.\nIf you don't need it, use --no-scapy-server flag.")
sys.exit(-1)
- if Mellanox_cnt:
- return MLX_EXIT_CODE
- else:
- return 0
+ to_bind_list = []
+ for key in if_list:
+ if key not in self.m_devices:
+ err=" %s does not exist " %key;
+ raise DpdkSetup(err)
+
+ if self.m_devices[key].get('Driver_str') not in (dpdk_nic_bind.dpdk_drivers + dpdk_nic_bind.dpdk_and_kernel):
+ to_bind_list.append(key)
+
+ if to_bind_list:
+ if Mellanox_cnt > 0:
+ ret = self.do_bind_all('mlx5_core', to_bind_list)
+ if ret:
+ raise DpdkSetup('Unable to bind interfaces to driver mlx5_core.')
+ return MLX_EXIT_CODE
+ else:
+ # if igb_uio is ready, use it as safer choice, afterwards try vfio-pci
+ if load_igb_uio():
+ print('Trying to bind to igb_uio ...')
+ ret = self.do_bind_all('igb_uio', to_bind_list)
+ if ret:
+ raise DpdkSetup('Unable to bind interfaces to driver igb_uio.') # module present, loaded, but unable to bind
+ return
+
+ try:
+ print('Trying to bind to vfio-pci ...')
+ self.try_bind_to_vfio_pci(to_bind_list)
+ return
+ except VFIOBindErr as e:
+ pass
+ #print(e)
+
+ print('Trying to compile and bind to igb_uio ...')
+ compile_and_load_igb_uio()
+ ret = self.do_bind_all('igb_uio', to_bind_list)
+ if ret:
+ raise DpdkSetup('Unable to bind interfaces to driver igb_uio.')
def do_return_to_linux(self):
if not self.m_devices:
self.run_dpdk_lspci()
dpdk_interfaces = []
+ check_drivers = set()
for device in self.m_devices.values():
if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
dpdk_interfaces.append(device['Slot'])
+ check_drivers.add(device['Driver_str'])
if not dpdk_interfaces:
print('No DPDK bound interfaces.')
return
- if dpdk_nic_bind.get_igb_uio_usage():
+ any_driver_used = False
+ for driver in check_drivers:
+ if dpdk_nic_bind.is_module_used(driver):
+ any_driver_used = True
+ if any_driver_used:
pid = dpdk_nic_bind.get_pid_using_pci(dpdk_interfaces)
if pid:
cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
print('DPDK interfaces are in use. Unbinding them might cause following process to hang:\npid: %s, cmd: %s' % (pid, cmdline))
if not dpdk_nic_bind.confirm('Confirm (y/N):'):
- return
+ sys.exit(-1)
+
+ # DPDK => Linux
drivers_table = {
'net_ixgbe': 'ixgbe',
- 'net_ixgbe_vf': 'ixgbe_vf',
- 'net_e1000_igb': 'e1000_igb',
+ 'net_ixgbe_vf': 'ixgbevf',
+ 'net_e1000_igb': 'igb',
'net_i40e': 'i40e',
- 'net_i40e_vf': 'i40e_vf',
- 'net_e1000_em': 'e1000_em',
+ 'net_i40e_vf': 'i40evf',
+ 'net_e1000_em': 'e1000',
'net_vmxnet3': 'vmxnet3',
'net_virtio': 'virtio-pci',
'net_enic': 'enic',
raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
dev = self.m_devices[pci]
if info['TRex_Driver'] not in drivers_table:
- print("Got unknown driver '%s', description: %s" % (info['TRex_Driver'], dev['Device_str']))
+ raise DpdkSetup("Got unknown driver '%s', description: %s" % (info['TRex_Driver'], dev['Device_str']))
+ linux_driver = drivers_table[info['TRex_Driver']]
+ if linux_driver not in dpdk_nic_bind.get_loaded_modules():
+ print("No Linux driver installed, or wrong module name: %s" % linux_driver)
else:
print('Returning to Linux %s' % pci)
- dpdk_nic_bind.bind_one(pci, drivers_table[info['TRex_Driver']], False)
+ dpdk_nic_bind.bind_one(pci, linux_driver, False)
def _get_cpu_topology(self):
cpu_topology_file = '/proc/cpuinfo'
To create TRex config file using interactive mode
sudo ./dpdk_set_ports.py -i
-To create a default config file (example1)
+To create a default config file (example)
sudo ./dpdk_setup_ports.py -c 02:00.0 02:00.1 -o /etc/trex_cfg.yaml
-To create a default config file (example2)
- sudo ./dpdk_setup_ports.py -c eth1 eth2 --dest-macs 11:11:11:11:11:11 22:22:22:22:22:22 --dump
-
To show interfaces status
sudo ./dpdk_set_ports.py -s
Code Review / trex.git / commitdiff