From ee8ba6877efb916b88255821f94ef33437724f75 Mon Sep 17 00:00:00 2001 From: Steven Luong Date: Thu, 14 Mar 2019 09:57:09 -0700 Subject: [PATCH] vmxnet3: auto bind support For creating the vmxnet3 interface, add the bind option to automatically bind the pci to vfio-pci module which removes the need for manual bind. Manual bind still works, should people prefer to go that route. Change-Id: Ife75926f8755d754a08dd0ecff0f1de326ad5ba1 Signed-off-by: Steven Luong --- src/plugins/vmxnet3/README.md | 21 +++++++++++------ src/plugins/vmxnet3/cli.c | 4 +++- src/plugins/vmxnet3/vmxnet3.api | 3 +++ src/plugins/vmxnet3/vmxnet3.c | 46 ++++++++++++++++++++++++++------------ src/plugins/vmxnet3/vmxnet3.h | 1 + src/plugins/vmxnet3/vmxnet3_api.c | 2 ++ src/plugins/vmxnet3/vmxnet3_test.c | 9 +++++++- 7 files changed, 63 insertions(+), 23 deletions(-) diff --git a/src/plugins/vmxnet3/README.md b/src/plugins/vmxnet3/README.md index b5829939506..722954d51a3 100644 --- a/src/plugins/vmxnet3/README.md +++ b/src/plugins/vmxnet3/README.md @@ -7,11 +7,11 @@ This plugin provides native PCI driver support for VMWare vmxnet3. * This code is tested with vfio-pci driver installed with Ubuntu 18.04 which has kernel version 4.15.0-33-generic. - * This code is tested with ESXi vSwitch version 6.0, release build 3620759. + * This driver is tested with ESXi vSwitch version 6.5/6.7 for LRO/TSO support, VMware Workstation 15 Pro (no LRO/TSO), and VMware Fusion 11 Pro (no LRO/TSO) * Driver requires MSI-X interrupt support, which is not supported by -uio_pci_generic driver, so vfio-pci needs to be used. On systems without IOMMU -vfio driver can still be used with recent kernels which support no-iommu mode. +uio_pci_generic driver. So vfio-pci must be used. On systems without IOMMU, +vfio driver can still be used with 4.15.0-33-generic kernel (Ubuntu 18.04) which supports no-iommu mode. ##Known issues @@ -25,20 +25,27 @@ vfio driver can still be used with recent kernels which support no-iommu mode. sudo modprobe vfio-pci ``` -2. (systems without IOMMU only) enable unsafe NOIOMMU mode +2. Make sure the interface is down +``` +sudo ifconfig down +``` + +Steps 3 and 4 are optional. They can be accomplished by specifying the optional keyword "bind" when creating the vmxnet3 interface. + +3. (systems without IOMMU only) enable unsafe NOIOMMU mode ``` echo Y | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode ``` -3. Bind interface to vfio-pci +4. Bind interface to vfio-pci ``` sudo dpdk-devbind.py --bind vfio-pci 0b:00.0 ``` ### Interface Creation -Interface can be dynamically created with following CLI: +Interface can be dynamically created with following CLI, with or without the bind option. If step 3 and 4 were executed, bind can be omitted. ``` -create interface vmxnet3 0000:0b:00.0 +create interface vmxnet3 0000:0b:00.0 bind set int state vmxnet3-0/b/0/0 up ``` diff --git a/src/plugins/vmxnet3/cli.c b/src/plugins/vmxnet3/cli.c index 0f0cf60fa40..776901e8ac2 100644 --- a/src/plugins/vmxnet3/cli.c +++ b/src/plugins/vmxnet3/cli.c @@ -44,6 +44,8 @@ vmxnet3_create_command_fn (vlib_main_t * vm, unformat_input_t * input, ; else if (unformat (line_input, "elog")) args.enable_elog = 1; + else if (unformat (line_input, "bind")) + args.bind = 1; else if (unformat (line_input, "rx-queue-size %u", &args.rxq_size)) ; else if (unformat (line_input, "tx-queue-size %u", &args.txq_size)) @@ -69,7 +71,7 @@ VLIB_CLI_COMMAND (vmxnet3_create_command, static) = { .path = "create interface vmxnet3", .short_help = "create interface vmxnet3 " "[rx-queue-size ] [tx-queue-size ]" - "[num-tx-queues ] [num-rx-queues ]", + "[num-tx-queues ] [num-rx-queues ] [bind]", .function = vmxnet3_create_command_fn, }; /* *INDENT-ON* */ diff --git a/src/plugins/vmxnet3/vmxnet3.api b/src/plugins/vmxnet3/vmxnet3.api index 804ba538be5..7408836af7b 100644 --- a/src/plugins/vmxnet3/vmxnet3.api +++ b/src/plugins/vmxnet3/vmxnet3.api @@ -28,6 +28,8 @@ option version = "1.0.0"; @param rxq_num - number of receive queues (optional - default is 1) @param txq_size - transmit queue size (optional - default is 1024) @param txq_num - number of transmit queues (optional - default is 1) + @param bind - automatically bind PCI to vfio-pci module + (optional - default is 0) */ define vmxnet3_create @@ -41,6 +43,7 @@ define vmxnet3_create u16 rxq_num; u16 txq_size; u16 txq_num; + u8 bind; }; /** \brief diff --git a/src/plugins/vmxnet3/vmxnet3.c b/src/plugins/vmxnet3/vmxnet3.c index 43f9cbed793..de1002381b6 100644 --- a/src/plugins/vmxnet3/vmxnet3.c +++ b/src/plugins/vmxnet3/vmxnet3.c @@ -658,9 +658,10 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args) pool_foreach (vd, vmxm->devices, ({ if (vd->pci_addr.as_u32 == args->addr.as_u32) { - args->rv = VNET_API_ERROR_INVALID_VALUE; + args->rv = VNET_API_ERROR_ADDRESS_IN_USE; args->error = - clib_error_return (error, "PCI address in use"); + clib_error_return (error, "%U: %s", format_vlib_pci_addr, + &args->addr, "pci address in use"); vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s", format_vlib_pci_addr, &args->addr, "pci address in use"); return; @@ -668,24 +669,31 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args) })); /* *INDENT-ON* */ - pool_get (vmxm->devices, vd); - vd->num_tx_queues = args->txq_num; - vd->num_rx_queues = args->rxq_num; - vd->dev_instance = vd - vmxm->devices; - vd->per_interface_next_index = ~0; - vd->pci_addr = args->addr; - - if (args->enable_elog) - vd->flags |= VMXNET3_DEVICE_F_ELOG; + if (args->bind) + { + error = vlib_pci_bind_to_uio (vm, &args->addr, (char *) "auto"); + if (error) + { + args->rv = VNET_API_ERROR_INVALID_INTERFACE; + args->error = + clib_error_return (error, "%U: %s", format_vlib_pci_addr, + &args->addr, + "error encountered on binding pci device"); + vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s", + format_vlib_pci_addr, &args->addr, + "error encountered on binding pci devicee"); + return; + } + } if ((error = vlib_pci_device_open (vm, &args->addr, vmxnet3_pci_device_ids, &h))) { - pool_put (vmxm->devices, vd); args->rv = VNET_API_ERROR_INVALID_INTERFACE; args->error = - clib_error_return (error, "pci-addr %U", format_vlib_pci_addr, - &args->addr); + clib_error_return (error, "%U: %s", format_vlib_pci_addr, + &args->addr, + "error encountered on pci device open"); vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s", format_vlib_pci_addr, &args->addr, "error encountered on pci device open"); @@ -696,6 +704,16 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args) * Do not use vmxnet3_log_error prior to this line since the macro * references vd->pci_dev_handle */ + pool_get (vmxm->devices, vd); + vd->num_tx_queues = args->txq_num; + vd->num_rx_queues = args->rxq_num; + vd->dev_instance = vd - vmxm->devices; + vd->per_interface_next_index = ~0; + vd->pci_addr = args->addr; + + if (args->enable_elog) + vd->flags |= VMXNET3_DEVICE_F_ELOG; + vd->pci_dev_handle = h; vd->numa_node = vlib_pci_get_numa_node (vm, h); vd->num_intrs = vd->num_rx_queues + 1; // +1 for the event interrupt diff --git a/src/plugins/vmxnet3/vmxnet3.h b/src/plugins/vmxnet3/vmxnet3.h index 7bc05ec4267..65aa929ddcd 100644 --- a/src/plugins/vmxnet3/vmxnet3.h +++ b/src/plugins/vmxnet3/vmxnet3.h @@ -605,6 +605,7 @@ typedef struct u16 rxq_num; u16 txq_size; u16 txq_num; + u8 bind; /* return */ i32 rv; u32 sw_if_index; diff --git a/src/plugins/vmxnet3/vmxnet3_api.c b/src/plugins/vmxnet3/vmxnet3_api.c index 0a66ffe0778..5e1bb9e2f18 100644 --- a/src/plugins/vmxnet3/vmxnet3_api.c +++ b/src/plugins/vmxnet3/vmxnet3_api.c @@ -72,6 +72,8 @@ vl_api_vmxnet3_create_t_handler (vl_api_vmxnet3_create_t * mp) args.rxq_size = ntohs (mp->rxq_size); args.txq_size = ntohs (mp->txq_size); args.txq_num = ntohs (mp->txq_num); + args.rxq_num = ntohs (mp->rxq_num); + args.bind = mp->bind; vmxnet3_create_if (vm, &args); rv = args.rv; diff --git a/src/plugins/vmxnet3/vmxnet3_test.c b/src/plugins/vmxnet3/vmxnet3_test.c index bb9f93e95cc..fe491b2a41d 100644 --- a/src/plugins/vmxnet3/vmxnet3_test.c +++ b/src/plugins/vmxnet3/vmxnet3_test.c @@ -117,12 +117,16 @@ api_vmxnet3_create (vat_main_t * vam) } else if (unformat (i, "elog")) args.enable_elog = 1; + else if (unformat (i, "bind")) + args.bind = 1; else if (unformat (i, "rx-queue-size %u", &args.rxq_size)) ; else if (unformat (i, "tx-queue-size %u", &args.txq_size)) ; else if (unformat (i, "num-tx-queues %u", &args.txq_num)) ; + else if (unformat (i, "num-rx-queues %u", &args.rxq_num)) + ; else { clib_warning ("unknown input '%U'", format_unformat_error, i); @@ -137,6 +141,8 @@ api_vmxnet3_create (vat_main_t * vam) mp->rxq_size = clib_host_to_net_u16 (args.rxq_size); mp->txq_size = clib_host_to_net_u16 (args.txq_size); mp->txq_num = clib_host_to_net_u16 (args.txq_num); + mp->rxq_num = clib_host_to_net_u16 (args.rxq_num); + mp->bind = args.bind; S (mp); W (ret); @@ -289,7 +295,8 @@ vl_api_vmxnet3_details_t_handler (vl_api_vmxnet3_details_t * mp) */ #define foreach_vpe_api_msg \ _(vmxnet3_create, " [rx-queue-size ] " \ - "[tx-queue-size ] [num-tx-queues ]") \ + "[tx-queue-size ] [num-tx-queues ]" \ + "[num-rx-queues ] [bind]") \ _(vmxnet3_delete, "sw_if_index ") \ _(vmxnet3_dump, "") -- 2.16.6