vmxnet3: auto bind support

For creating the vmxnet3 interface, add the bind option to automatically bind
the pci to vfio-pci module which removes the need for manual bind. Manual bind
still works, should people prefer to go that route.

Change-Id: Ife75926f8755d754a08dd0ecff0f1de326ad5ba1
Signed-off-by: Steven Luong <sluong@cisco.com>

diff --git a/src/plugins/vmxnet3/README.md b/src/plugins/vmxnet3/README.md
index b582993..722954d 100644 (file)
--- a/src/plugins/vmxnet3/README.md
+++ b/src/plugins/vmxnet3/README.md
@@ -7,11 +7,11 @@ This plugin provides native PCI driver support for VMWare vmxnet3.
  * This code is tested with vfio-pci driver installed with Ubuntu 18.04 which
 has kernel version 4.15.0-33-generic.
 
- * This code is tested with ESXi vSwitch version 6.0, release build 3620759.
+ * This driver is tested with ESXi vSwitch version 6.5/6.7 for LRO/TSO support, VMware Workstation 15 Pro (no LRO/TSO), and VMware Fusion 11 Pro (no LRO/TSO)
 
  * Driver requires MSI-X interrupt support, which is not supported by
-uio_pci_generic driver, so vfio-pci needs to be used. On systems without IOMMU
-vfio driver can still be used with recent kernels which support no-iommu mode.
+uio_pci_generic driver. So vfio-pci must be used. On systems without IOMMU,
+vfio driver can still be used with 4.15.0-33-generic kernel (Ubuntu 18.04) which supports no-iommu mode.
 
 ##Known issues
 
@@ -25,20 +25,27 @@ vfio driver can still be used with recent kernels which support no-iommu mode.
 sudo modprobe vfio-pci
 ```
 
-2. (systems without IOMMU only) enable unsafe NOIOMMU mode
+2. Make sure the interface is down
+```
+sudo ifconfig <if-name> down
+```
+
+Steps 3 and 4 are optional. They can be accomplished by specifying the optional keyword "bind" when creating the vmxnet3 interface.
+
+3. (systems without IOMMU only) enable unsafe NOIOMMU mode
 ```
 echo Y | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
 ```
 
-3. Bind interface to vfio-pci
+4. Bind interface to vfio-pci
 ```
 sudo dpdk-devbind.py --bind vfio-pci 0b:00.0
 ```
 
 ### Interface Creation
-Interface can be dynamically created with following CLI:
+Interface can be dynamically created with following CLI, with or without the bind option. If step 3 and 4 were executed, bind can be omitted.
 ```
-create interface vmxnet3 0000:0b:00.0
+create interface vmxnet3 0000:0b:00.0 bind
 set int state vmxnet3-0/b/0/0 up
 ```
 

diff --git a/src/plugins/vmxnet3/cli.c b/src/plugins/vmxnet3/cli.c
index 0f0cf60..776901e 100644 (file)
--- a/src/plugins/vmxnet3/cli.c
+++ b/src/plugins/vmxnet3/cli.c
@@ -44,6 +44,8 @@ vmxnet3_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
        ;
       else if (unformat (line_input, "elog"))
        args.enable_elog = 1;
+      else if (unformat (line_input, "bind"))
+       args.bind = 1;
       else if (unformat (line_input, "rx-queue-size %u", &args.rxq_size))
        ;
       else if (unformat (line_input, "tx-queue-size %u", &args.txq_size))
@@ -69,7 +71,7 @@ VLIB_CLI_COMMAND (vmxnet3_create_command, static) = {
   .path = "create interface vmxnet3",
   .short_help = "create interface vmxnet3 <pci-address>"
                 "[rx-queue-size <size>] [tx-queue-size <size>]"
-                "[num-tx-queues <number>] [num-rx-queues <number>]",
+                "[num-tx-queues <number>] [num-rx-queues <number>] [bind]",
   .function = vmxnet3_create_command_fn,
 };
 /* *INDENT-ON* */

diff --git a/src/plugins/vmxnet3/vmxnet3.api b/src/plugins/vmxnet3/vmxnet3.api
index 804ba53..7408836 100644 (file)
--- a/src/plugins/vmxnet3/vmxnet3.api
+++ b/src/plugins/vmxnet3/vmxnet3.api
@@ -28,6 +28,8 @@ option version = "1.0.0";
     @param rxq_num - number of receive queues (optional - default is 1)
     @param txq_size - transmit queue size (optional - default is 1024)
     @param txq_num - number of transmit queues (optional - default is 1)
+    @param bind - automatically bind PCI to vfio-pci module
+                     (optional - default is 0)
 */
 
 define vmxnet3_create
@@ -41,6 +43,7 @@ define vmxnet3_create
   u16 rxq_num;
   u16 txq_size;
   u16 txq_num;
+  u8 bind;
 };
 
 /** \brief

diff --git a/src/plugins/vmxnet3/vmxnet3.c b/src/plugins/vmxnet3/vmxnet3.c
index 43f9cbe..de10023 100644 (file)
--- a/src/plugins/vmxnet3/vmxnet3.c
+++ b/src/plugins/vmxnet3/vmxnet3.c
@@ -658,9 +658,10 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
   pool_foreach (vd, vmxm->devices, ({
     if (vd->pci_addr.as_u32 == args->addr.as_u32)
       {
-       args->rv = VNET_API_ERROR_INVALID_VALUE;
+       args->rv = VNET_API_ERROR_ADDRESS_IN_USE;
        args->error =
-         clib_error_return (error, "PCI address in use");
+         clib_error_return (error, "%U: %s", format_vlib_pci_addr,
+                            &args->addr, "pci address in use");
        vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
                  format_vlib_pci_addr, &args->addr, "pci address in use");
        return;
@@ -668,24 +669,31 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
   }));
   /* *INDENT-ON* */
 
-  pool_get (vmxm->devices, vd);
-  vd->num_tx_queues = args->txq_num;
-  vd->num_rx_queues = args->rxq_num;
-  vd->dev_instance = vd - vmxm->devices;
-  vd->per_interface_next_index = ~0;
-  vd->pci_addr = args->addr;
-
-  if (args->enable_elog)
-    vd->flags |= VMXNET3_DEVICE_F_ELOG;
+  if (args->bind)
+    {
+      error = vlib_pci_bind_to_uio (vm, &args->addr, (char *) "auto");
+      if (error)
+       {
+         args->rv = VNET_API_ERROR_INVALID_INTERFACE;
+         args->error =
+           clib_error_return (error, "%U: %s", format_vlib_pci_addr,
+                              &args->addr,
+                              "error encountered on binding pci device");
+         vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
+                   format_vlib_pci_addr, &args->addr,
+                   "error encountered on binding pci devicee");
+         return;
+       }
+    }
 
   if ((error =
        vlib_pci_device_open (vm, &args->addr, vmxnet3_pci_device_ids, &h)))
     {
-      pool_put (vmxm->devices, vd);
       args->rv = VNET_API_ERROR_INVALID_INTERFACE;
       args->error =
-       clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
-                          &args->addr);
+       clib_error_return (error, "%U: %s", format_vlib_pci_addr,
+                          &args->addr,
+                          "error encountered on pci device open");
       vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
                format_vlib_pci_addr, &args->addr,
                "error encountered on pci device open");
@@ -696,6 +704,16 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
    * Do not use vmxnet3_log_error prior to this line since the macro
    * references vd->pci_dev_handle
    */
+  pool_get (vmxm->devices, vd);
+  vd->num_tx_queues = args->txq_num;
+  vd->num_rx_queues = args->rxq_num;
+  vd->dev_instance = vd - vmxm->devices;
+  vd->per_interface_next_index = ~0;
+  vd->pci_addr = args->addr;
+
+  if (args->enable_elog)
+    vd->flags |= VMXNET3_DEVICE_F_ELOG;
+
   vd->pci_dev_handle = h;
   vd->numa_node = vlib_pci_get_numa_node (vm, h);
   vd->num_intrs = vd->num_rx_queues + 1;       // +1 for the event interrupt

diff --git a/src/plugins/vmxnet3/vmxnet3.h b/src/plugins/vmxnet3/vmxnet3.h
index 7bc05ec..65aa929 100644 (file)
--- a/src/plugins/vmxnet3/vmxnet3.h
+++ b/src/plugins/vmxnet3/vmxnet3.h
@@ -605,6 +605,7 @@ typedef struct
   u16 rxq_num;
   u16 txq_size;
   u16 txq_num;
+  u8 bind;
   /* return */
   i32 rv;
   u32 sw_if_index;

diff --git a/src/plugins/vmxnet3/vmxnet3_api.c b/src/plugins/vmxnet3/vmxnet3_api.c
index 0a66ffe..5e1bb9e 100644 (file)
--- a/src/plugins/vmxnet3/vmxnet3_api.c
+++ b/src/plugins/vmxnet3/vmxnet3_api.c
@@ -72,6 +72,8 @@ vl_api_vmxnet3_create_t_handler (vl_api_vmxnet3_create_t * mp)
   args.rxq_size = ntohs (mp->rxq_size);
   args.txq_size = ntohs (mp->txq_size);
   args.txq_num = ntohs (mp->txq_num);
+  args.rxq_num = ntohs (mp->rxq_num);
+  args.bind = mp->bind;
 
   vmxnet3_create_if (vm, &args);
   rv = args.rv;

diff --git a/src/plugins/vmxnet3/vmxnet3_test.c b/src/plugins/vmxnet3/vmxnet3_test.c
index bb9f93e..fe491b2 100644 (file)
--- a/src/plugins/vmxnet3/vmxnet3_test.c
+++ b/src/plugins/vmxnet3/vmxnet3_test.c
@@ -117,12 +117,16 @@ api_vmxnet3_create (vat_main_t * vam)
        }
       else if (unformat (i, "elog"))
        args.enable_elog = 1;
+      else if (unformat (i, "bind"))
+       args.bind = 1;
       else if (unformat (i, "rx-queue-size %u", &args.rxq_size))
        ;
       else if (unformat (i, "tx-queue-size %u", &args.txq_size))
        ;
       else if (unformat (i, "num-tx-queues %u", &args.txq_num))
        ;
+      else if (unformat (i, "num-rx-queues %u", &args.rxq_num))
+       ;
       else
        {
          clib_warning ("unknown input '%U'", format_unformat_error, i);
@@ -137,6 +141,8 @@ api_vmxnet3_create (vat_main_t * vam)
   mp->rxq_size = clib_host_to_net_u16 (args.rxq_size);
   mp->txq_size = clib_host_to_net_u16 (args.txq_size);
   mp->txq_num = clib_host_to_net_u16 (args.txq_num);
+  mp->rxq_num = clib_host_to_net_u16 (args.rxq_num);
+  mp->bind = args.bind;
 
   S (mp);
   W (ret);
@@ -289,7 +295,8 @@ vl_api_vmxnet3_details_t_handler (vl_api_vmxnet3_details_t * mp)
  */
 #define foreach_vpe_api_msg                                    \
 _(vmxnet3_create, "<pci-address> [rx-queue-size <size>] "      \
-              "[tx-queue-size <size>] [num-tx-queues <num>]")  \
+  "[tx-queue-size <size>] [num-tx-queues <num>]"               \
+  "[num-rx-queues <num>] [bind]")                              \
 _(vmxnet3_delete, "sw_if_index <sw_if_index>")                  \
 _(vmxnet3_dump, "")