2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vppinfra/types.h>
17 #include <vlib/vlib.h>
18 #include <vlib/pci/pci.h>
19 #include <vnet/ethernet/ethernet.h>
20 #include <vnet/plugin/plugin.h>
21 #include <vpp/app/version.h>
23 #include <vmxnet3/vmxnet3.h>
25 #define PCI_VENDOR_ID_VMWARE 0x15ad
26 #define PCI_DEVICE_ID_VMWARE_VMXNET3 0x07b0
28 vmxnet3_main_t vmxnet3_main;
30 static pci_device_id_t vmxnet3_pci_device_ids[] = {
32 .vendor_id = PCI_VENDOR_ID_VMWARE,
33 .device_id = PCI_DEVICE_ID_VMWARE_VMXNET3},
38 vmxnet3_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index,
41 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
42 vmxnet3_main_t *vmxm = &vmxnet3_main;
43 vmxnet3_device_t *vd = vec_elt_at_index (vmxm->devices, hi->dev_instance);
44 uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
46 if (vd->flags & VMXNET3_DEVICE_F_ERROR)
47 return clib_error_return (0, "device is in error state");
51 vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
52 VNET_HW_INTERFACE_FLAG_LINK_UP);
53 vd->flags |= VMXNET3_DEVICE_F_ADMIN_UP;
57 vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
58 vd->flags &= ~VMXNET3_DEVICE_F_ADMIN_UP;
64 vmxnet3_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
65 vnet_hw_interface_rx_mode mode)
67 vmxnet3_main_t *vmxm = &vmxnet3_main;
68 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
69 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
70 vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
72 if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
81 vmxnet3_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
84 vmxnet3_main_t *vmxm = &vmxnet3_main;
85 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
86 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
88 /* Shut off redirection */
91 vd->per_interface_next_index = node_index;
95 vd->per_interface_next_index =
96 vlib_node_add_next (vlib_get_main (), vmxnet3_input_node.index,
100 static char *vmxnet3_tx_func_error_strings[] = {
102 foreach_vmxnet3_tx_func_error
107 VNET_DEVICE_CLASS (vmxnet3_device_class,) =
109 .name = "VMXNET3 interface",
110 .format_device = format_vmxnet3_device,
111 .format_device_name = format_vmxnet3_device_name,
112 .admin_up_down_function = vmxnet3_interface_admin_up_down,
113 .rx_mode_change_function = vmxnet3_interface_rx_mode_change,
114 .rx_redirect_to_node = vmxnet3_set_interface_next_node,
115 .tx_function_n_errors = VMXNET3_TX_N_ERROR,
116 .tx_function_error_strings = vmxnet3_tx_func_error_strings,
121 vmxnet3_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
127 vmxnet3_write_mac (vmxnet3_device_t * vd)
131 memcpy (&val, vd->mac_addr, 4);
132 vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACL, val);
135 memcpy (&val, vd->mac_addr + 4, 2);
136 vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACH, val);
139 static clib_error_t *
140 vmxnet3_provision_driver_shared (vlib_main_t * vm, vmxnet3_device_t * vd)
142 vmxnet3_main_t *vmxm = &vmxnet3_main;
143 vmxnet3_shared *shared;
148 vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
149 vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, qid);
151 vd->dma = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
152 sizeof (*vd->dma), 512);
156 memset (vd->dma, 0, sizeof (*vd->dma));
158 q = &vd->dma->queues;
159 q->tx.cfg.desc_address = vmxnet3_dma_addr (vm, vd, txq->tx_desc);
160 q->tx.cfg.comp_address = vmxnet3_dma_addr (vm, vd, txq->tx_comp);
161 q->tx.cfg.num_desc = txq->size;
162 q->tx.cfg.num_comp = txq->size;
163 for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
165 q->rx.cfg.desc_address[rid] = vmxnet3_dma_addr (vm, vd,
167 q->rx.cfg.num_desc[rid] = rxq->size;
169 q->rx.cfg.comp_address = vmxnet3_dma_addr (vm, vd, rxq->rx_comp);
170 q->rx.cfg.num_comp = rxq->size;
172 shared = &vd->dma->shared;
173 shared->magic = VMXNET3_SHARED_MAGIC;
174 shared->misc.version = VMXNET3_VERSION_MAGIC;
175 if (sizeof (void *) == 4)
176 shared->misc.guest_info = VMXNET3_GOS_BITS_32;
178 shared->misc.guest_info = VMXNET3_GOS_BITS_64;
179 shared->misc.guest_info |= VMXNET3_GOS_TYPE_LINUX;
180 shared->misc.version_support = VMXNET3_VERSION_SELECT;
181 shared->misc.upt_version_support = VMXNET3_UPT_VERSION_SELECT;
182 shared->misc.queue_desc_address = vmxnet3_dma_addr (vm, vd, q);
183 shared->misc.queue_desc_len = sizeof (*q);
184 shared->misc.mtu = VMXNET3_MTU;
185 shared->misc.num_tx_queues = vd->num_tx_queues;
186 shared->misc.num_rx_queues = vd->num_rx_queues;
187 shared->interrupt.num_intrs = vd->num_intrs;
188 shared->interrupt.event_intr_index = 1;
189 shared->interrupt.control = VMXNET3_IC_DISABLE_ALL;
190 shared->rx_filter.mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST |
191 VMXNET3_RXMODE_ALL_MULTI;
192 shared_dma = vmxnet3_dma_addr (vm, vd, shared);
194 vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAL, shared_dma);
195 vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAH, shared_dma >> 32);
201 vmxnet3_enable_interrupt (vmxnet3_device_t * vd)
204 vmxnet3_shared *shared = &vd->dma->shared;
206 shared->interrupt.control &= ~VMXNET3_IC_DISABLE_ALL;
207 for (i = 0; i < vd->num_intrs; i++)
208 vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 0);
212 vmxnet3_disable_interrupt (vmxnet3_device_t * vd)
215 vmxnet3_shared *shared = &vd->dma->shared;
217 shared->interrupt.control |= VMXNET3_IC_DISABLE_ALL;
218 for (i = 0; i < vd->num_intrs; i++)
219 vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 1);
222 static clib_error_t *
223 vmxnet3_rxq_init (vlib_main_t * vm, vmxnet3_device_t * vd, u16 qid, u16 qsz)
225 vmxnet3_main_t *vmxm = &vmxnet3_main;
230 vec_validate_aligned (vd->rxqs, qid, CLIB_CACHE_LINE_BYTES);
231 rxq = vec_elt_at_index (vd->rxqs, qid);
232 memset (rxq, 0, sizeof (*rxq));
234 for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
237 vlib_physmem_alloc_aligned (vm, vmxm->physmem_region,
238 &error, qsz * sizeof (*rxq->rx_desc[rid]),
242 memset (rxq->rx_desc[rid], 0, qsz * sizeof (*rxq->rx_desc[rid]));
244 rxq->rx_comp = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
245 qsz * sizeof (*rxq->rx_comp),
249 memset (rxq->rx_comp, 0, qsz * sizeof (*rxq->rx_comp));
250 for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
252 vmxnet3_rx_ring *ring;
254 ring = &rxq->rx_ring[rid];
255 ring->gen = VMXNET3_RXF_GEN;
257 vec_validate_aligned (ring->bufs, rxq->size, CLIB_CACHE_LINE_BYTES);
259 rxq->rx_comp_ring.gen = VMXNET3_RXCF_GEN;
264 static clib_error_t *
265 vmxnet3_txq_init (vlib_main_t * vm, vmxnet3_device_t * vd, u16 qid, u16 qsz)
267 vmxnet3_main_t *vmxm = &vmxnet3_main;
271 if (qid >= vd->num_tx_queues)
273 qid = qid % vd->num_tx_queues;
274 txq = vec_elt_at_index (vd->txqs, qid);
276 clib_spinlock_init (&txq->lock);
277 vd->flags |= VMXNET3_DEVICE_F_SHARED_TXQ_LOCK;
281 vec_validate_aligned (vd->txqs, qid, CLIB_CACHE_LINE_BYTES);
282 txq = vec_elt_at_index (vd->txqs, qid);
283 memset (txq, 0, sizeof (*txq));
285 txq->tx_desc = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
286 qsz * sizeof (*txq->tx_desc),
290 memset (txq->tx_desc, 0, qsz * sizeof (*txq->tx_desc));
291 txq->tx_comp = vlib_physmem_alloc_aligned (vm, vmxm->physmem_region, &error,
292 qsz * sizeof (*txq->tx_comp),
296 memset (txq->tx_comp, 0, qsz * sizeof (*txq->tx_comp));
297 vec_validate_aligned (txq->tx_ring.bufs, txq->size, CLIB_CACHE_LINE_BYTES);
298 txq->tx_ring.gen = VMXNET3_TXF_GEN;
299 txq->tx_comp_ring.gen = VMXNET3_TXCF_GEN;
304 static clib_error_t *
305 vmxnet3_device_init (vlib_main_t * vm, vmxnet3_device_t * vd,
306 vmxnet3_create_if_args_t * args)
308 clib_error_t *error = 0;
310 vmxnet3_main_t *vmxm = &vmxnet3_main;
311 vlib_thread_main_t *tm = vlib_get_thread_main ();
313 vd->num_tx_queues = 1;
314 vd->num_rx_queues = 1;
317 /* Quiesce the device */
318 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
319 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
322 error = clib_error_return (0, "error on quisecing device rc (%u)", ret);
326 /* Reset the device */
327 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
328 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
331 error = clib_error_return (0, "error on resetting device rc (%u)", ret);
335 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_VRRS);
336 vd->version = count_leading_zeros (ret);
337 vd->version = uword_bits - vd->version;
339 if (vd->version == 0 || vd->version > 3)
341 error = clib_error_return (0, "unsupported hardware version %u",
346 vmxnet3_reg_write (vd, 1, VMXNET3_REG_VRRS, 1 << (vd->version - 1));
348 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_UVRS);
350 vmxnet3_reg_write (vd, 1, VMXNET3_REG_UVRS, 1);
353 error = clib_error_return (0, "unsupported upt version %u", ret);
357 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
358 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
361 vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
362 vd->link_speed = ret >> 16;
366 vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
369 /* Get the mac address */
370 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACL);
371 clib_memcpy (vd->mac_addr, &ret, 4);
372 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACH);
373 clib_memcpy (vd->mac_addr + 4, &ret, 2);
375 if (vmxm->physmem_region_alloc == 0)
377 u32 flags = VLIB_PHYSMEM_F_INIT_MHEAP | VLIB_PHYSMEM_F_HUGETLB;
379 vlib_physmem_region_alloc (vm, "vmxnet3 descriptors", 4 << 20, 0,
380 flags, &vmxm->physmem_region);
383 vmxm->physmem_region_alloc = 1;
386 error = vmxnet3_rxq_init (vm, vd, 0, args->rxq_size);
390 for (i = 0; i < tm->n_vlib_mains; i++)
392 error = vmxnet3_txq_init (vm, vd, i, args->txq_size);
397 error = vmxnet3_provision_driver_shared (vm, vd);
401 vmxnet3_write_mac (vd);
403 /* Activate device */
404 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV);
405 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
409 clib_error_return (0, "error on activating device rc (%u)", ret);
413 /* Disable interrupts */
414 vmxnet3_disable_interrupt (vd);
416 vec_foreach_index (i, vd->rxqs)
418 vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
420 vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
421 vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
423 vd->flags |= VMXNET3_DEVICE_F_INITIALIZED;
425 vmxnet3_enable_interrupt (vd);
431 vmxnet3_irq_0_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
433 vnet_main_t *vnm = vnet_get_main ();
434 vmxnet3_main_t *vmxm = &vmxnet3_main;
435 uword pd = vlib_pci_get_private_data (vm, h);
436 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
439 if (vec_len (vd->rxqs) > qid && vd->rxqs[qid].int_mode != 0)
440 vnet_device_input_set_interrupt_pending (vnm, vd->hw_if_index, qid);
444 vmxnet3_irq_1_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
446 vnet_main_t *vnm = vnet_get_main ();
447 vmxnet3_main_t *vmxm = &vmxnet3_main;
448 uword pd = vlib_pci_get_private_data (vm, h);
449 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
452 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
453 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
456 vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
457 vd->link_speed = ret >> 16;
458 vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
459 VNET_HW_INTERFACE_FLAG_LINK_UP);
463 vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
464 vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
469 vmxnet3_queue_size_valid (u16 qsz)
471 if (qsz < 64 || qsz > 4096)
479 vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
481 vnet_main_t *vnm = vnet_get_main ();
482 vmxnet3_main_t *vmxm = &vmxnet3_main;
483 vmxnet3_device_t *vd;
484 vlib_pci_dev_handle_t h;
485 clib_error_t *error = 0;
487 if (args->rxq_size == 0)
488 args->rxq_size = VMXNET3_NUM_RX_DESC;
489 if (args->txq_size == 0)
490 args->txq_size = VMXNET3_NUM_TX_DESC;
492 if (!vmxnet3_queue_size_valid (args->rxq_size) ||
493 !vmxnet3_queue_size_valid (args->txq_size))
495 args->rv = VNET_API_ERROR_INVALID_VALUE;
497 clib_error_return (error,
498 "queue size must be <= 4096, >= 64, "
499 "and multiples of 64");
504 pool_foreach (vd, vmxm->devices, ({
505 if (vd->pci_addr.as_u32 == args->addr.as_u32)
507 args->rv = VNET_API_ERROR_INVALID_VALUE;
509 clib_error_return (error, "PCI address in use");
515 pool_get (vmxm->devices, vd);
516 vd->dev_instance = vd - vmxm->devices;
517 vd->per_interface_next_index = ~0;
518 vd->pci_addr = args->addr;
520 if (args->enable_elog)
521 vd->flags |= VMXNET3_DEVICE_F_ELOG;
524 vlib_pci_device_open (vm, &args->addr, vmxnet3_pci_device_ids, &h)))
526 pool_put (vmxm->devices, vd);
527 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
529 clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
533 vd->pci_dev_handle = h;
535 vlib_pci_set_private_data (vm, h, vd->dev_instance);
537 if ((error = vlib_pci_bus_master_enable (vm, h)))
540 if ((error = vlib_pci_map_region (vm, h, 0, (void **) &vd->bar[0])))
543 if ((error = vlib_pci_map_region (vm, h, 1, (void **) &vd->bar[1])))
546 if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
547 &vmxnet3_irq_0_handler)))
550 if ((error = vlib_pci_register_msix_handler (vm, h, 1, 1,
551 &vmxnet3_irq_1_handler)))
554 if ((error = vlib_pci_enable_msix_irq (vm, h, 0, 2)))
557 if ((error = vlib_pci_intr_enable (vm, h)))
560 if ((error = vmxnet3_device_init (vm, vd, args)))
563 /* create interface */
564 error = ethernet_register_interface (vnm, vmxnet3_device_class.index,
565 vd->dev_instance, vd->mac_addr,
566 &vd->hw_if_index, vmxnet3_flag_change);
571 vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vd->hw_if_index);
572 vd->sw_if_index = sw->sw_if_index;
573 args->sw_if_index = sw->sw_if_index;
575 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vd->hw_if_index);
576 hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
577 vnet_hw_interface_set_input_node (vnm, vd->hw_if_index,
578 vmxnet3_input_node.index);
579 vnet_hw_interface_assign_rx_thread (vnm, vd->hw_if_index, 0, ~0);
580 if (vd->flags & VMXNET3_DEVICE_F_LINK_UP)
581 vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
582 VNET_HW_INTERFACE_FLAG_LINK_UP);
584 vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
588 vmxnet3_delete_if (vm, vd);
589 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
594 vmxnet3_delete_if (vlib_main_t * vm, vmxnet3_device_t * vd)
596 vnet_main_t *vnm = vnet_get_main ();
597 vmxnet3_main_t *vmxm = &vmxnet3_main;
601 /* Quiesce the device */
602 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
604 /* Reset the device */
605 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
609 vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
610 vnet_hw_interface_unassign_rx_thread (vnm, vd->hw_if_index, 0);
611 ethernet_delete_interface (vnm, vd->hw_if_index);
614 vlib_pci_device_close (vm, vd->pci_dev_handle);
617 vec_foreach_index (i, vd->rxqs)
619 vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
620 u16 mask = rxq->size - 1;
623 for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
625 vmxnet3_rx_ring *ring;
627 ring = &rxq->rx_ring[rid];
628 desc_idx = (ring->consume + 1) & mask;
629 vlib_buffer_free_from_ring (vm, ring->bufs, desc_idx, rxq->size,
631 vec_free (ring->bufs);
632 vlib_physmem_free (vm, vmxm->physmem_region, rxq->rx_desc[rid]);
634 vlib_physmem_free (vm, vmxm->physmem_region, rxq->rx_comp);
640 vec_foreach_index (i, vd->txqs)
642 vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, i);
643 u16 mask = txq->size - 1;
646 desc_idx = txq->tx_ring.consume;
647 end_idx = txq->tx_ring.produce;
648 while (desc_idx != end_idx)
650 bi = txq->tx_ring.bufs[desc_idx];
651 vlib_buffer_free_no_next (vm, &bi, 1);
655 clib_spinlock_free (&txq->lock);
656 vec_free (txq->tx_ring.bufs);
657 vlib_physmem_free (vm, vmxm->physmem_region, txq->tx_desc);
658 vlib_physmem_free (vm, vmxm->physmem_region, txq->tx_comp);
663 vlib_physmem_free (vm, vmxm->physmem_region, vd->dma);
665 clib_error_free (vd->error);
666 memset (vd, 0, sizeof (*vd));
667 pool_put (vmxm->devices, vd);
671 * fd.io coding-style-patch-verification: ON
674 * eval: (c-set-style "gnu")