2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vppinfra/types.h>
17 #include <vlib/vlib.h>
18 #include <vlib/pci/pci.h>
19 #include <vnet/ethernet/ethernet.h>
20 #include <vnet/plugin/plugin.h>
21 #include <vpp/app/version.h>
23 #include <vmxnet3/vmxnet3.h>
25 #define PCI_VENDOR_ID_VMWARE 0x15ad
26 #define PCI_DEVICE_ID_VMWARE_VMXNET3 0x07b0
28 vmxnet3_main_t vmxnet3_main;
30 static pci_device_id_t vmxnet3_pci_device_ids[] = {
32 .vendor_id = PCI_VENDOR_ID_VMWARE,
33 .device_id = PCI_DEVICE_ID_VMWARE_VMXNET3},
38 vmxnet3_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index,
41 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
42 vmxnet3_main_t *vmxm = &vmxnet3_main;
43 vmxnet3_device_t *vd = vec_elt_at_index (vmxm->devices, hi->dev_instance);
44 uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
46 if (vd->flags & VMXNET3_DEVICE_F_ERROR)
47 return clib_error_return (0, "device is in error state");
51 vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
52 VNET_HW_INTERFACE_FLAG_LINK_UP);
53 vd->flags |= VMXNET3_DEVICE_F_ADMIN_UP;
57 vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
58 vd->flags &= ~VMXNET3_DEVICE_F_ADMIN_UP;
64 vmxnet3_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
65 vnet_hw_interface_rx_mode mode)
67 vmxnet3_main_t *vmxm = &vmxnet3_main;
68 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
69 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
70 vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
72 if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
81 vmxnet3_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
84 vmxnet3_main_t *vmxm = &vmxnet3_main;
85 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
86 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, hw->dev_instance);
88 /* Shut off redirection */
91 vd->per_interface_next_index = node_index;
95 vd->per_interface_next_index =
96 vlib_node_add_next (vlib_get_main (), vmxnet3_input_node.index,
101 vmxnet3_clear_hw_interface_counters (u32 instance)
103 vmxnet3_main_t *vmxm = &vmxnet3_main;
104 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, instance);
105 vmxnet3_tx_queue *tx = VMXNET3_TX_START (vd);
106 vmxnet3_rx_queue *rx = VMXNET3_RX_START (vd);
110 * Set the "last_cleared_stats" to the current stats, so that
111 * things appear to clear from a display perspective.
113 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
115 vec_foreach_index (qid, vd->txqs)
117 vmxnet3_tx_stats *txs = vec_elt_at_index (vd->tx_stats, qid);
118 clib_memcpy (txs, &tx->stats, sizeof (*txs));
121 vec_foreach_index (qid, vd->rxqs)
123 vmxnet3_rx_stats *rxs = vec_elt_at_index (vd->rx_stats, qid);
124 clib_memcpy (rxs, &rx->stats, sizeof (*rxs));
129 static char *vmxnet3_tx_func_error_strings[] = {
131 foreach_vmxnet3_tx_func_error
136 VNET_DEVICE_CLASS (vmxnet3_device_class,) =
138 .name = "VMXNET3 interface",
139 .format_device = format_vmxnet3_device,
140 .format_device_name = format_vmxnet3_device_name,
141 .admin_up_down_function = vmxnet3_interface_admin_up_down,
142 .clear_counters = vmxnet3_clear_hw_interface_counters,
143 .rx_mode_change_function = vmxnet3_interface_rx_mode_change,
144 .rx_redirect_to_node = vmxnet3_set_interface_next_node,
145 .tx_function_n_errors = VMXNET3_TX_N_ERROR,
146 .tx_function_error_strings = vmxnet3_tx_func_error_strings,
151 vmxnet3_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
157 vmxnet3_write_mac (vmxnet3_device_t * vd)
161 memcpy (&val, vd->mac_addr, 4);
162 vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACL, val);
165 memcpy (&val, vd->mac_addr + 4, 2);
166 vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACH, val);
169 static clib_error_t *
170 vmxnet3_provision_driver_shared (vlib_main_t * vm, vmxnet3_device_t * vd)
172 vmxnet3_shared *shared;
175 vmxnet3_tx_queue *tx = VMXNET3_TX_START (vd);
176 vmxnet3_rx_queue *rx = VMXNET3_RX_START (vd);
179 vlib_physmem_alloc_aligned_on_numa (vm, sizeof (*vd->driver_shared), 512,
181 if (vd->driver_shared == 0)
182 return vlib_physmem_last_error (vm);
184 clib_memset (vd->driver_shared, 0, sizeof (*vd->driver_shared));
186 vec_foreach_index (qid, vd->txqs)
188 vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, qid);
190 tx->cfg.desc_address = vmxnet3_dma_addr (vm, vd, txq->tx_desc);
191 tx->cfg.comp_address = vmxnet3_dma_addr (vm, vd, txq->tx_comp);
192 tx->cfg.num_desc = txq->size;
193 tx->cfg.num_comp = txq->size;
197 vec_foreach_index (qid, vd->rxqs)
199 vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
201 for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
203 rx->cfg.desc_address[rid] = vmxnet3_dma_addr (vm, vd,
205 rx->cfg.num_desc[rid] = rxq->size;
207 rx->cfg.comp_address = vmxnet3_dma_addr (vm, vd, rxq->rx_comp);
208 rx->cfg.num_comp = rxq->size;
209 rx->cfg.intr_index = qid;
213 shared = vd->driver_shared;
214 shared->magic = VMXNET3_SHARED_MAGIC;
215 shared->misc.version = VMXNET3_VERSION_MAGIC;
216 if (sizeof (void *) == 4)
217 shared->misc.guest_info = VMXNET3_GOS_BITS_32;
219 shared->misc.guest_info = VMXNET3_GOS_BITS_64;
220 shared->misc.guest_info |= VMXNET3_GOS_TYPE_LINUX;
221 shared->misc.version_support = VMXNET3_VERSION_SELECT;
222 shared->misc.upt_features = VMXNET3_F_RXCSUM;
224 shared->misc.upt_features |= VMXNET3_F_LRO;
225 if (vd->num_rx_queues > 1)
227 shared->misc.upt_features |= VMXNET3_F_RSS;
228 shared->rss.version = 1;
229 shared->rss.address = vmxnet3_dma_addr (vm, vd, vd->rss);
230 shared->rss.length = sizeof (*vd->rss);
232 shared->misc.max_num_rx_sg = 0;
233 shared->misc.upt_version_support = VMXNET3_UPT_VERSION_SELECT;
234 shared->misc.queue_desc_address = vmxnet3_dma_addr (vm, vd, vd->queues);
235 shared->misc.queue_desc_len = sizeof (*tx) * vd->num_tx_queues +
236 sizeof (*rx) * vd->num_rx_queues;
237 shared->misc.mtu = VMXNET3_MTU;
238 shared->misc.num_tx_queues = vd->num_tx_queues;
239 shared->misc.num_rx_queues = vd->num_rx_queues;
240 shared->interrupt.num_intrs = vd->num_intrs;
241 shared->interrupt.event_intr_index = vd->num_rx_queues;
242 shared->interrupt.control = VMXNET3_IC_DISABLE_ALL;
243 shared->rx_filter.mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST |
244 VMXNET3_RXMODE_ALL_MULTI | VMXNET3_RXMODE_PROMISC;
245 shared_dma = vmxnet3_dma_addr (vm, vd, shared);
247 vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAL, shared_dma);
248 vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAH, shared_dma >> 32);
254 vmxnet3_enable_interrupt (vmxnet3_device_t * vd)
257 vmxnet3_shared *shared = vd->driver_shared;
259 shared->interrupt.control &= ~VMXNET3_IC_DISABLE_ALL;
260 for (i = 0; i < vd->num_intrs; i++)
261 vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 0);
265 vmxnet3_disable_interrupt (vmxnet3_device_t * vd)
268 vmxnet3_shared *shared = vd->driver_shared;
270 shared->interrupt.control |= VMXNET3_IC_DISABLE_ALL;
271 for (i = 0; i < vd->num_intrs; i++)
272 vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 1);
275 static clib_error_t *
276 vmxnet3_rxq_init (vlib_main_t * vm, vmxnet3_device_t * vd, u16 qid, u16 qsz)
279 vmxnet3_rx_stats *rxs;
282 vec_validate (vd->rx_stats, qid);
283 rxs = vec_elt_at_index (vd->rx_stats, qid);
284 clib_memset (rxs, 0, sizeof (*rxs));
286 vec_validate_aligned (vd->rxqs, qid, CLIB_CACHE_LINE_BYTES);
287 rxq = vec_elt_at_index (vd->rxqs, qid);
288 clib_memset (rxq, 0, sizeof (*rxq));
290 for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
292 rxq->rx_desc[rid] = vlib_physmem_alloc_aligned_on_numa
293 (vm, qsz * sizeof (*rxq->rx_desc[rid]), 512, vd->numa_node);
295 if (rxq->rx_desc[rid] == 0)
296 return vlib_physmem_last_error (vm);
298 clib_memset (rxq->rx_desc[rid], 0, qsz * sizeof (*rxq->rx_desc[rid]));
301 vlib_physmem_alloc_aligned_on_numa (vm, qsz * sizeof (*rxq->rx_comp), 512,
303 if (rxq->rx_comp == 0)
304 return vlib_physmem_last_error (vm);
306 clib_memset (rxq->rx_comp, 0, qsz * sizeof (*rxq->rx_comp));
307 for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
309 vmxnet3_rx_ring *ring;
311 ring = &rxq->rx_ring[rid];
312 ring->gen = VMXNET3_RXF_GEN;
314 vec_validate_aligned (ring->bufs, rxq->size, CLIB_CACHE_LINE_BYTES);
316 rxq->rx_comp_ring.gen = VMXNET3_RXCF_GEN;
321 static clib_error_t *
322 vmxnet3_txq_init (vlib_main_t * vm, vmxnet3_device_t * vd, u16 qid, u16 qsz)
325 vmxnet3_tx_stats *txs;
328 if (qid >= vd->num_tx_queues)
330 qid = qid % vd->num_tx_queues;
331 txq = vec_elt_at_index (vd->txqs, qid);
333 clib_spinlock_init (&txq->lock);
334 vd->flags |= VMXNET3_DEVICE_F_SHARED_TXQ_LOCK;
338 vec_validate (vd->tx_stats, qid);
339 txs = vec_elt_at_index (vd->tx_stats, qid);
340 clib_memset (txs, 0, sizeof (*txs));
342 vec_validate_aligned (vd->txqs, qid, CLIB_CACHE_LINE_BYTES);
343 txq = vec_elt_at_index (vd->txqs, qid);
344 clib_memset (txq, 0, sizeof (*txq));
346 txq->reg_txprod = qid * 8 + VMXNET3_REG_TXPROD;
348 size = qsz * sizeof (*txq->tx_desc);
350 vlib_physmem_alloc_aligned_on_numa (vm, size, 512, vd->numa_node);
351 if (txq->tx_desc == 0)
352 return vlib_physmem_last_error (vm);
354 memset (txq->tx_desc, 0, size);
356 size = qsz * sizeof (*txq->tx_comp);
358 vlib_physmem_alloc_aligned_on_numa (vm, size, 512, vd->numa_node);
359 if (txq->tx_comp == 0)
360 return vlib_physmem_last_error (vm);
362 clib_memset (txq->tx_comp, 0, size);
363 vec_validate_aligned (txq->tx_ring.bufs, txq->size, CLIB_CACHE_LINE_BYTES);
364 txq->tx_ring.gen = VMXNET3_TXF_GEN;
365 txq->tx_comp_ring.gen = VMXNET3_TXCF_GEN;
370 static const u8 vmxnet3_rss_key[VMXNET3_RSS_MAX_KEY_SZ] = {
371 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
372 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
373 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
374 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
375 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
378 static clib_error_t *
379 vmxnet3_rss_init (vlib_main_t * vm, vmxnet3_device_t * vd)
381 vmxnet3_rss_shared *rss;
382 size_t size = sizeof (*rss);
385 vd->rss = vlib_physmem_alloc_aligned_on_numa (vm, size, 512, vd->numa_node);
387 return vlib_physmem_last_error (vm);
389 clib_memset (vd->rss, 0, size);
392 VMXNET3_RSS_HASH_TYPE_IPV4 | VMXNET3_RSS_HASH_TYPE_TCP_IPV4 |
393 VMXNET3_RSS_HASH_TYPE_IPV6 | VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
394 rss->hash_func = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
395 rss->hash_key_sz = VMXNET3_RSS_MAX_KEY_SZ;
396 rss->ind_table_sz = VMXNET3_RSS_MAX_IND_TABLE_SZ;
397 clib_memcpy (rss->hash_key, vmxnet3_rss_key, VMXNET3_RSS_MAX_KEY_SZ);
398 for (i = 0; i < rss->ind_table_sz; i++)
399 rss->ind_table[i] = i % vd->num_rx_queues;
404 static clib_error_t *
405 vmxnet3_device_init (vlib_main_t * vm, vmxnet3_device_t * vd,
406 vmxnet3_create_if_args_t * args)
408 vnet_main_t *vnm = vnet_get_main ();
409 clib_error_t *error = 0;
411 vlib_thread_main_t *tm = vlib_get_thread_main ();
413 /* Quiesce the device */
414 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
415 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
418 error = clib_error_return (0, "error on quiescing device rc (%u)", ret);
422 /* Reset the device */
423 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
424 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
427 error = clib_error_return (0, "error on resetting device rc (%u)", ret);
431 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_VRRS);
432 vd->version = count_leading_zeros (ret);
433 vd->version = uword_bits - vd->version;
435 if (vd->version == 0)
437 error = clib_error_return (0, "unsupported hardware version %u",
442 /* cap support version to 3 */
443 vmxnet3_reg_write (vd, 1, VMXNET3_REG_VRRS,
444 1 << (clib_min (3, vd->version) - 1));
446 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_UVRS);
448 vmxnet3_reg_write (vd, 1, VMXNET3_REG_UVRS, 1);
451 error = clib_error_return (0, "unsupported upt version %u", ret);
455 /* GSO is only supported for version >= 3 */
456 if (args->enable_gso && (vd->version >= 3))
459 vnm->interface_main.gso_interface_count++;
462 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
463 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
466 vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
467 vd->link_speed = ret >> 16;
470 vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
472 /* Get the mac address */
473 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACL);
474 clib_memcpy (vd->mac_addr, &ret, 4);
475 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACH);
476 clib_memcpy (vd->mac_addr + 4, &ret, 2);
478 size = sizeof (vmxnet3_rx_queue) * vd->num_rx_queues +
479 sizeof (vmxnet3_tx_queue) * vd->num_tx_queues;
482 vlib_physmem_alloc_aligned_on_numa (vm, size, 512, vd->numa_node);
484 return vlib_physmem_last_error (vm);
486 clib_memset (vd->queues, 0, size);
488 if (vd->num_rx_queues > 1)
490 error = vmxnet3_rss_init (vm, vd);
495 for (i = 0; i < vd->num_rx_queues; i++)
497 error = vmxnet3_rxq_init (vm, vd, i, args->rxq_size);
502 for (i = 0; i < tm->n_vlib_mains; i++)
504 error = vmxnet3_txq_init (vm, vd, i, args->txq_size);
509 error = vmxnet3_provision_driver_shared (vm, vd);
513 vmxnet3_write_mac (vd);
515 /* Activate device */
516 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV);
517 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
521 clib_error_return (0, "error on activating device rc (%u)", ret);
529 vmxnet3_rxq_irq_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
531 vnet_main_t *vnm = vnet_get_main ();
532 vmxnet3_main_t *vmxm = &vmxnet3_main;
533 uword pd = vlib_pci_get_private_data (vm, h);
534 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
537 if (vec_len (vd->rxqs) > qid && vd->rxqs[qid].int_mode != 0)
538 vnet_device_input_set_interrupt_pending (vnm, vd->hw_if_index, qid);
542 vmxnet3_event_irq_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h,
545 vnet_main_t *vnm = vnet_get_main ();
546 vmxnet3_main_t *vmxm = &vmxnet3_main;
547 uword pd = vlib_pci_get_private_data (vm, h);
548 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
551 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
552 ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
555 vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
556 vd->link_speed = ret >> 16;
557 vnet_hw_interface_set_link_speed (vnm, vd->hw_if_index,
558 vd->link_speed * 1000);
559 vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
560 VNET_HW_INTERFACE_FLAG_LINK_UP);
564 vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
565 vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
570 vmxnet3_queue_size_valid (u16 qsz)
572 if (qsz < 64 || qsz > 4096)
580 vmxnet3_tx_queue_num_valid (u16 num)
582 vlib_thread_main_t *tm = vlib_get_thread_main ();
584 if ((num > VMXNET3_TXQ_MAX) || (num > tm->n_vlib_mains))
590 vmxnet3_rx_queue_num_valid (u16 num)
592 if (num > VMXNET3_RXQ_MAX)
598 vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args)
600 vnet_main_t *vnm = vnet_get_main ();
601 vmxnet3_main_t *vmxm = &vmxnet3_main;
602 vmxnet3_device_t *vd;
603 vlib_pci_dev_handle_t h;
604 clib_error_t *error = 0;
608 if (args->txq_num == 0)
610 if (args->rxq_num == 0)
612 if (!vmxnet3_rx_queue_num_valid (args->rxq_num))
614 args->rv = VNET_API_ERROR_INVALID_VALUE;
616 clib_error_return (error, "number of rx queues must be <= %u",
618 vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
619 format_vlib_pci_addr, &args->addr,
620 "number of rx queues must be <= %u", VMXNET3_RXQ_MAX);
624 if (!vmxnet3_tx_queue_num_valid (args->txq_num))
626 args->rv = VNET_API_ERROR_INVALID_VALUE;
628 clib_error_return (error,
629 "number of tx queues must be <= %u and <= number of "
630 "CPU's assigned to VPP", VMXNET3_TXQ_MAX);
631 vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
632 format_vlib_pci_addr, &args->addr,
633 "number of tx queues must be <= %u and <= number of "
634 "CPU's assigned to VPP", VMXNET3_TXQ_MAX);
637 if (args->rxq_size == 0)
638 args->rxq_size = VMXNET3_NUM_RX_DESC;
639 if (args->txq_size == 0)
640 args->txq_size = VMXNET3_NUM_TX_DESC;
642 if (!vmxnet3_queue_size_valid (args->rxq_size) ||
643 !vmxnet3_queue_size_valid (args->txq_size))
645 args->rv = VNET_API_ERROR_INVALID_VALUE;
647 clib_error_return (error,
648 "queue size must be <= 4096, >= 64, "
649 "and multiples of 64");
650 vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
651 format_vlib_pci_addr, &args->addr,
652 "queue size must be <= 4096, >= 64, and multiples of 64");
657 pool_foreach (vd, vmxm->devices, ({
658 if (vd->pci_addr.as_u32 == args->addr.as_u32)
660 args->rv = VNET_API_ERROR_ADDRESS_IN_USE;
662 clib_error_return (error, "%U: %s", format_vlib_pci_addr,
663 &args->addr, "pci address in use");
664 vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
665 format_vlib_pci_addr, &args->addr, "pci address in use");
673 error = vlib_pci_bind_to_uio (vm, &args->addr, (char *) "auto");
676 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
678 clib_error_return (error, "%U: %s", format_vlib_pci_addr,
680 "error encountered on binding pci device");
681 vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
682 format_vlib_pci_addr, &args->addr,
683 "error encountered on binding pci devicee");
689 vlib_pci_device_open (vm, &args->addr, vmxnet3_pci_device_ids, &h)))
691 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
693 clib_error_return (error, "%U: %s", format_vlib_pci_addr,
695 "error encountered on pci device open");
696 vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
697 format_vlib_pci_addr, &args->addr,
698 "error encountered on pci device open");
703 * Do not use vmxnet3_log_error prior to this line since the macro
704 * references vd->pci_dev_handle
706 pool_get (vmxm->devices, vd);
707 vd->num_tx_queues = args->txq_num;
708 vd->num_rx_queues = args->rxq_num;
709 vd->dev_instance = vd - vmxm->devices;
710 vd->per_interface_next_index = ~0;
711 vd->pci_addr = args->addr;
713 if (args->enable_elog)
714 vd->flags |= VMXNET3_DEVICE_F_ELOG;
716 vd->pci_dev_handle = h;
717 vd->numa_node = vlib_pci_get_numa_node (vm, h);
718 vd->num_intrs = vd->num_rx_queues + 1; // +1 for the event interrupt
720 vlib_pci_set_private_data (vm, h, vd->dev_instance);
722 if ((error = vlib_pci_bus_master_enable (vm, h)))
724 vmxnet3_log_error (vd, "error encountered on pci bus master enable");
728 if ((error = vlib_pci_map_region (vm, h, 0, (void **) &vd->bar[0])))
730 vmxnet3_log_error (vd, "error encountered on pci map region for bar 0");
734 if ((error = vlib_pci_map_region (vm, h, 1, (void **) &vd->bar[1])))
736 vmxnet3_log_error (vd, "error encountered on pci map region for bar 1");
740 num_intr = vlib_pci_get_num_msix_interrupts (vm, h);
741 if (num_intr < vd->num_rx_queues + 1)
743 vmxnet3_log_error (vd,
744 "No sufficient interrupt lines (%u) for rx queues",
748 if ((error = vlib_pci_register_msix_handler (vm, h, 0, vd->num_rx_queues,
749 &vmxnet3_rxq_irq_handler)))
751 vmxnet3_log_error (vd,
752 "error encountered on pci register msix handler 0");
756 if ((error = vlib_pci_register_msix_handler (vm, h, vd->num_rx_queues, 1,
757 &vmxnet3_event_irq_handler)))
759 vmxnet3_log_error (vd,
760 "error encountered on pci register msix handler 1");
764 if ((error = vlib_pci_enable_msix_irq (vm, h, 0, vd->num_rx_queues + 1)))
766 vmxnet3_log_error (vd, "error encountered on pci enable msix irq");
770 if ((error = vlib_pci_intr_enable (vm, h)))
772 vmxnet3_log_error (vd, "error encountered on pci interrupt enable");
776 if ((error = vmxnet3_device_init (vm, vd, args)))
778 vmxnet3_log_error (vd, "error encountered on device init");
782 /* create interface */
783 error = ethernet_register_interface (vnm, vmxnet3_device_class.index,
784 vd->dev_instance, vd->mac_addr,
785 &vd->hw_if_index, vmxnet3_flag_change);
789 vmxnet3_log_error (vd,
790 "error encountered on ethernet register interface");
794 vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vd->hw_if_index);
795 vd->sw_if_index = sw->sw_if_index;
796 args->sw_if_index = sw->sw_if_index;
798 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vd->hw_if_index);
799 hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
801 hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO;
803 vnet_hw_interface_set_input_node (vnm, vd->hw_if_index,
804 vmxnet3_input_node.index);
805 /* Disable interrupts */
806 vmxnet3_disable_interrupt (vd);
807 vec_foreach_index (qid, vd->rxqs)
809 vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
813 vnet_hw_interface_assign_rx_thread (vnm, vd->hw_if_index, qid, ~0);
814 thread_index = vnet_get_device_input_thread_index (vnm, vd->hw_if_index,
816 numa_node = vlib_mains[thread_index]->numa_node;
817 rxq->buffer_pool_index =
818 vlib_buffer_pool_get_default_for_numa (vm, numa_node);
819 vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
820 vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
822 vd->flags |= VMXNET3_DEVICE_F_INITIALIZED;
823 vmxnet3_enable_interrupt (vd);
825 vnet_hw_interface_set_link_speed (vnm, vd->hw_if_index,
826 vd->link_speed * 1000);
827 if (vd->flags & VMXNET3_DEVICE_F_LINK_UP)
828 vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
829 VNET_HW_INTERFACE_FLAG_LINK_UP);
831 vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
835 vmxnet3_delete_if (vm, vd);
836 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
841 vmxnet3_delete_if (vlib_main_t * vm, vmxnet3_device_t * vd)
843 vnet_main_t *vnm = vnet_get_main ();
844 vmxnet3_main_t *vmxm = &vmxnet3_main;
848 /* Quiesce the device */
849 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
851 /* Reset the device */
852 vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
856 vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
857 vec_foreach_index (qid, vd->rxqs)
858 vnet_hw_interface_unassign_rx_thread (vnm, vd->hw_if_index, qid);
859 ethernet_delete_interface (vnm, vd->hw_if_index);
862 vlib_pci_device_close (vm, vd->pci_dev_handle);
865 vec_foreach_index (i, vd->rxqs)
867 vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
868 u16 mask = rxq->size - 1;
871 for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
873 vmxnet3_rx_ring *ring;
875 ring = &rxq->rx_ring[rid];
876 desc_idx = (ring->consume + 1) & mask;
877 vlib_buffer_free_from_ring (vm, ring->bufs, desc_idx, rxq->size,
879 vec_free (ring->bufs);
880 vlib_physmem_free (vm, rxq->rx_desc[rid]);
882 vlib_physmem_free (vm, rxq->rx_comp);
886 vec_free (vd->rx_stats);
889 vec_foreach_index (i, vd->txqs)
891 vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, i);
892 u16 mask = txq->size - 1;
895 desc_idx = txq->tx_ring.consume;
896 end_idx = txq->tx_ring.produce;
897 while (desc_idx != end_idx)
899 bi = txq->tx_ring.bufs[desc_idx];
900 vlib_buffer_free_no_next (vm, &bi, 1);
904 clib_spinlock_free (&txq->lock);
905 vec_free (txq->tx_ring.bufs);
906 vlib_physmem_free (vm, txq->tx_desc);
907 vlib_physmem_free (vm, txq->tx_comp);
911 vec_free (vd->tx_stats);
913 vlib_physmem_free (vm, vd->driver_shared);
914 vlib_physmem_free (vm, vd->queues);
915 vlib_physmem_free (vm, vd->rss);
917 clib_error_free (vd->error);
918 clib_memset (vd, 0, sizeof (*vd));
919 pool_put (vmxm->devices, vd);
922 vnm->interface_main.gso_interface_count--;
926 * fd.io coding-style-patch-verification: ON
929 * eval: (c-set-style "gnu")