drivers/net/enic/enic_main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7
   8 #include <sys/stat.h>
   9 #include <sys/mman.h>
  10 #include <fcntl.h>
  11 #include <libgen.h>
  12
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_string_fns.h>
  19 #include <rte_ethdev_driver.h>
  20
  21 #include "enic_compat.h"
  22 #include "enic.h"
  23 #include "wq_enet_desc.h"
  24 #include "rq_enet_desc.h"
  25 #include "cq_enet_desc.h"
  26 #include "vnic_enet.h"
  27 #include "vnic_dev.h"
  28 #include "vnic_wq.h"
  29 #include "vnic_rq.h"
  30 #include "vnic_cq.h"
  31 #include "vnic_intr.h"
  32 #include "vnic_nic.h"
  33
  34 static inline int enic_is_sriov_vf(struct enic *enic)
  35 {
  36         return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
  37 }
  38
  39 static int is_zero_addr(uint8_t *addr)
  40 {
  41         return !(addr[0] |  addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
  42 }
  43
  44 static int is_mcast_addr(uint8_t *addr)
  45 {
  46         return addr[0] & 1;
  47 }
  48
  49 static int is_eth_addr_valid(uint8_t *addr)
  50 {
  51         return !is_mcast_addr(addr) && !is_zero_addr(addr);
  52 }
  53
  54 static void
  55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
  56 {
  57         uint16_t i;
  58
  59         if (!rq || !rq->mbuf_ring) {
  60                 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
  61                 return;
  62         }
  63
  64         for (i = 0; i < rq->ring.desc_count; i++) {
  65                 if (rq->mbuf_ring[i]) {
  66                         rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
  67                         rq->mbuf_ring[i] = NULL;
  68                 }
  69         }
  70 }
  71
  72 static void enic_free_wq_buf(struct rte_mbuf **buf)
  73 {
  74         struct rte_mbuf *mbuf = *buf;
  75
  76         rte_pktmbuf_free_seg(mbuf);
  77         *buf = NULL;
  78 }
  79
  80 static void enic_log_q_error(struct enic *enic)
  81 {
  82         unsigned int i;
  83         u32 error_status;
  84
  85         for (i = 0; i < enic->wq_count; i++) {
  86                 error_status = vnic_wq_error_status(&enic->wq[i]);
  87                 if (error_status)
  88                         dev_err(enic, "WQ[%d] error_status %d\n", i,
  89                                 error_status);
  90         }
  91
  92         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
  93                 if (!enic->rq[i].in_use)
  94                         continue;
  95                 error_status = vnic_rq_error_status(&enic->rq[i]);
  96                 if (error_status)
  97                         dev_err(enic, "RQ[%d] error_status %d\n", i,
  98                                 error_status);
  99         }
 100 }
 101
 102 static void enic_clear_soft_stats(struct enic *enic)
 103 {
 104         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 105         rte_atomic64_clear(&soft_stats->rx_nombuf);
 106         rte_atomic64_clear(&soft_stats->rx_packet_errors);
 107         rte_atomic64_clear(&soft_stats->tx_oversized);
 108 }
 109
 110 static void enic_init_soft_stats(struct enic *enic)
 111 {
 112         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 113         rte_atomic64_init(&soft_stats->rx_nombuf);
 114         rte_atomic64_init(&soft_stats->rx_packet_errors);
 115         rte_atomic64_init(&soft_stats->tx_oversized);
 116         enic_clear_soft_stats(enic);
 117 }
 118
 119 void enic_dev_stats_clear(struct enic *enic)
 120 {
 121         if (vnic_dev_stats_clear(enic->vdev))
 122                 dev_err(enic, "Error in clearing stats\n");
 123         enic_clear_soft_stats(enic);
 124 }
 125
 126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
 127 {
 128         struct vnic_stats *stats;
 129         struct enic_soft_stats *soft_stats = &enic->soft_stats;
 130         int64_t rx_truncated;
 131         uint64_t rx_packet_errors;
 132         int ret = vnic_dev_stats_dump(enic->vdev, &stats);
 133
 134         if (ret) {
 135                 dev_err(enic, "Error in getting stats\n");
 136                 return ret;
 137         }
 138
 139         /* The number of truncated packets can only be calculated by
 140          * subtracting a hardware counter from error packets received by
 141          * the driver. Note: this causes transient inaccuracies in the
 142          * ipackets count. Also, the length of truncated packets are
 143          * counted in ibytes even though truncated packets are dropped
 144          * which can make ibytes be slightly higher than it should be.
 145          */
 146         rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
 147         rx_truncated = rx_packet_errors - stats->rx.rx_errors;
 148
 149         r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
 150         r_stats->opackets = stats->tx.tx_frames_ok;
 151
 152         r_stats->ibytes = stats->rx.rx_bytes_ok;
 153         r_stats->obytes = stats->tx.tx_bytes_ok;
 154
 155         r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
 156         r_stats->oerrors = stats->tx.tx_errors
 157                            + rte_atomic64_read(&soft_stats->tx_oversized);
 158
 159         r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
 160
 161         r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
 162         return 0;
 163 }
 164
 165 int enic_del_mac_address(struct enic *enic, int mac_index)
 166 {
 167         struct rte_eth_dev *eth_dev = enic->rte_dev;
 168         uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
 169
 170         return vnic_dev_del_addr(enic->vdev, mac_addr);
 171 }
 172
 173 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
 174 {
 175         int err;
 176
 177         if (!is_eth_addr_valid(mac_addr)) {
 178                 dev_err(enic, "invalid mac address\n");
 179                 return -EINVAL;
 180         }
 181
 182         err = vnic_dev_add_addr(enic->vdev, mac_addr);
 183         if (err)
 184                 dev_err(enic, "add mac addr failed\n");
 185         return err;
 186 }
 187
 188 static void
 189 enic_free_rq_buf(struct rte_mbuf **mbuf)
 190 {
 191         if (*mbuf == NULL)
 192                 return;
 193
 194         rte_pktmbuf_free(*mbuf);
 195         *mbuf = NULL;
 196 }
 197
 198 void enic_init_vnic_resources(struct enic *enic)
 199 {
 200         unsigned int error_interrupt_enable = 1;
 201         unsigned int error_interrupt_offset = 0;
 202         unsigned int rxq_interrupt_enable = 0;
 203         unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
 204         unsigned int index = 0;
 205         unsigned int cq_idx;
 206         struct vnic_rq *data_rq;
 207
 208         if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
 209                 rxq_interrupt_enable = 1;
 210
 211         for (index = 0; index < enic->rq_count; index++) {
 212                 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
 213
 214                 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
 215                         cq_idx,
 216                         error_interrupt_enable,
 217                         error_interrupt_offset);
 218
 219                 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
 220                 if (data_rq->in_use)
 221                         vnic_rq_init(data_rq,
 222                                      cq_idx,
 223                                      error_interrupt_enable,
 224                                      error_interrupt_offset);
 225
 226                 vnic_cq_init(&enic->cq[cq_idx],
 227                         0 /* flow_control_enable */,
 228                         1 /* color_enable */,
 229                         0 /* cq_head */,
 230                         0 /* cq_tail */,
 231                         1 /* cq_tail_color */,
 232                         rxq_interrupt_enable,
 233                         1 /* cq_entry_enable */,
 234                         0 /* cq_message_enable */,
 235                         rxq_interrupt_offset,
 236                         0 /* cq_message_addr */);
 237                 if (rxq_interrupt_enable)
 238                         rxq_interrupt_offset++;
 239         }
 240
 241         for (index = 0; index < enic->wq_count; index++) {
 242                 vnic_wq_init(&enic->wq[index],
 243                         enic_cq_wq(enic, index),
 244                         error_interrupt_enable,
 245                         error_interrupt_offset);
 246                 /* Compute unsupported ol flags for enic_prep_pkts() */
 247                 enic->wq[index].tx_offload_notsup_mask =
 248                         PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
 249
 250                 cq_idx = enic_cq_wq(enic, index);
 251                 vnic_cq_init(&enic->cq[cq_idx],
 252                         0 /* flow_control_enable */,
 253                         1 /* color_enable */,
 254                         0 /* cq_head */,
 255                         0 /* cq_tail */,
 256                         1 /* cq_tail_color */,
 257                         0 /* interrupt_enable */,
 258                         0 /* cq_entry_enable */,
 259                         1 /* cq_message_enable */,
 260                         0 /* interrupt offset */,
 261                         (u64)enic->wq[index].cqmsg_rz->iova);
 262         }
 263
 264         for (index = 0; index < enic->intr_count; index++) {
 265                 vnic_intr_init(&enic->intr[index],
 266                                enic->config.intr_timer_usec,
 267                                enic->config.intr_timer_type,
 268                                /*mask_on_assertion*/1);
 269         }
 270 }
 271
 272
 273 static int
 274 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 275 {
 276         struct rte_mbuf *mb;
 277         struct rq_enet_desc *rqd = rq->ring.descs;
 278         unsigned i;
 279         dma_addr_t dma_addr;
 280         uint32_t max_rx_pkt_len;
 281         uint16_t rq_buf_len;
 282
 283         if (!rq->in_use)
 284                 return 0;
 285
 286         dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
 287                   rq->ring.desc_count);
 288
 289         /*
 290          * If *not* using scatter and the mbuf size is greater than the
 291          * requested max packet size (max_rx_pkt_len), then reduce the
 292          * posted buffer size to max_rx_pkt_len. HW still receives packets
 293          * larger than max_rx_pkt_len, but they will be truncated, which we
 294          * drop in the rx handler. Not ideal, but better than returning
 295          * large packets when the user is not expecting them.
 296          */
 297         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 298         rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
 299         if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
 300                 rq_buf_len = max_rx_pkt_len;
 301         for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
 302                 mb = rte_mbuf_raw_alloc(rq->mp);
 303                 if (mb == NULL) {
 304                         dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
 305                         (unsigned)rq->index);
 306                         return -ENOMEM;
 307                 }
 308
 309                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 310                 dma_addr = (dma_addr_t)(mb->buf_iova
 311                            + RTE_PKTMBUF_HEADROOM);
 312                 rq_enet_desc_enc(rqd, dma_addr,
 313                                 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
 314                                 : RQ_ENET_TYPE_NOT_SOP),
 315                                 rq_buf_len);
 316                 rq->mbuf_ring[i] = mb;
 317         }
 318         /*
 319          * Do not post the buffers to the NIC until we enable the RQ via
 320          * enic_start_rq().
 321          */
 322         rq->need_initial_post = true;
 323         /* Initialize fetch index while RQ is disabled */
 324         iowrite32(0, &rq->ctrl->fetch_index);
 325         return 0;
 326 }
 327
 328 /*
 329  * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
 330  * allocated the buffers and filled the RQ descriptor ring. Just need to push
 331  * the post index to the NIC.
 332  */
 333 static void
 334 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
 335 {
 336         if (!rq->in_use || !rq->need_initial_post)
 337                 return;
 338
 339         /* make sure all prior writes are complete before doing the PIO write */
 340         rte_rmb();
 341
 342         /* Post all but the last buffer to VIC. */
 343         rq->posted_index = rq->ring.desc_count - 1;
 344
 345         rq->rx_nb_hold = 0;
 346
 347         dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 348                 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 349         iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 350         rte_rmb();
 351         rq->need_initial_post = false;
 352 }
 353
 354 static void *
 355 enic_alloc_consistent(void *priv, size_t size,
 356         dma_addr_t *dma_handle, u8 *name)
 357 {
 358         void *vaddr;
 359         const struct rte_memzone *rz;
 360         *dma_handle = 0;
 361         struct enic *enic = (struct enic *)priv;
 362         struct enic_memzone_entry *mze;
 363
 364         rz = rte_memzone_reserve_aligned((const char *)name, size,
 365                         SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 366         if (!rz) {
 367                 pr_err("%s : Failed to allocate memory requested for %s\n",
 368                         __func__, name);
 369                 return NULL;
 370         }
 371
 372         vaddr = rz->addr;
 373         *dma_handle = (dma_addr_t)rz->iova;
 374
 375         mze = rte_malloc("enic memzone entry",
 376                          sizeof(struct enic_memzone_entry), 0);
 377
 378         if (!mze) {
 379                 pr_err("%s : Failed to allocate memory for memzone list\n",
 380                        __func__);
 381                 rte_memzone_free(rz);
 382                 return NULL;
 383         }
 384
 385         mze->rz = rz;
 386
 387         rte_spinlock_lock(&enic->memzone_list_lock);
 388         LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
 389         rte_spinlock_unlock(&enic->memzone_list_lock);
 390
 391         return vaddr;
 392 }
 393
 394 static void
 395 enic_free_consistent(void *priv,
 396                      __rte_unused size_t size,
 397                      void *vaddr,
 398                      dma_addr_t dma_handle)
 399 {
 400         struct enic_memzone_entry *mze;
 401         struct enic *enic = (struct enic *)priv;
 402
 403         rte_spinlock_lock(&enic->memzone_list_lock);
 404         LIST_FOREACH(mze, &enic->memzone_list, entries) {
 405                 if (mze->rz->addr == vaddr &&
 406                     mze->rz->iova == dma_handle)
 407                         break;
 408         }
 409         if (mze == NULL) {
 410                 rte_spinlock_unlock(&enic->memzone_list_lock);
 411                 dev_warning(enic,
 412                             "Tried to free memory, but couldn't find it in the memzone list\n");
 413                 return;
 414         }
 415         LIST_REMOVE(mze, entries);
 416         rte_spinlock_unlock(&enic->memzone_list_lock);
 417         rte_memzone_free(mze->rz);
 418         rte_free(mze);
 419 }
 420
 421 int enic_link_update(struct enic *enic)
 422 {
 423         struct rte_eth_dev *eth_dev = enic->rte_dev;
 424         struct rte_eth_link link;
 425
 426         memset(&link, 0, sizeof(link));
 427         link.link_status = enic_get_link_status(enic);
 428         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 429         link.link_speed = vnic_dev_port_speed(enic->vdev);
 430
 431         return rte_eth_linkstatus_set(eth_dev, &link);
 432 }
 433
 434 static void
 435 enic_intr_handler(void *arg)
 436 {
 437         struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
 438         struct enic *enic = pmd_priv(dev);
 439
 440         vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 441
 442         enic_link_update(enic);
 443         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 444         enic_log_q_error(enic);
 445 }
 446
 447 static int enic_rxq_intr_init(struct enic *enic)
 448 {
 449         struct rte_intr_handle *intr_handle;
 450         uint32_t rxq_intr_count, i;
 451         int err;
 452
 453         intr_handle = enic->rte_dev->intr_handle;
 454         if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
 455                 return 0;
 456         /*
 457          * Rx queue interrupts only work when we have MSI-X interrupts,
 458          * one per queue. Sharing one interrupt is technically
 459          * possible with VIC, but it is not worth the complications it brings.
 460          */
 461         if (!rte_intr_cap_multiple(intr_handle)) {
 462                 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
 463                         " (vfio-pci driver)\n");
 464                 return -ENOTSUP;
 465         }
 466         rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
 467         err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
 468         if (err) {
 469                 dev_err(enic, "Failed to enable event fds for Rx queue"
 470                         " interrupts\n");
 471                 return err;
 472         }
 473         intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
 474                                             rxq_intr_count * sizeof(int), 0);
 475         if (intr_handle->intr_vec == NULL) {
 476                 dev_err(enic, "Failed to allocate intr_vec\n");
 477                 return -ENOMEM;
 478         }
 479         for (i = 0; i < rxq_intr_count; i++)
 480                 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
 481         return 0;
 482 }
 483
 484 static void enic_rxq_intr_deinit(struct enic *enic)
 485 {
 486         struct rte_intr_handle *intr_handle;
 487
 488         intr_handle = enic->rte_dev->intr_handle;
 489         rte_intr_efd_disable(intr_handle);
 490         if (intr_handle->intr_vec != NULL) {
 491                 rte_free(intr_handle->intr_vec);
 492                 intr_handle->intr_vec = NULL;
 493         }
 494 }
 495
 496 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
 497 {
 498         struct wq_enet_desc *desc;
 499         struct vnic_wq *wq;
 500         unsigned int i;
 501
 502         /*
 503          * Fill WQ descriptor fields that never change. Every descriptor is
 504          * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
 505          * descriptors (i.e. request one completion update every 32 packets).
 506          */
 507         wq = &enic->wq[queue_idx];
 508         desc = (struct wq_enet_desc *)wq->ring.descs;
 509         for (i = 0; i < wq->ring.desc_count; i++, desc++) {
 510                 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
 511                 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
 512                         desc->header_length_flags |=
 513                                 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
 514         }
 515 }
 516
 517 /*
 518  * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
 519  * used when that file is not compiled.
 520  */
 521 bool __attribute__((weak))
 522 enic_use_vector_rx_handler(__rte_unused struct enic *enic)
 523 {
 524         return false;
 525 }
 526
 527 static void pick_rx_handler(struct enic *enic)
 528 {
 529         struct rte_eth_dev *eth_dev;
 530
 531         /*
 532          * Preference order:
 533          * 1. The vectorized handler if possible and requested.
 534          * 2. The non-scatter, simplified handler if scatter Rx is not used.
 535          * 3. The default handler as a fallback.
 536          */
 537         eth_dev = enic->rte_dev;
 538         if (enic_use_vector_rx_handler(enic))
 539                 return;
 540         if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
 541                 PMD_INIT_LOG(DEBUG, " use the non-scatter Rx handler");
 542                 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
 543         } else {
 544                 PMD_INIT_LOG(DEBUG, " use the normal Rx handler");
 545                 eth_dev->rx_pkt_burst = &enic_recv_pkts;
 546         }
 547 }
 548
 549 int enic_enable(struct enic *enic)
 550 {
 551         unsigned int index;
 552         int err;
 553         struct rte_eth_dev *eth_dev = enic->rte_dev;
 554         uint64_t simple_tx_offloads;
 555         uintptr_t p;
 556
 557         if (enic->enable_avx2_rx) {
 558                 struct rte_mbuf mb_def = { .buf_addr = 0 };
 559
 560                 /*
 561                  * mbuf_initializer contains const-after-init fields of
 562                  * receive mbufs (i.e. 64 bits of fields from rearm_data).
 563                  * It is currently used by the vectorized handler.
 564                  */
 565                 mb_def.nb_segs = 1;
 566                 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 567                 mb_def.port = enic->port_id;
 568                 rte_mbuf_refcnt_set(&mb_def, 1);
 569                 rte_compiler_barrier();
 570                 p = (uintptr_t)&mb_def.rearm_data;
 571                 enic->mbuf_initializer = *(uint64_t *)p;
 572         }
 573
 574         eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 575         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 576
 577         /* vnic notification of link status has already been turned on in
 578          * enic_dev_init() which is called during probe time.  Here we are
 579          * just turning on interrupt vector 0 if needed.
 580          */
 581         if (eth_dev->data->dev_conf.intr_conf.lsc)
 582                 vnic_dev_notify_set(enic->vdev, 0);
 583
 584         err = enic_rxq_intr_init(enic);
 585         if (err)
 586                 return err;
 587         if (enic_clsf_init(enic))
 588                 dev_warning(enic, "Init of hash table for clsf failed."\
 589                         "Flow director feature will not work\n");
 590
 591         for (index = 0; index < enic->rq_count; index++) {
 592                 err = enic_alloc_rx_queue_mbufs(enic,
 593                         &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 594                 if (err) {
 595                         dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
 596                         return err;
 597                 }
 598                 err = enic_alloc_rx_queue_mbufs(enic,
 599                         &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
 600                 if (err) {
 601                         /* release the allocated mbufs for the sop rq*/
 602                         enic_rxmbuf_queue_release(enic,
 603                                 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
 604
 605                         dev_err(enic, "Failed to alloc data RX queue mbufs\n");
 606                         return err;
 607                 }
 608         }
 609
 610         /*
 611          * Use the simple TX handler if possible. Only checksum offloads
 612          * and vlan insertion are supported.
 613          */
 614         simple_tx_offloads = enic->tx_offload_capa &
 615                 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 616                  DEV_TX_OFFLOAD_VLAN_INSERT |
 617                  DEV_TX_OFFLOAD_IPV4_CKSUM |
 618                  DEV_TX_OFFLOAD_UDP_CKSUM |
 619                  DEV_TX_OFFLOAD_TCP_CKSUM);
 620         if ((eth_dev->data->dev_conf.txmode.offloads &
 621              ~simple_tx_offloads) == 0) {
 622                 PMD_INIT_LOG(DEBUG, " use the simple tx handler");
 623                 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 624                 for (index = 0; index < enic->wq_count; index++)
 625                         enic_prep_wq_for_simple_tx(enic, index);
 626         } else {
 627                 PMD_INIT_LOG(DEBUG, " use the default tx handler");
 628                 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
 629         }
 630
 631         pick_rx_handler(enic);
 632
 633         for (index = 0; index < enic->wq_count; index++)
 634                 enic_start_wq(enic, index);
 635         for (index = 0; index < enic->rq_count; index++)
 636                 enic_start_rq(enic, index);
 637
 638         vnic_dev_add_addr(enic->vdev, enic->mac_addr);
 639
 640         vnic_dev_enable_wait(enic->vdev);
 641
 642         /* Register and enable error interrupt */
 643         rte_intr_callback_register(&(enic->pdev->intr_handle),
 644                 enic_intr_handler, (void *)enic->rte_dev);
 645
 646         rte_intr_enable(&(enic->pdev->intr_handle));
 647         /* Unmask LSC interrupt */
 648         vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
 649
 650         return 0;
 651 }
 652
 653 int enic_alloc_intr_resources(struct enic *enic)
 654 {
 655         int err;
 656         unsigned int i;
 657
 658         dev_info(enic, "vNIC resources used:  "\
 659                 "wq %d rq %d cq %d intr %d\n",
 660                 enic->wq_count, enic_vnic_rq_count(enic),
 661                 enic->cq_count, enic->intr_count);
 662
 663         for (i = 0; i < enic->intr_count; i++) {
 664                 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
 665                 if (err) {
 666                         enic_free_vnic_resources(enic);
 667                         return err;
 668                 }
 669         }
 670         return 0;
 671 }
 672
 673 void enic_free_rq(void *rxq)
 674 {
 675         struct vnic_rq *rq_sop, *rq_data;
 676         struct enic *enic;
 677
 678         if (rxq == NULL)
 679                 return;
 680
 681         rq_sop = (struct vnic_rq *)rxq;
 682         enic = vnic_dev_priv(rq_sop->vdev);
 683         rq_data = &enic->rq[rq_sop->data_queue_idx];
 684
 685         if (rq_sop->free_mbufs) {
 686                 struct rte_mbuf **mb;
 687                 int i;
 688
 689                 mb = rq_sop->free_mbufs;
 690                 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
 691                      i < ENIC_RX_BURST_MAX; i++)
 692                         rte_pktmbuf_free(mb[i]);
 693                 rte_free(rq_sop->free_mbufs);
 694                 rq_sop->free_mbufs = NULL;
 695                 rq_sop->num_free_mbufs = 0;
 696         }
 697
 698         enic_rxmbuf_queue_release(enic, rq_sop);
 699         if (rq_data->in_use)
 700                 enic_rxmbuf_queue_release(enic, rq_data);
 701
 702         rte_free(rq_sop->mbuf_ring);
 703         if (rq_data->in_use)
 704                 rte_free(rq_data->mbuf_ring);
 705
 706         rq_sop->mbuf_ring = NULL;
 707         rq_data->mbuf_ring = NULL;
 708
 709         vnic_rq_free(rq_sop);
 710         if (rq_data->in_use)
 711                 vnic_rq_free(rq_data);
 712
 713         vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
 714
 715         rq_sop->in_use = 0;
 716         rq_data->in_use = 0;
 717 }
 718
 719 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
 720 {
 721         struct rte_eth_dev *eth_dev = enic->rte_dev;
 722         vnic_wq_enable(&enic->wq[queue_idx]);
 723         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 724 }
 725
 726 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 727 {
 728         struct rte_eth_dev *eth_dev = enic->rte_dev;
 729         int ret;
 730
 731         ret = vnic_wq_disable(&enic->wq[queue_idx]);
 732         if (ret)
 733                 return ret;
 734
 735         eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 736         return 0;
 737 }
 738
 739 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
 740 {
 741         struct vnic_rq *rq_sop;
 742         struct vnic_rq *rq_data;
 743         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 744         rq_data = &enic->rq[rq_sop->data_queue_idx];
 745         struct rte_eth_dev *eth_dev = enic->rte_dev;
 746
 747         if (rq_data->in_use) {
 748                 vnic_rq_enable(rq_data);
 749                 enic_initial_post_rx(enic, rq_data);
 750         }
 751         rte_mb();
 752         vnic_rq_enable(rq_sop);
 753         enic_initial_post_rx(enic, rq_sop);
 754         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
 755 }
 756
 757 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
 758 {
 759         int ret1 = 0, ret2 = 0;
 760         struct rte_eth_dev *eth_dev = enic->rte_dev;
 761         struct vnic_rq *rq_sop;
 762         struct vnic_rq *rq_data;
 763         rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 764         rq_data = &enic->rq[rq_sop->data_queue_idx];
 765
 766         ret2 = vnic_rq_disable(rq_sop);
 767         rte_mb();
 768         if (rq_data->in_use)
 769                 ret1 = vnic_rq_disable(rq_data);
 770
 771         if (ret2)
 772                 return ret2;
 773         else if (ret1)
 774                 return ret1;
 775
 776         eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 777         return 0;
 778 }
 779
 780 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 781         unsigned int socket_id, struct rte_mempool *mp,
 782         uint16_t nb_desc, uint16_t free_thresh)
 783 {
 784         int rc;
 785         uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
 786         uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
 787         struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
 788         struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
 789         unsigned int mbuf_size, mbufs_per_pkt;
 790         unsigned int nb_sop_desc, nb_data_desc;
 791         uint16_t min_sop, max_sop, min_data, max_data;
 792         uint32_t max_rx_pkt_len;
 793
 794         rq_sop->is_sop = 1;
 795         rq_sop->data_queue_idx = data_queue_idx;
 796         rq_data->is_sop = 0;
 797         rq_data->data_queue_idx = 0;
 798         rq_sop->socket_id = socket_id;
 799         rq_sop->mp = mp;
 800         rq_data->socket_id = socket_id;
 801         rq_data->mp = mp;
 802         rq_sop->in_use = 1;
 803         rq_sop->rx_free_thresh = free_thresh;
 804         rq_data->rx_free_thresh = free_thresh;
 805         dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
 806                   free_thresh);
 807
 808         mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
 809                                RTE_PKTMBUF_HEADROOM);
 810         /* max_rx_pkt_len includes the ethernet header and CRC. */
 811         max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 812
 813         if (enic->rte_dev->data->dev_conf.rxmode.offloads &
 814             DEV_RX_OFFLOAD_SCATTER) {
 815                 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
 816                 /* ceil((max pkt len)/mbuf_size) */
 817                 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
 818         } else {
 819                 dev_info(enic, "Scatter rx mode disabled\n");
 820                 mbufs_per_pkt = 1;
 821                 if (max_rx_pkt_len > mbuf_size) {
 822                         dev_warning(enic, "The maximum Rx packet size (%u) is"
 823                                     " larger than the mbuf size (%u), and"
 824                                     " scatter is disabled. Larger packets will"
 825                                     " be truncated.\n",
 826                                     max_rx_pkt_len, mbuf_size);
 827                 }
 828         }
 829
 830         if (mbufs_per_pkt > 1) {
 831                 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
 832                 rq_sop->data_queue_enable = 1;
 833                 rq_data->in_use = 1;
 834                 /*
 835                  * HW does not directly support rxmode.max_rx_pkt_len. HW always
 836                  * receives packet sizes up to the "max" MTU.
 837                  * If not using scatter, we can achieve the effect of dropping
 838                  * larger packets by reducing the size of posted buffers.
 839                  * See enic_alloc_rx_queue_mbufs().
 840                  */
 841                 if (max_rx_pkt_len <
 842                     enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
 843                         dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
 844                                     " when scatter rx mode is in use.\n");
 845                 }
 846         } else {
 847                 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
 848                          queue_idx);
 849                 rq_sop->data_queue_enable = 0;
 850                 rq_data->in_use = 0;
 851         }
 852
 853         /* number of descriptors have to be a multiple of 32 */
 854         nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
 855         nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
 856
 857         rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 858         rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 859
 860         if (mbufs_per_pkt > 1) {
 861                 min_sop = ENIC_RX_BURST_MAX;
 862                 max_sop = ((enic->config.rq_desc_count /
 863                             (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
 864                 min_data = min_sop * (mbufs_per_pkt - 1);
 865                 max_data = enic->config.rq_desc_count;
 866         } else {
 867                 min_sop = ENIC_RX_BURST_MAX;
 868                 max_sop = enic->config.rq_desc_count;
 869                 min_data = 0;
 870                 max_data = 0;
 871         }
 872
 873         if (nb_desc < (min_sop + min_data)) {
 874                 dev_warning(enic,
 875                             "Number of rx descs too low, adjusting to minimum\n");
 876                 nb_sop_desc = min_sop;
 877                 nb_data_desc = min_data;
 878         } else if (nb_desc > (max_sop + max_data)) {
 879                 dev_warning(enic,
 880                             "Number of rx_descs too high, adjusting to maximum\n");
 881                 nb_sop_desc = max_sop;
 882                 nb_data_desc = max_data;
 883         }
 884         if (mbufs_per_pkt > 1) {
 885                 dev_info(enic, "For max packet size %u and mbuf size %u valid"
 886                          " rx descriptor range is %u to %u\n",
 887                          max_rx_pkt_len, mbuf_size, min_sop + min_data,
 888                          max_sop + max_data);
 889         }
 890         dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
 891                  nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
 892
 893         /* Allocate sop queue resources */
 894         rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
 895                 nb_sop_desc, sizeof(struct rq_enet_desc));
 896         if (rc) {
 897                 dev_err(enic, "error in allocation of sop rq\n");
 898                 goto err_exit;
 899         }
 900         nb_sop_desc = rq_sop->ring.desc_count;
 901
 902         if (rq_data->in_use) {
 903                 /* Allocate data queue resources */
 904                 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
 905                                    nb_data_desc,
 906                                    sizeof(struct rq_enet_desc));
 907                 if (rc) {
 908                         dev_err(enic, "error in allocation of data rq\n");
 909                         goto err_free_rq_sop;
 910                 }
 911                 nb_data_desc = rq_data->ring.desc_count;
 912         }
 913         rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
 914                            socket_id, nb_sop_desc + nb_data_desc,
 915                            sizeof(struct cq_enet_rq_desc));
 916         if (rc) {
 917                 dev_err(enic, "error in allocation of cq for rq\n");
 918                 goto err_free_rq_data;
 919         }
 920
 921         /* Allocate the mbuf rings */
 922         rq_sop->mbuf_ring = (struct rte_mbuf **)
 923                 rte_zmalloc_socket("rq->mbuf_ring",
 924                                    sizeof(struct rte_mbuf *) * nb_sop_desc,
 925                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 926         if (rq_sop->mbuf_ring == NULL)
 927                 goto err_free_cq;
 928
 929         if (rq_data->in_use) {
 930                 rq_data->mbuf_ring = (struct rte_mbuf **)
 931                         rte_zmalloc_socket("rq->mbuf_ring",
 932                                 sizeof(struct rte_mbuf *) * nb_data_desc,
 933                                 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 934                 if (rq_data->mbuf_ring == NULL)
 935                         goto err_free_sop_mbuf;
 936         }
 937
 938         rq_sop->free_mbufs = (struct rte_mbuf **)
 939                 rte_zmalloc_socket("rq->free_mbufs",
 940                                    sizeof(struct rte_mbuf *) *
 941                                    ENIC_RX_BURST_MAX,
 942                                    RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
 943         if (rq_sop->free_mbufs == NULL)
 944                 goto err_free_data_mbuf;
 945         rq_sop->num_free_mbufs = 0;
 946
 947         rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 948
 949         return 0;
 950
 951 err_free_data_mbuf:
 952         rte_free(rq_data->mbuf_ring);
 953 err_free_sop_mbuf:
 954         rte_free(rq_sop->mbuf_ring);
 955 err_free_cq:
 956         /* cleanup on error */
 957         vnic_cq_free(&enic->cq[queue_idx]);
 958 err_free_rq_data:
 959         if (rq_data->in_use)
 960                 vnic_rq_free(rq_data);
 961 err_free_rq_sop:
 962         vnic_rq_free(rq_sop);
 963 err_exit:
 964         return -ENOMEM;
 965 }
 966
 967 void enic_free_wq(void *txq)
 968 {
 969         struct vnic_wq *wq;
 970         struct enic *enic;
 971
 972         if (txq == NULL)
 973                 return;
 974
 975         wq = (struct vnic_wq *)txq;
 976         enic = vnic_dev_priv(wq->vdev);
 977         rte_memzone_free(wq->cqmsg_rz);
 978         vnic_wq_free(wq);
 979         vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 980 }
 981
 982 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 983         unsigned int socket_id, uint16_t nb_desc)
 984 {
 985         int err;
 986         struct vnic_wq *wq = &enic->wq[queue_idx];
 987         unsigned int cq_index = enic_cq_wq(enic, queue_idx);
 988         char name[NAME_MAX];
 989         static int instance;
 990
 991         wq->socket_id = socket_id;
 992         /*
 993          * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
 994          * print an info message for diagnostics.
 995          */
 996         dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
 997
 998         /* Allocate queue resources */
 999         err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1000                 nb_desc,
1001                 sizeof(struct wq_enet_desc));
1002         if (err) {
1003                 dev_err(enic, "error in allocation of wq\n");
1004                 return err;
1005         }
1006
1007         err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1008                 socket_id, nb_desc,
1009                 sizeof(struct cq_enet_wq_desc));
1010         if (err) {
1011                 vnic_wq_free(wq);
1012                 dev_err(enic, "error in allocation of cq for wq\n");
1013         }
1014
1015         /* setup up CQ message */
1016         snprintf((char *)name, sizeof(name),
1017                  "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1018                 instance++);
1019
1020         wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1021                         sizeof(uint32_t), SOCKET_ID_ANY,
1022                         RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
1023         if (!wq->cqmsg_rz)
1024                 return -ENOMEM;
1025
1026         return err;
1027 }
1028
1029 int enic_disable(struct enic *enic)
1030 {
1031         unsigned int i;
1032         int err;
1033
1034         for (i = 0; i < enic->intr_count; i++) {
1035                 vnic_intr_mask(&enic->intr[i]);
1036                 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1037         }
1038         enic_rxq_intr_deinit(enic);
1039         rte_intr_disable(&enic->pdev->intr_handle);
1040         rte_intr_callback_unregister(&enic->pdev->intr_handle,
1041                                      enic_intr_handler,
1042                                      (void *)enic->rte_dev);
1043
1044         vnic_dev_disable(enic->vdev);
1045
1046         enic_clsf_destroy(enic);
1047
1048         if (!enic_is_sriov_vf(enic))
1049                 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1050
1051         for (i = 0; i < enic->wq_count; i++) {
1052                 err = vnic_wq_disable(&enic->wq[i]);
1053                 if (err)
1054                         return err;
1055         }
1056         for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1057                 if (enic->rq[i].in_use) {
1058                         err = vnic_rq_disable(&enic->rq[i]);
1059                         if (err)
1060                                 return err;
1061                 }
1062         }
1063
1064         /* If we were using interrupts, set the interrupt vector to -1
1065          * to disable interrupts.  We are not disabling link notifcations,
1066          * though, as we want the polling of link status to continue working.
1067          */
1068         if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1069                 vnic_dev_notify_set(enic->vdev, -1);
1070
1071         vnic_dev_set_reset_flag(enic->vdev, 1);
1072
1073         for (i = 0; i < enic->wq_count; i++)
1074                 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1075
1076         for (i = 0; i < enic_vnic_rq_count(enic); i++)
1077                 if (enic->rq[i].in_use)
1078                         vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1079         for (i = 0; i < enic->cq_count; i++)
1080                 vnic_cq_clean(&enic->cq[i]);
1081         for (i = 0; i < enic->intr_count; i++)
1082                 vnic_intr_clean(&enic->intr[i]);
1083
1084         return 0;
1085 }
1086
1087 static int enic_dev_wait(struct vnic_dev *vdev,
1088         int (*start)(struct vnic_dev *, int),
1089         int (*finished)(struct vnic_dev *, int *),
1090         int arg)
1091 {
1092         int done;
1093         int err;
1094         int i;
1095
1096         err = start(vdev, arg);
1097         if (err)
1098                 return err;
1099
1100         /* Wait for func to complete...2 seconds max */
1101         for (i = 0; i < 2000; i++) {
1102                 err = finished(vdev, &done);
1103                 if (err)
1104                         return err;
1105                 if (done)
1106                         return 0;
1107                 usleep(1000);
1108         }
1109         return -ETIMEDOUT;
1110 }
1111
1112 static int enic_dev_open(struct enic *enic)
1113 {
1114         int err;
1115         int flags = CMD_OPENF_IG_DESCCACHE;
1116
1117         err = enic_dev_wait(enic->vdev, vnic_dev_open,
1118                 vnic_dev_open_done, flags);
1119         if (err)
1120                 dev_err(enic_get_dev(enic),
1121                         "vNIC device open failed, err %d\n", err);
1122
1123         return err;
1124 }
1125
1126 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1127 {
1128         dma_addr_t rss_key_buf_pa;
1129         union vnic_rss_key *rss_key_buf_va = NULL;
1130         int err, i;
1131         u8 name[NAME_MAX];
1132
1133         RTE_ASSERT(user_key != NULL);
1134         snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1135         rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1136                 &rss_key_buf_pa, name);
1137         if (!rss_key_buf_va)
1138                 return -ENOMEM;
1139
1140         for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1141                 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1142
1143         err = enic_set_rss_key(enic,
1144                 rss_key_buf_pa,
1145                 sizeof(union vnic_rss_key));
1146
1147         /* Save for later queries */
1148         if (!err) {
1149                 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1150                            sizeof(union vnic_rss_key));
1151         }
1152         enic_free_consistent(enic, sizeof(union vnic_rss_key),
1153                 rss_key_buf_va, rss_key_buf_pa);
1154
1155         return err;
1156 }
1157
1158 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1159 {
1160         dma_addr_t rss_cpu_buf_pa;
1161         union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1162         int err;
1163         u8 name[NAME_MAX];
1164
1165         snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1166         rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1167                 &rss_cpu_buf_pa, name);
1168         if (!rss_cpu_buf_va)
1169                 return -ENOMEM;
1170
1171         rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1172
1173         err = enic_set_rss_cpu(enic,
1174                 rss_cpu_buf_pa,
1175                 sizeof(union vnic_rss_cpu));
1176
1177         enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1178                 rss_cpu_buf_va, rss_cpu_buf_pa);
1179
1180         /* Save for later queries */
1181         if (!err)
1182                 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1183         return err;
1184 }
1185
1186 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1187         u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1188 {
1189         const u8 tso_ipid_split_en = 0;
1190         int err;
1191
1192         err = enic_set_nic_cfg(enic,
1193                 rss_default_cpu, rss_hash_type,
1194                 rss_hash_bits, rss_base_cpu,
1195                 rss_enable, tso_ipid_split_en,
1196                 enic->ig_vlan_strip_en);
1197
1198         return err;
1199 }
1200
1201 /* Initialize RSS with defaults, called from dev_configure */
1202 int enic_init_rss_nic_cfg(struct enic *enic)
1203 {
1204         static uint8_t default_rss_key[] = {
1205                 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1206                 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1207                 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1208                 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1209         };
1210         struct rte_eth_rss_conf rss_conf;
1211         union vnic_rss_cpu rss_cpu;
1212         int ret, i;
1213
1214         rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1215         /*
1216          * If setting key for the first time, and the user gives us none, then
1217          * push the default key to NIC.
1218          */
1219         if (rss_conf.rss_key == NULL) {
1220                 rss_conf.rss_key = default_rss_key;
1221                 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1222         }
1223         ret = enic_set_rss_conf(enic, &rss_conf);
1224         if (ret) {
1225                 dev_err(enic, "Failed to configure RSS\n");
1226                 return ret;
1227         }
1228         if (enic->rss_enable) {
1229                 /* If enabling RSS, use the default reta */
1230                 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1231                         rss_cpu.cpu[i / 4].b[i % 4] =
1232                                 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1233                 }
1234                 ret = enic_set_rss_reta(enic, &rss_cpu);
1235                 if (ret)
1236                         dev_err(enic, "Failed to set RSS indirection table\n");
1237         }
1238         return ret;
1239 }
1240
1241 int enic_setup_finish(struct enic *enic)
1242 {
1243         enic_init_soft_stats(enic);
1244
1245         /* Default conf */
1246         vnic_dev_packet_filter(enic->vdev,
1247                 1 /* directed  */,
1248                 1 /* multicast */,
1249                 1 /* broadcast */,
1250                 0 /* promisc   */,
1251                 1 /* allmulti  */);
1252
1253         enic->promisc = 0;
1254         enic->allmulti = 1;
1255
1256         return 0;
1257 }
1258
1259 static int enic_rss_conf_valid(struct enic *enic,
1260                                struct rte_eth_rss_conf *rss_conf)
1261 {
1262         /* RSS is disabled per VIC settings. Ignore rss_conf. */
1263         if (enic->flow_type_rss_offloads == 0)
1264                 return 0;
1265         if (rss_conf->rss_key != NULL &&
1266             rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1267                 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1268                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1269                 return -EINVAL;
1270         }
1271         if (rss_conf->rss_hf != 0 &&
1272             (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1273                 dev_err(enic, "Given rss_hf contains none of the supported"
1274                         " types\n");
1275                 return -EINVAL;
1276         }
1277         return 0;
1278 }
1279
1280 /* Set hash type and key according to rss_conf */
1281 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1282 {
1283         struct rte_eth_dev *eth_dev;
1284         uint64_t rss_hf;
1285         u8 rss_hash_type;
1286         u8 rss_enable;
1287         int ret;
1288
1289         RTE_ASSERT(rss_conf != NULL);
1290         ret = enic_rss_conf_valid(enic, rss_conf);
1291         if (ret) {
1292                 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1293                 return ret;
1294         }
1295
1296         eth_dev = enic->rte_dev;
1297         rss_hash_type = 0;
1298         rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1299         if (enic->rq_count > 1 &&
1300             (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1301             rss_hf != 0) {
1302                 rss_enable = 1;
1303                 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1304                               ETH_RSS_NONFRAG_IPV4_OTHER))
1305                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1306                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1307                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1308                 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1309                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1310                         if (enic->udp_rss_weak) {
1311                                 /*
1312                                  * 'TCP' is not a typo. The "weak" version of
1313                                  * UDP RSS requires both the TCP and UDP bits
1314                                  * be set. It does enable TCP RSS as well.
1315                                  */
1316                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1317                         }
1318                 }
1319                 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1320                               ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1321                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1322                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1323                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1324                 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1325                         rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1326                         if (enic->udp_rss_weak)
1327                                 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1328                 }
1329         } else {
1330                 rss_enable = 0;
1331                 rss_hf = 0;
1332         }
1333
1334         /* Set the hash key if provided */
1335         if (rss_enable && rss_conf->rss_key) {
1336                 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1337                 if (ret) {
1338                         dev_err(enic, "Failed to set RSS key\n");
1339                         return ret;
1340                 }
1341         }
1342
1343         ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1344                               ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1345                               rss_enable);
1346         if (!ret) {
1347                 enic->rss_hf = rss_hf;
1348                 enic->rss_hash_type = rss_hash_type;
1349                 enic->rss_enable = rss_enable;
1350         } else {
1351                 dev_err(enic, "Failed to update RSS configurations."
1352                         " hash=0x%x\n", rss_hash_type);
1353         }
1354         return ret;
1355 }
1356
1357 int enic_set_vlan_strip(struct enic *enic)
1358 {
1359         /*
1360          * Unfortunately, VLAN strip on/off and RSS on/off are configured
1361          * together. So, re-do niccfg, preserving the current RSS settings.
1362          */
1363         return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1364                                ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1365                                enic->rss_enable);
1366 }
1367
1368 void enic_add_packet_filter(struct enic *enic)
1369 {
1370         /* Args -> directed, multicast, broadcast, promisc, allmulti */
1371         vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1372                 enic->promisc, enic->allmulti);
1373 }
1374
1375 int enic_get_link_status(struct enic *enic)
1376 {
1377         return vnic_dev_link_status(enic->vdev);
1378 }
1379
1380 static void enic_dev_deinit(struct enic *enic)
1381 {
1382         struct rte_eth_dev *eth_dev = enic->rte_dev;
1383
1384         /* stop link status checking */
1385         vnic_dev_notify_unset(enic->vdev);
1386
1387         rte_free(eth_dev->data->mac_addrs);
1388         rte_free(enic->cq);
1389         rte_free(enic->intr);
1390         rte_free(enic->rq);
1391         rte_free(enic->wq);
1392 }
1393
1394
1395 int enic_set_vnic_res(struct enic *enic)
1396 {
1397         struct rte_eth_dev *eth_dev = enic->rte_dev;
1398         int rc = 0;
1399         unsigned int required_rq, required_wq, required_cq, required_intr;
1400
1401         /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1402         required_rq = eth_dev->data->nb_rx_queues * 2;
1403         required_wq = eth_dev->data->nb_tx_queues;
1404         required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1405         required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1406         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1407                 required_intr += eth_dev->data->nb_rx_queues;
1408         }
1409
1410         if (enic->conf_rq_count < required_rq) {
1411                 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1412                         eth_dev->data->nb_rx_queues,
1413                         required_rq, enic->conf_rq_count);
1414                 rc = -EINVAL;
1415         }
1416         if (enic->conf_wq_count < required_wq) {
1417                 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1418                         eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1419                 rc = -EINVAL;
1420         }
1421
1422         if (enic->conf_cq_count < required_cq) {
1423                 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1424                         required_cq, enic->conf_cq_count);
1425                 rc = -EINVAL;
1426         }
1427         if (enic->conf_intr_count < required_intr) {
1428                 dev_err(dev, "Not enough Interrupts to support Rx queue"
1429                         " interrupts. Required:%u, Configured:%u\n",
1430                         required_intr, enic->conf_intr_count);
1431                 rc = -EINVAL;
1432         }
1433
1434         if (rc == 0) {
1435                 enic->rq_count = eth_dev->data->nb_rx_queues;
1436                 enic->wq_count = eth_dev->data->nb_tx_queues;
1437                 enic->cq_count = enic->rq_count + enic->wq_count;
1438                 enic->intr_count = required_intr;
1439         }
1440
1441         return rc;
1442 }
1443
1444 /* Initialize the completion queue for an RQ */
1445 static int
1446 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1447 {
1448         struct vnic_rq *sop_rq, *data_rq;
1449         unsigned int cq_idx;
1450         int rc = 0;
1451
1452         sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1453         data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1454         cq_idx = rq_idx;
1455
1456         vnic_cq_clean(&enic->cq[cq_idx]);
1457         vnic_cq_init(&enic->cq[cq_idx],
1458                      0 /* flow_control_enable */,
1459                      1 /* color_enable */,
1460                      0 /* cq_head */,
1461                      0 /* cq_tail */,
1462                      1 /* cq_tail_color */,
1463                      0 /* interrupt_enable */,
1464                      1 /* cq_entry_enable */,
1465                      0 /* cq_message_enable */,
1466                      0 /* interrupt offset */,
1467                      0 /* cq_message_addr */);
1468
1469
1470         vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1471                            enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1472                            sop_rq->ring.desc_count - 1, 1, 0);
1473         if (data_rq->in_use) {
1474                 vnic_rq_init_start(data_rq,
1475                                    enic_cq_rq(enic,
1476                                    enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1477                                    data_rq->ring.desc_count - 1, 1, 0);
1478         }
1479
1480         rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1481         if (rc)
1482                 return rc;
1483
1484         if (data_rq->in_use) {
1485                 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1486                 if (rc) {
1487                         enic_rxmbuf_queue_release(enic, sop_rq);
1488                         return rc;
1489                 }
1490         }
1491
1492         return 0;
1493 }
1494
1495 /* The Cisco NIC can send and receive packets up to a max packet size
1496  * determined by the NIC type and firmware. There is also an MTU
1497  * configured into the NIC via the CIMC/UCSM management interface
1498  * which can be overridden by this function (up to the max packet size).
1499  * Depending on the network setup, doing so may cause packet drops
1500  * and unexpected behavior.
1501  */
1502 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1503 {
1504         unsigned int rq_idx;
1505         struct vnic_rq *rq;
1506         int rc = 0;
1507         uint16_t old_mtu;       /* previous setting */
1508         uint16_t config_mtu;    /* Value configured into NIC via CIMC/UCSM */
1509         struct rte_eth_dev *eth_dev = enic->rte_dev;
1510
1511         old_mtu = eth_dev->data->mtu;
1512         config_mtu = enic->config.mtu;
1513
1514         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1515                 return -E_RTE_SECONDARY;
1516
1517         if (new_mtu > enic->max_mtu) {
1518                 dev_err(enic,
1519                         "MTU not updated: requested (%u) greater than max (%u)\n",
1520                         new_mtu, enic->max_mtu);
1521                 return -EINVAL;
1522         }
1523         if (new_mtu < ENIC_MIN_MTU) {
1524                 dev_info(enic,
1525                         "MTU not updated: requested (%u) less than min (%u)\n",
1526                         new_mtu, ENIC_MIN_MTU);
1527                 return -EINVAL;
1528         }
1529         if (new_mtu > config_mtu)
1530                 dev_warning(enic,
1531                         "MTU (%u) is greater than value configured in NIC (%u)\n",
1532                         new_mtu, config_mtu);
1533
1534         /* Update the MTU and maximum packet length */
1535         eth_dev->data->mtu = new_mtu;
1536         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1537                 enic_mtu_to_max_rx_pktlen(new_mtu);
1538
1539         /*
1540          * If the device has not started (enic_enable), nothing to do.
1541          * Later, enic_enable() will set up RQs reflecting the new maximum
1542          * packet length.
1543          */
1544         if (!eth_dev->data->dev_started)
1545                 goto set_mtu_done;
1546
1547         /*
1548          * The device has started, re-do RQs on the fly. In the process, we
1549          * pick up the new maximum packet length.
1550          *
1551          * Some applications rely on the ability to change MTU without stopping
1552          * the device. So keep this behavior for now.
1553          */
1554         rte_spinlock_lock(&enic->mtu_lock);
1555
1556         /* Stop traffic on all RQs */
1557         for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1558                 rq = &enic->rq[rq_idx];
1559                 if (rq->is_sop && rq->in_use) {
1560                         rc = enic_stop_rq(enic,
1561                                           enic_sop_rq_idx_to_rte_idx(rq_idx));
1562                         if (rc) {
1563                                 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1564                                 goto set_mtu_done;
1565                         }
1566                 }
1567         }
1568
1569         /* replace Rx function with a no-op to avoid getting stale pkts */
1570         eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1571         rte_mb();
1572
1573         /* Allow time for threads to exit the real Rx function. */
1574         usleep(100000);
1575
1576         /* now it is safe to reconfigure the RQs */
1577
1578
1579         /* free and reallocate RQs with the new MTU */
1580         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1581                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1582                 if (!rq->in_use)
1583                         continue;
1584
1585                 enic_free_rq(rq);
1586                 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1587                                    rq->tot_nb_desc, rq->rx_free_thresh);
1588                 if (rc) {
1589                         dev_err(enic,
1590                                 "Fatal MTU alloc error- No traffic will pass\n");
1591                         goto set_mtu_done;
1592                 }
1593
1594                 rc = enic_reinit_rq(enic, rq_idx);
1595                 if (rc) {
1596                         dev_err(enic,
1597                                 "Fatal MTU RQ reinit- No traffic will pass\n");
1598                         goto set_mtu_done;
1599                 }
1600         }
1601
1602         /* put back the real receive function */
1603         rte_mb();
1604         pick_rx_handler(enic);
1605         rte_mb();
1606
1607         /* restart Rx traffic */
1608         for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1609                 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1610                 if (rq->is_sop && rq->in_use)
1611                         enic_start_rq(enic, rq_idx);
1612         }
1613
1614 set_mtu_done:
1615         dev_info(enic, "MTU changed from %u to %u\n",  old_mtu, new_mtu);
1616         rte_spinlock_unlock(&enic->mtu_lock);
1617         return rc;
1618 }
1619
1620 static int enic_dev_init(struct enic *enic)
1621 {
1622         int err;
1623         struct rte_eth_dev *eth_dev = enic->rte_dev;
1624
1625         vnic_dev_intr_coal_timer_info_default(enic->vdev);
1626
1627         /* Get vNIC configuration
1628         */
1629         err = enic_get_vnic_config(enic);
1630         if (err) {
1631                 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1632                 return err;
1633         }
1634
1635         /* Get available resource counts */
1636         enic_get_res_counts(enic);
1637         if (enic->conf_rq_count == 1) {
1638                 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1639                 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1640                 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1641                 return -EINVAL;
1642         }
1643         /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1644         enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1645                                enic->conf_cq_count, 8);
1646         enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1647                                  enic->conf_intr_count, 8);
1648         enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1649                                enic->conf_rq_count, 8);
1650         enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1651                                enic->conf_wq_count, 8);
1652         if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1653                 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1654                 return -1;
1655         }
1656         if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1657                 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1658                 return -1;
1659         }
1660         if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1661                 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1662                 return -1;
1663         }
1664         if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1665                 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1666                 return -1;
1667         }
1668
1669         /* Get the supported filters */
1670         enic_fdir_info(enic);
1671
1672         eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1673                                                 * ENIC_MAX_MAC_ADDR, 0);
1674         if (!eth_dev->data->mac_addrs) {
1675                 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1676                 return -1;
1677         }
1678         ether_addr_copy((struct ether_addr *) enic->mac_addr,
1679                         eth_dev->data->mac_addrs);
1680
1681         vnic_dev_set_reset_flag(enic->vdev, 0);
1682
1683         LIST_INIT(&enic->flows);
1684         rte_spinlock_init(&enic->flows_lock);
1685         enic->max_flow_counter = -1;
1686
1687         /* set up link status checking */
1688         vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1689
1690         enic->overlay_offload = false;
1691         if (enic->disable_overlay && enic->vxlan) {
1692                 /*
1693                  * Explicitly disable overlay offload as the setting is
1694                  * sticky, and resetting vNIC does not disable it.
1695                  */
1696                 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1697                                                   OVERLAY_FEATURE_VXLAN,
1698                                                   OVERLAY_OFFLOAD_DISABLE)) {
1699                         dev_err(enic, "failed to disable overlay offload\n");
1700                 } else {
1701                         dev_info(enic, "Overlay offload is disabled\n");
1702                 }
1703         }
1704         if (!enic->disable_overlay && enic->vxlan &&
1705             /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1706             vnic_dev_overlay_offload_ctrl(enic->vdev,
1707                                           OVERLAY_FEATURE_VXLAN,
1708                                           OVERLAY_OFFLOAD_ENABLE) == 0) {
1709                 enic->tx_offload_capa |=
1710                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1711                         DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1712                         DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1713                 enic->tx_offload_mask |=
1714                         PKT_TX_OUTER_IPV6 |
1715                         PKT_TX_OUTER_IPV4 |
1716                         PKT_TX_OUTER_IP_CKSUM |
1717                         PKT_TX_TUNNEL_MASK;
1718                 enic->overlay_offload = true;
1719                 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1720                 dev_info(enic, "Overlay offload is enabled\n");
1721                 /*
1722                  * Reset the vxlan port to the default, as the NIC firmware
1723                  * does not reset it automatically and keeps the old setting.
1724                  */
1725                 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1726                                                  OVERLAY_CFG_VXLAN_PORT_UPDATE,
1727                                                  ENIC_DEFAULT_VXLAN_PORT)) {
1728                         dev_err(enic, "failed to update vxlan port\n");
1729                         return -EINVAL;
1730                 }
1731         }
1732
1733         return 0;
1734
1735 }
1736
1737 int enic_probe(struct enic *enic)
1738 {
1739         struct rte_pci_device *pdev = enic->pdev;
1740         int err = -1;
1741
1742         dev_debug(enic, " Initializing ENIC PMD\n");
1743
1744         /* if this is a secondary process the hardware is already initialized */
1745         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1746                 return 0;
1747
1748         enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1749         enic->bar0.len = pdev->mem_resource[0].len;
1750
1751         /* Register vNIC device */
1752         enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1753         if (!enic->vdev) {
1754                 dev_err(enic, "vNIC registration failed, aborting\n");
1755                 goto err_out;
1756         }
1757
1758         LIST_INIT(&enic->memzone_list);
1759         rte_spinlock_init(&enic->memzone_list_lock);
1760
1761         vnic_register_cbacks(enic->vdev,
1762                 enic_alloc_consistent,
1763                 enic_free_consistent);
1764
1765         /*
1766          * Allocate the consistent memory for stats and counters upfront so
1767          * both primary and secondary processes can access them.
1768          */
1769         err = vnic_dev_alloc_stats_mem(enic->vdev);
1770         if (err) {
1771                 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1772                 goto err_out_unregister;
1773         }
1774         err = vnic_dev_alloc_counter_mem(enic->vdev);
1775         if (err) {
1776                 dev_err(enic, "Failed to allocate counter memory, aborting\n");
1777                 goto err_out_unregister;
1778         }
1779
1780         /* Issue device open to get device in known state */
1781         err = enic_dev_open(enic);
1782         if (err) {
1783                 dev_err(enic, "vNIC dev open failed, aborting\n");
1784                 goto err_out_unregister;
1785         }
1786
1787         /* Set ingress vlan rewrite mode before vnic initialization */
1788         dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1789                   enic->ig_vlan_rewrite_mode);
1790         err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1791                 enic->ig_vlan_rewrite_mode);
1792         if (err) {
1793                 dev_err(enic,
1794                         "Failed to set ingress vlan rewrite mode, aborting.\n");
1795                 goto err_out_dev_close;
1796         }
1797
1798         /* Issue device init to initialize the vnic-to-switch link.
1799          * We'll start with carrier off and wait for link UP
1800          * notification later to turn on carrier.  We don't need
1801          * to wait here for the vnic-to-switch link initialization
1802          * to complete; link UP notification is the indication that
1803          * the process is complete.
1804          */
1805
1806         err = vnic_dev_init(enic->vdev, 0);
1807         if (err) {
1808                 dev_err(enic, "vNIC dev init failed, aborting\n");
1809                 goto err_out_dev_close;
1810         }
1811
1812         err = enic_dev_init(enic);
1813         if (err) {
1814                 dev_err(enic, "Device initialization failed, aborting\n");
1815                 goto err_out_dev_close;
1816         }
1817
1818         return 0;
1819
1820 err_out_dev_close:
1821         vnic_dev_close(enic->vdev);
1822 err_out_unregister:
1823         vnic_dev_unregister(enic->vdev);
1824 err_out:
1825         return err;
1826 }
1827
1828 void enic_remove(struct enic *enic)
1829 {
1830         enic_dev_deinit(enic);
1831         vnic_dev_close(enic->vdev);
1832         vnic_dev_unregister(enic->vdev);
1833 }