drivers/net/vmxnet3/vmxnet3_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35
  36 #include <stdio.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <errno.h>
  40 #include <stdint.h>
  41 #include <stdarg.h>
  42 #include <unistd.h>
  43 #include <inttypes.h>
  44
  45 #include <rte_byteorder.h>
  46 #include <rte_common.h>
  47 #include <rte_cycles.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_interrupts.h>
  51 #include <rte_pci.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_launch.h>
  55 #include <rte_eal.h>
  56 #include <rte_per_lcore.h>
  57 #include <rte_lcore.h>
  58 #include <rte_atomic.h>
  59 #include <rte_branch_prediction.h>
  60 #include <rte_mempool.h>
  61 #include <rte_malloc.h>
  62 #include <rte_mbuf.h>
  63 #include <rte_ether.h>
  64 #include <rte_ethdev.h>
  65 #include <rte_prefetch.h>
  66 #include <rte_ip.h>
  67 #include <rte_udp.h>
  68 #include <rte_tcp.h>
  69 #include <rte_sctp.h>
  70 #include <rte_string_fns.h>
  71 #include <rte_errno.h>
  72 #include <rte_net.h>
  73
  74 #include "base/vmxnet3_defs.h"
  75 #include "vmxnet3_ring.h"
  76
  77 #include "vmxnet3_logs.h"
  78 #include "vmxnet3_ethdev.h"
  79
  80 #define VMXNET3_TX_OFFLOAD_MASK ( \
  81                 PKT_TX_VLAN_PKT | \
  82                 PKT_TX_L4_MASK |  \
  83                 PKT_TX_TCP_SEG)
  84
  85 #define VMXNET3_TX_OFFLOAD_NOTSUP_MASK  \
  86         (PKT_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK)
  87
  88 static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
  89
  90 static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t);
  91 static void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
  92 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  93 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *);
  94 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *);
  95 #endif
  96
  97 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  98 static void
  99 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
 100 {
 101         uint32_t avail = 0;
 102
 103         if (rxq == NULL)
 104                 return;
 105
 106         PMD_RX_LOG(DEBUG,
 107                    "RXQ: cmd0 base : %p cmd1 base : %p comp ring base : %p.",
 108                    rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
 109         PMD_RX_LOG(DEBUG,
 110                    "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.",
 111                    (unsigned long)rxq->cmd_ring[0].basePA,
 112                    (unsigned long)rxq->cmd_ring[1].basePA,
 113                    (unsigned long)rxq->comp_ring.basePA);
 114
 115         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
 116         PMD_RX_LOG(DEBUG,
 117                    "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u",
 118                    (uint32_t)rxq->cmd_ring[0].size, avail,
 119                    rxq->comp_ring.next2proc,
 120                    rxq->cmd_ring[0].size - avail);
 121
 122         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
 123         PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u",
 124                    (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
 125                    rxq->cmd_ring[1].size - avail);
 126
 127 }
 128
 129 static void
 130 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq)
 131 {
 132         uint32_t avail = 0;
 133
 134         if (txq == NULL)
 135                 return;
 136
 137         PMD_TX_LOG(DEBUG, "TXQ: cmd base : %p comp ring base : %p data ring base : %p.",
 138                    txq->cmd_ring.base, txq->comp_ring.base, txq->data_ring.base);
 139         PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx data ring basePA : 0x%lx.",
 140                    (unsigned long)txq->cmd_ring.basePA,
 141                    (unsigned long)txq->comp_ring.basePA,
 142                    (unsigned long)txq->data_ring.basePA);
 143
 144         avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 145         PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u",
 146                    (uint32_t)txq->cmd_ring.size, avail,
 147                    txq->comp_ring.next2proc, txq->cmd_ring.size - avail);
 148 }
 149 #endif
 150
 151 static void
 152 vmxnet3_tx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 153 {
 154         while (ring->next2comp != ring->next2fill) {
 155                 /* No need to worry about desc ownership, device is quiesced by now. */
 156                 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp;
 157
 158                 if (buf_info->m) {
 159                         rte_pktmbuf_free(buf_info->m);
 160                         buf_info->m = NULL;
 161                         buf_info->bufPA = 0;
 162                         buf_info->len = 0;
 163                 }
 164                 vmxnet3_cmd_ring_adv_next2comp(ring);
 165         }
 166 }
 167
 168 static void
 169 vmxnet3_rx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 170 {
 171         uint32_t i;
 172
 173         for (i = 0; i < ring->size; i++) {
 174                 /* No need to worry about desc ownership, device is quiesced by now. */
 175                 vmxnet3_buf_info_t *buf_info = &ring->buf_info[i];
 176
 177                 if (buf_info->m) {
 178                         rte_pktmbuf_free_seg(buf_info->m);
 179                         buf_info->m = NULL;
 180                         buf_info->bufPA = 0;
 181                         buf_info->len = 0;
 182                 }
 183                 vmxnet3_cmd_ring_adv_next2comp(ring);
 184         }
 185 }
 186
 187 static void
 188 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring)
 189 {
 190         rte_free(ring->buf_info);
 191         ring->buf_info = NULL;
 192 }
 193
 194 void
 195 vmxnet3_dev_tx_queue_release(void *txq)
 196 {
 197         vmxnet3_tx_queue_t *tq = txq;
 198
 199         if (tq != NULL) {
 200                 /* Release mbufs */
 201                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 202                 /* Release the cmd_ring */
 203                 vmxnet3_cmd_ring_release(&tq->cmd_ring);
 204                 /* Release the memzone */
 205                 rte_memzone_free(tq->mz);
 206         }
 207 }
 208
 209 void
 210 vmxnet3_dev_rx_queue_release(void *rxq)
 211 {
 212         int i;
 213         vmxnet3_rx_queue_t *rq = rxq;
 214
 215         if (rq != NULL) {
 216                 /* Release mbufs */
 217                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 218                         vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 219
 220                 /* Release both the cmd_rings */
 221                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 222                         vmxnet3_cmd_ring_release(&rq->cmd_ring[i]);
 223
 224                 /* Release the memzone */
 225                 rte_memzone_free(rq->mz);
 226         }
 227 }
 228
 229 static void
 230 vmxnet3_dev_tx_queue_reset(void *txq)
 231 {
 232         vmxnet3_tx_queue_t *tq = txq;
 233         struct vmxnet3_cmd_ring *ring = &tq->cmd_ring;
 234         struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
 235         struct vmxnet3_data_ring *data_ring = &tq->data_ring;
 236         int size;
 237
 238         if (tq != NULL) {
 239                 /* Release the cmd_ring mbufs */
 240                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 241         }
 242
 243         /* Tx vmxnet rings structure initialization*/
 244         ring->next2fill = 0;
 245         ring->next2comp = 0;
 246         ring->gen = VMXNET3_INIT_GEN;
 247         comp_ring->next2proc = 0;
 248         comp_ring->gen = VMXNET3_INIT_GEN;
 249
 250         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
 251         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
 252         size += tq->txdata_desc_size * data_ring->size;
 253
 254         memset(ring->base, 0, size);
 255 }
 256
 257 static void
 258 vmxnet3_dev_rx_queue_reset(void *rxq)
 259 {
 260         int i;
 261         vmxnet3_rx_queue_t *rq = rxq;
 262         struct vmxnet3_hw *hw = rq->hw;
 263         struct vmxnet3_cmd_ring *ring0, *ring1;
 264         struct vmxnet3_comp_ring *comp_ring;
 265         struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring;
 266         int size;
 267
 268         if (rq != NULL) {
 269                 /* Release both the cmd_rings mbufs */
 270                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 271                         vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 272         }
 273
 274         ring0 = &rq->cmd_ring[0];
 275         ring1 = &rq->cmd_ring[1];
 276         comp_ring = &rq->comp_ring;
 277
 278         /* Rx vmxnet rings structure initialization */
 279         ring0->next2fill = 0;
 280         ring1->next2fill = 0;
 281         ring0->next2comp = 0;
 282         ring1->next2comp = 0;
 283         ring0->gen = VMXNET3_INIT_GEN;
 284         ring1->gen = VMXNET3_INIT_GEN;
 285         comp_ring->next2proc = 0;
 286         comp_ring->gen = VMXNET3_INIT_GEN;
 287
 288         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
 289         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
 290         if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size)
 291                 size += rq->data_desc_size * data_ring->size;
 292
 293         memset(ring0->base, 0, size);
 294 }
 295
 296 void
 297 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev)
 298 {
 299         unsigned i;
 300
 301         PMD_INIT_FUNC_TRACE();
 302
 303         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 304                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
 305
 306                 if (txq != NULL) {
 307                         txq->stopped = TRUE;
 308                         vmxnet3_dev_tx_queue_reset(txq);
 309                 }
 310         }
 311
 312         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 313                 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
 314
 315                 if (rxq != NULL) {
 316                         rxq->stopped = TRUE;
 317                         vmxnet3_dev_rx_queue_reset(rxq);
 318                 }
 319         }
 320 }
 321
 322 static int
 323 vmxnet3_unmap_pkt(uint16_t eop_idx, vmxnet3_tx_queue_t *txq)
 324 {
 325         int completed = 0;
 326         struct rte_mbuf *mbuf;
 327
 328         /* Release cmd_ring descriptor and free mbuf */
 329         RTE_ASSERT(txq->cmd_ring.base[eop_idx].txd.eop == 1);
 330
 331         mbuf = txq->cmd_ring.buf_info[eop_idx].m;
 332         if (mbuf == NULL)
 333                 rte_panic("EOP desc does not point to a valid mbuf");
 334         rte_pktmbuf_free(mbuf);
 335
 336         txq->cmd_ring.buf_info[eop_idx].m = NULL;
 337
 338         while (txq->cmd_ring.next2comp != eop_idx) {
 339                 /* no out-of-order completion */
 340                 RTE_ASSERT(txq->cmd_ring.base[txq->cmd_ring.next2comp].txd.cq == 0);
 341                 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 342                 completed++;
 343         }
 344
 345         /* Mark the txd for which tcd was generated as completed */
 346         vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 347
 348         return completed + 1;
 349 }
 350
 351 static void
 352 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
 353 {
 354         int completed = 0;
 355         vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
 356         struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
 357                 (comp_ring->base + comp_ring->next2proc);
 358
 359         while (tcd->gen == comp_ring->gen) {
 360                 completed += vmxnet3_unmap_pkt(tcd->txdIdx, txq);
 361
 362                 vmxnet3_comp_ring_adv_next2proc(comp_ring);
 363                 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
 364                                                     comp_ring->next2proc);
 365         }
 366
 367         PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
 368 }
 369
 370 uint16_t
 371 vmxnet3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
 372         uint16_t nb_pkts)
 373 {
 374         int32_t ret;
 375         uint32_t i;
 376         uint64_t ol_flags;
 377         struct rte_mbuf *m;
 378
 379         for (i = 0; i != nb_pkts; i++) {
 380                 m = tx_pkts[i];
 381                 ol_flags = m->ol_flags;
 382
 383                 /* Non-TSO packet cannot occupy more than
 384                  * VMXNET3_MAX_TXD_PER_PKT TX descriptors.
 385                  */
 386                 if ((ol_flags & PKT_TX_TCP_SEG) == 0 &&
 387                                 m->nb_segs > VMXNET3_MAX_TXD_PER_PKT) {
 388                         rte_errno = -EINVAL;
 389                         return i;
 390                 }
 391
 392                 /* check that only supported TX offloads are requested. */
 393                 if ((ol_flags & VMXNET3_TX_OFFLOAD_NOTSUP_MASK) != 0 ||
 394                                 (ol_flags & PKT_TX_L4_MASK) ==
 395                                 PKT_TX_SCTP_CKSUM) {
 396                         rte_errno = -ENOTSUP;
 397                         return i;
 398                 }
 399
 400 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
 401                 ret = rte_validate_tx_offload(m);
 402                 if (ret != 0) {
 403                         rte_errno = ret;
 404                         return i;
 405                 }
 406 #endif
 407                 ret = rte_net_intel_cksum_prepare(m);
 408                 if (ret != 0) {
 409                         rte_errno = ret;
 410                         return i;
 411                 }
 412         }
 413
 414         return i;
 415 }
 416
 417 uint16_t
 418 vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 419                   uint16_t nb_pkts)
 420 {
 421         uint16_t nb_tx;
 422         vmxnet3_tx_queue_t *txq = tx_queue;
 423         struct vmxnet3_hw *hw = txq->hw;
 424         Vmxnet3_TxQueueCtrl *txq_ctrl = &txq->shared->ctrl;
 425         uint32_t deferred = rte_le_to_cpu_32(txq_ctrl->txNumDeferred);
 426
 427         if (unlikely(txq->stopped)) {
 428                 PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
 429                 return 0;
 430         }
 431
 432         /* Free up the comp_descriptors aggressively */
 433         vmxnet3_tq_tx_complete(txq);
 434
 435         nb_tx = 0;
 436         while (nb_tx < nb_pkts) {
 437                 Vmxnet3_GenericDesc *gdesc;
 438                 vmxnet3_buf_info_t *tbi;
 439                 uint32_t first2fill, avail, dw2;
 440                 struct rte_mbuf *txm = tx_pkts[nb_tx];
 441                 struct rte_mbuf *m_seg = txm;
 442                 int copy_size = 0;
 443                 bool tso = (txm->ol_flags & PKT_TX_TCP_SEG) != 0;
 444                 /* # of descriptors needed for a packet. */
 445                 unsigned count = txm->nb_segs;
 446
 447                 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 448                 if (count > avail) {
 449                         /* Is command ring full? */
 450                         if (unlikely(avail == 0)) {
 451                                 PMD_TX_LOG(DEBUG, "No free ring descriptors");
 452                                 txq->stats.tx_ring_full++;
 453                                 txq->stats.drop_total += (nb_pkts - nb_tx);
 454                                 break;
 455                         }
 456
 457                         /* Command ring is not full but cannot handle the
 458                          * multi-segmented packet. Let's try the next packet
 459                          * in this case.
 460                          */
 461                         PMD_TX_LOG(DEBUG, "Running out of ring descriptors "
 462                                    "(avail %d needed %d)", avail, count);
 463                         txq->stats.drop_total++;
 464                         if (tso)
 465                                 txq->stats.drop_tso++;
 466                         rte_pktmbuf_free(txm);
 467                         nb_tx++;
 468                         continue;
 469                 }
 470
 471                 /* Drop non-TSO packet that is excessively fragmented */
 472                 if (unlikely(!tso && count > VMXNET3_MAX_TXD_PER_PKT)) {
 473                         PMD_TX_LOG(ERR, "Non-TSO packet cannot occupy more than %d tx "
 474                                    "descriptors. Packet dropped.", VMXNET3_MAX_TXD_PER_PKT);
 475                         txq->stats.drop_too_many_segs++;
 476                         txq->stats.drop_total++;
 477                         rte_pktmbuf_free(txm);
 478                         nb_tx++;
 479                         continue;
 480                 }
 481
 482                 if (txm->nb_segs == 1 &&
 483                     rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) {
 484                         struct Vmxnet3_TxDataDesc *tdd;
 485
 486                         tdd = (struct Vmxnet3_TxDataDesc *)
 487                                 ((uint8 *)txq->data_ring.base +
 488                                  txq->cmd_ring.next2fill *
 489                                  txq->txdata_desc_size);
 490                         copy_size = rte_pktmbuf_pkt_len(txm);
 491                         rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size);
 492                 }
 493
 494                 /* use the previous gen bit for the SOP desc */
 495                 dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
 496                 first2fill = txq->cmd_ring.next2fill;
 497                 do {
 498                         /* Remember the transmit buffer for cleanup */
 499                         tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
 500
 501                         /* NB: the following assumes that VMXNET3 maximum
 502                          * transmit buffer size (16K) is greater than
 503                          * maximum size of mbuf segment size.
 504                          */
 505                         gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill;
 506                         if (copy_size) {
 507                                 uint64 offset = txq->cmd_ring.next2fill *
 508                                                 txq->txdata_desc_size;
 509                                 gdesc->txd.addr =
 510                                         rte_cpu_to_le_64(txq->data_ring.basePA +
 511                                                          offset);
 512                         } else {
 513                                 gdesc->txd.addr = rte_mbuf_data_dma_addr(m_seg);
 514                         }
 515
 516                         gdesc->dword[2] = dw2 | m_seg->data_len;
 517                         gdesc->dword[3] = 0;
 518
 519                         /* move to the next2fill descriptor */
 520                         vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
 521
 522                         /* use the right gen for non-SOP desc */
 523                         dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 524                 } while ((m_seg = m_seg->next) != NULL);
 525
 526                 /* set the last buf_info for the pkt */
 527                 tbi->m = txm;
 528                 /* Update the EOP descriptor */
 529                 gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ;
 530
 531                 /* Add VLAN tag if present */
 532                 gdesc = txq->cmd_ring.base + first2fill;
 533                 if (txm->ol_flags & PKT_TX_VLAN_PKT) {
 534                         gdesc->txd.ti = 1;
 535                         gdesc->txd.tci = txm->vlan_tci;
 536                 }
 537
 538                 if (tso) {
 539                         uint16_t mss = txm->tso_segsz;
 540
 541                         RTE_ASSERT(mss > 0);
 542
 543                         gdesc->txd.hlen = txm->l2_len + txm->l3_len + txm->l4_len;
 544                         gdesc->txd.om = VMXNET3_OM_TSO;
 545                         gdesc->txd.msscof = mss;
 546
 547                         deferred += (rte_pktmbuf_pkt_len(txm) - gdesc->txd.hlen + mss - 1) / mss;
 548                 } else if (txm->ol_flags & PKT_TX_L4_MASK) {
 549                         gdesc->txd.om = VMXNET3_OM_CSUM;
 550                         gdesc->txd.hlen = txm->l2_len + txm->l3_len;
 551
 552                         switch (txm->ol_flags & PKT_TX_L4_MASK) {
 553                         case PKT_TX_TCP_CKSUM:
 554                                 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct tcp_hdr, cksum);
 555                                 break;
 556                         case PKT_TX_UDP_CKSUM:
 557                                 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct udp_hdr, dgram_cksum);
 558                                 break;
 559                         default:
 560                                 PMD_TX_LOG(WARNING, "requested cksum offload not supported %#llx",
 561                                            txm->ol_flags & PKT_TX_L4_MASK);
 562                                 abort();
 563                         }
 564                         deferred++;
 565                 } else {
 566                         gdesc->txd.hlen = 0;
 567                         gdesc->txd.om = VMXNET3_OM_NONE;
 568                         gdesc->txd.msscof = 0;
 569                         deferred++;
 570                 }
 571
 572                 /* flip the GEN bit on the SOP */
 573                 rte_compiler_barrier();
 574                 gdesc->dword[2] ^= VMXNET3_TXD_GEN;
 575
 576                 txq_ctrl->txNumDeferred = rte_cpu_to_le_32(deferred);
 577                 nb_tx++;
 578         }
 579
 580         PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", rte_le_to_cpu_32(txq_ctrl->txThreshold));
 581
 582         if (deferred >= rte_le_to_cpu_32(txq_ctrl->txThreshold)) {
 583                 txq_ctrl->txNumDeferred = 0;
 584                 /* Notify vSwitch that packets are available. */
 585                 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
 586                                        txq->cmd_ring.next2fill);
 587         }
 588
 589         return nb_tx;
 590 }
 591
 592 static inline void
 593 vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id,
 594                    struct rte_mbuf *mbuf)
 595 {
 596         uint32_t val = 0;
 597         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 598         struct Vmxnet3_RxDesc *rxd =
 599                 (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
 600         vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
 601
 602         if (ring_id == 0)
 603                 val = VMXNET3_RXD_BTYPE_HEAD;
 604         else
 605                 val = VMXNET3_RXD_BTYPE_BODY;
 606
 607         buf_info->m = mbuf;
 608         buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM);
 609         buf_info->bufPA = rte_mbuf_data_dma_addr_default(mbuf);
 610
 611         rxd->addr = buf_info->bufPA;
 612         rxd->btype = val;
 613         rxd->len = buf_info->len;
 614         rxd->gen = ring->gen;
 615
 616         vmxnet3_cmd_ring_adv_next2fill(ring);
 617 }
 618 /*
 619  *  Allocates mbufs and clusters. Post rx descriptors with buffer details
 620  *  so that device can receive packets in those buffers.
 621  *  Ring layout:
 622  *      Among the two rings, 1st ring contains buffers of type 0 and type 1.
 623  *      bufs_per_pkt is set such that for non-LRO cases all the buffers required
 624  *      by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
 625  *      2nd ring contains buffers of type 1 alone. Second ring mostly be used
 626  *      only for LRO.
 627  */
 628 static int
 629 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id)
 630 {
 631         int err = 0;
 632         uint32_t i = 0, val = 0;
 633         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 634
 635         if (ring_id == 0) {
 636                 /* Usually: One HEAD type buf per packet
 637                  * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
 638                  * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
 639                  */
 640
 641                 /* We use single packet buffer so all heads here */
 642                 val = VMXNET3_RXD_BTYPE_HEAD;
 643         } else {
 644                 /* All BODY type buffers for 2nd ring */
 645                 val = VMXNET3_RXD_BTYPE_BODY;
 646         }
 647
 648         while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
 649                 struct Vmxnet3_RxDesc *rxd;
 650                 struct rte_mbuf *mbuf;
 651                 vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
 652
 653                 rxd = (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
 654
 655                 /* Allocate blank mbuf for the current Rx Descriptor */
 656                 mbuf = rte_mbuf_raw_alloc(rxq->mp);
 657                 if (unlikely(mbuf == NULL)) {
 658                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 659                         rxq->stats.rx_buf_alloc_failure++;
 660                         err = ENOMEM;
 661                         break;
 662                 }
 663
 664                 /*
 665                  * Load mbuf pointer into buf_info[ring_size]
 666                  * buf_info structure is equivalent to cookie for virtio-virtqueue
 667                  */
 668                 buf_info->m = mbuf;
 669                 buf_info->len = (uint16_t)(mbuf->buf_len -
 670                                            RTE_PKTMBUF_HEADROOM);
 671                 buf_info->bufPA = rte_mbuf_data_dma_addr_default(mbuf);
 672
 673                 /* Load Rx Descriptor with the buffer's GPA */
 674                 rxd->addr = buf_info->bufPA;
 675
 676                 /* After this point rxd->addr MUST not be NULL */
 677                 rxd->btype = val;
 678                 rxd->len = buf_info->len;
 679                 /* Flip gen bit at the end to change ownership */
 680                 rxd->gen = ring->gen;
 681
 682                 vmxnet3_cmd_ring_adv_next2fill(ring);
 683                 i++;
 684         }
 685
 686         /* Return error only if no buffers are posted at present */
 687         if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1))
 688                 return -err;
 689         else
 690                 return i;
 691 }
 692
 693
 694 /* Receive side checksum and other offloads */
 695 static void
 696 vmxnet3_rx_offload(const Vmxnet3_RxCompDesc *rcd, struct rte_mbuf *rxm)
 697 {
 698         /* Check for RSS */
 699         if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE) {
 700                 rxm->ol_flags |= PKT_RX_RSS_HASH;
 701                 rxm->hash.rss = rcd->rssHash;
 702         }
 703
 704         /* Check packet type, checksum errors, etc. Only support IPv4 for now. */
 705         if (rcd->v4) {
 706                 struct ether_hdr *eth = rte_pktmbuf_mtod(rxm, struct ether_hdr *);
 707                 struct ipv4_hdr *ip = (struct ipv4_hdr *)(eth + 1);
 708
 709                 if (((ip->version_ihl & 0xf) << 2) > (int)sizeof(struct ipv4_hdr))
 710                         rxm->packet_type = RTE_PTYPE_L3_IPV4_EXT;
 711                 else
 712                         rxm->packet_type = RTE_PTYPE_L3_IPV4;
 713
 714                 if (!rcd->cnc) {
 715                         if (!rcd->ipc)
 716                                 rxm->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 717
 718                         if ((rcd->tcp || rcd->udp) && !rcd->tuc)
 719                                 rxm->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 720                 }
 721         }
 722 }
 723
 724 /*
 725  * Process the Rx Completion Ring of given vmxnet3_rx_queue
 726  * for nb_pkts burst and return the number of packets received
 727  */
 728 uint16_t
 729 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 730 {
 731         uint16_t nb_rx;
 732         uint32_t nb_rxd, idx;
 733         uint8_t ring_idx;
 734         vmxnet3_rx_queue_t *rxq;
 735         Vmxnet3_RxCompDesc *rcd;
 736         vmxnet3_buf_info_t *rbi;
 737         Vmxnet3_RxDesc *rxd;
 738         struct rte_mbuf *rxm = NULL;
 739         struct vmxnet3_hw *hw;
 740
 741         nb_rx = 0;
 742         ring_idx = 0;
 743         nb_rxd = 0;
 744         idx = 0;
 745
 746         rxq = rx_queue;
 747         hw = rxq->hw;
 748
 749         rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 750
 751         if (unlikely(rxq->stopped)) {
 752                 PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
 753                 return 0;
 754         }
 755
 756         while (rcd->gen == rxq->comp_ring.gen) {
 757                 struct rte_mbuf *newm;
 758
 759                 if (nb_rx >= nb_pkts)
 760                         break;
 761
 762                 newm = rte_mbuf_raw_alloc(rxq->mp);
 763                 if (unlikely(newm == NULL)) {
 764                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 765                         rxq->stats.rx_buf_alloc_failure++;
 766                         break;
 767                 }
 768
 769                 idx = rcd->rxdIdx;
 770                 ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID);
 771                 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
 772                 RTE_SET_USED(rxd); /* used only for assert when enabled */
 773                 rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
 774
 775                 PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
 776
 777                 RTE_ASSERT(rcd->len <= rxd->len);
 778                 RTE_ASSERT(rbi->m);
 779
 780                 /* Get the packet buffer pointer from buf_info */
 781                 rxm = rbi->m;
 782
 783                 /* Clear descriptor associated buf_info to be reused */
 784                 rbi->m = NULL;
 785                 rbi->bufPA = 0;
 786
 787                 /* Update the index that we received a packet */
 788                 rxq->cmd_ring[ring_idx].next2comp = idx;
 789
 790                 /* For RCD with EOP set, check if there is frame error */
 791                 if (unlikely(rcd->eop && rcd->err)) {
 792                         rxq->stats.drop_total++;
 793                         rxq->stats.drop_err++;
 794
 795                         if (!rcd->fcs) {
 796                                 rxq->stats.drop_fcs++;
 797                                 PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
 798                         }
 799                         PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
 800                                    (int)(rcd - (struct Vmxnet3_RxCompDesc *)
 801                                          rxq->comp_ring.base), rcd->rxdIdx);
 802                         rte_pktmbuf_free_seg(rxm);
 803                         goto rcd_done;
 804                 }
 805
 806                 /* Initialize newly received packet buffer */
 807                 rxm->port = rxq->port_id;
 808                 rxm->nb_segs = 1;
 809                 rxm->next = NULL;
 810                 rxm->pkt_len = (uint16_t)rcd->len;
 811                 rxm->data_len = (uint16_t)rcd->len;
 812                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 813                 rxm->ol_flags = 0;
 814                 rxm->vlan_tci = 0;
 815
 816                 /*
 817                  * If this is the first buffer of the received packet,
 818                  * set the pointer to the first mbuf of the packet
 819                  * Otherwise, update the total length and the number of segments
 820                  * of the current scattered packet, and update the pointer to
 821                  * the last mbuf of the current packet.
 822                  */
 823                 if (rcd->sop) {
 824                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
 825
 826                         if (unlikely(rcd->len == 0)) {
 827                                 RTE_ASSERT(rcd->eop);
 828
 829                                 PMD_RX_LOG(DEBUG,
 830                                            "Rx buf was skipped. rxring[%d][%d])",
 831                                            ring_idx, idx);
 832                                 rte_pktmbuf_free_seg(rxm);
 833                                 goto rcd_done;
 834                         }
 835
 836                         if (vmxnet3_rx_data_ring(hw, rcd->rqID)) {
 837                                 uint8_t *rdd = rxq->data_ring.base +
 838                                         idx * rxq->data_desc_size;
 839
 840                                 RTE_ASSERT(VMXNET3_VERSION_GE_3(hw));
 841                                 rte_memcpy(rte_pktmbuf_mtod(rxm, char *),
 842                                            rdd, rcd->len);
 843                         }
 844
 845                         rxq->start_seg = rxm;
 846                         vmxnet3_rx_offload(rcd, rxm);
 847                 } else {
 848                         struct rte_mbuf *start = rxq->start_seg;
 849
 850                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_BODY);
 851
 852                         start->pkt_len += rxm->data_len;
 853                         start->nb_segs++;
 854
 855                         rxq->last_seg->next = rxm;
 856                 }
 857                 rxq->last_seg = rxm;
 858
 859                 if (rcd->eop) {
 860                         struct rte_mbuf *start = rxq->start_seg;
 861
 862                         /* Check for hardware stripped VLAN tag */
 863                         if (rcd->ts) {
 864                                 start->ol_flags |= (PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED);
 865                                 start->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci);
 866                         }
 867
 868                         rx_pkts[nb_rx++] = start;
 869                         rxq->start_seg = NULL;
 870                 }
 871
 872 rcd_done:
 873                 rxq->cmd_ring[ring_idx].next2comp = idx;
 874                 VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp,
 875                                           rxq->cmd_ring[ring_idx].size);
 876
 877                 /* It's time to renew descriptors */
 878                 vmxnet3_renew_desc(rxq, ring_idx, newm);
 879                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
 880                         VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
 881                                                rxq->cmd_ring[ring_idx].next2fill);
 882                 }
 883
 884                 /* Advance to the next descriptor in comp_ring */
 885                 vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
 886
 887                 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 888                 nb_rxd++;
 889                 if (nb_rxd > rxq->cmd_ring[0].size) {
 890                         PMD_RX_LOG(ERR, "Used up quota of receiving packets,"
 891                                    " relinquish control.");
 892                         break;
 893                 }
 894         }
 895
 896         return nb_rx;
 897 }
 898
 899 int
 900 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev,
 901                            uint16_t queue_idx,
 902                            uint16_t nb_desc,
 903                            unsigned int socket_id,
 904                            __rte_unused const struct rte_eth_txconf *tx_conf)
 905 {
 906         struct vmxnet3_hw *hw = dev->data->dev_private;
 907         const struct rte_memzone *mz;
 908         struct vmxnet3_tx_queue *txq;
 909         struct vmxnet3_cmd_ring *ring;
 910         struct vmxnet3_comp_ring *comp_ring;
 911         struct vmxnet3_data_ring *data_ring;
 912         int size;
 913
 914         PMD_INIT_FUNC_TRACE();
 915
 916         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMSCTP) !=
 917             ETH_TXQ_FLAGS_NOXSUMSCTP) {
 918                 PMD_INIT_LOG(ERR, "SCTP checksum offload not supported");
 919                 return -EINVAL;
 920         }
 921
 922         txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue),
 923                           RTE_CACHE_LINE_SIZE);
 924         if (txq == NULL) {
 925                 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure");
 926                 return -ENOMEM;
 927         }
 928
 929         txq->queue_id = queue_idx;
 930         txq->port_id = dev->data->port_id;
 931         txq->shared = &hw->tqd_start[queue_idx];
 932         txq->hw = hw;
 933         txq->qid = queue_idx;
 934         txq->stopped = TRUE;
 935         txq->txdata_desc_size = hw->txdata_desc_size;
 936
 937         ring = &txq->cmd_ring;
 938         comp_ring = &txq->comp_ring;
 939         data_ring = &txq->data_ring;
 940
 941         /* Tx vmxnet ring length should be between 512-4096 */
 942         if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) {
 943                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u",
 944                              VMXNET3_DEF_TX_RING_SIZE);
 945                 return -EINVAL;
 946         } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) {
 947                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u",
 948                              VMXNET3_TX_RING_MAX_SIZE);
 949                 return -EINVAL;
 950         } else {
 951                 ring->size = nb_desc;
 952                 ring->size &= ~VMXNET3_RING_SIZE_MASK;
 953         }
 954         comp_ring->size = data_ring->size = ring->size;
 955
 956         /* Tx vmxnet rings structure initialization*/
 957         ring->next2fill = 0;
 958         ring->next2comp = 0;
 959         ring->gen = VMXNET3_INIT_GEN;
 960         comp_ring->next2proc = 0;
 961         comp_ring->gen = VMXNET3_INIT_GEN;
 962
 963         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
 964         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
 965         size += txq->txdata_desc_size * data_ring->size;
 966
 967         mz = rte_eth_dma_zone_reserve(dev, "txdesc", queue_idx, size,
 968                                       VMXNET3_RING_BA_ALIGN, socket_id);
 969         if (mz == NULL) {
 970                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
 971                 return -ENOMEM;
 972         }
 973         txq->mz = mz;
 974         memset(mz->addr, 0, mz->len);
 975
 976         /* cmd_ring initialization */
 977         ring->base = mz->addr;
 978         ring->basePA = mz->phys_addr;
 979
 980         /* comp_ring initialization */
 981         comp_ring->base = ring->base + ring->size;
 982         comp_ring->basePA = ring->basePA +
 983                 (sizeof(struct Vmxnet3_TxDesc) * ring->size);
 984
 985         /* data_ring initialization */
 986         data_ring->base = (Vmxnet3_TxDataDesc *)(comp_ring->base + comp_ring->size);
 987         data_ring->basePA = comp_ring->basePA +
 988                         (sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size);
 989
 990         /* cmd_ring0 buf_info allocation */
 991         ring->buf_info = rte_zmalloc("tx_ring_buf_info",
 992                                      ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE);
 993         if (ring->buf_info == NULL) {
 994                 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure");
 995                 return -ENOMEM;
 996         }
 997
 998         /* Update the data portion with txq */
 999         dev->data->tx_queues[queue_idx] = txq;
1000
1001         return 0;
1002 }
1003
1004 int
1005 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
1006                            uint16_t queue_idx,
1007                            uint16_t nb_desc,
1008                            unsigned int socket_id,
1009                            __rte_unused const struct rte_eth_rxconf *rx_conf,
1010                            struct rte_mempool *mp)
1011 {
1012         const struct rte_memzone *mz;
1013         struct vmxnet3_rx_queue *rxq;
1014         struct vmxnet3_hw *hw = dev->data->dev_private;
1015         struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
1016         struct vmxnet3_comp_ring *comp_ring;
1017         struct vmxnet3_rx_data_ring *data_ring;
1018         int size;
1019         uint8_t i;
1020         char mem_name[32];
1021
1022         PMD_INIT_FUNC_TRACE();
1023
1024         rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue),
1025                           RTE_CACHE_LINE_SIZE);
1026         if (rxq == NULL) {
1027                 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure");
1028                 return -ENOMEM;
1029         }
1030
1031         rxq->mp = mp;
1032         rxq->queue_id = queue_idx;
1033         rxq->port_id = dev->data->port_id;
1034         rxq->shared = &hw->rqd_start[queue_idx];
1035         rxq->hw = hw;
1036         rxq->qid1 = queue_idx;
1037         rxq->qid2 = queue_idx + hw->num_rx_queues;
1038         rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues;
1039         rxq->data_desc_size = hw->rxdata_desc_size;
1040         rxq->stopped = TRUE;
1041
1042         ring0 = &rxq->cmd_ring[0];
1043         ring1 = &rxq->cmd_ring[1];
1044         comp_ring = &rxq->comp_ring;
1045         data_ring = &rxq->data_ring;
1046
1047         /* Rx vmxnet rings length should be between 256-4096 */
1048         if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
1049                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256");
1050                 return -EINVAL;
1051         } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) {
1052                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096");
1053                 return -EINVAL;
1054         } else {
1055                 ring0->size = nb_desc;
1056                 ring0->size &= ~VMXNET3_RING_SIZE_MASK;
1057                 ring1->size = ring0->size;
1058         }
1059
1060         comp_ring->size = ring0->size + ring1->size;
1061         data_ring->size = ring0->size;
1062
1063         /* Rx vmxnet rings structure initialization */
1064         ring0->next2fill = 0;
1065         ring1->next2fill = 0;
1066         ring0->next2comp = 0;
1067         ring1->next2comp = 0;
1068         ring0->gen = VMXNET3_INIT_GEN;
1069         ring1->gen = VMXNET3_INIT_GEN;
1070         comp_ring->next2proc = 0;
1071         comp_ring->gen = VMXNET3_INIT_GEN;
1072
1073         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
1074         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1075         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size)
1076                 size += rxq->data_desc_size * data_ring->size;
1077
1078         mz = rte_eth_dma_zone_reserve(dev, "rxdesc", queue_idx, size,
1079                                       VMXNET3_RING_BA_ALIGN, socket_id);
1080         if (mz == NULL) {
1081                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1082                 return -ENOMEM;
1083         }
1084         rxq->mz = mz;
1085         memset(mz->addr, 0, mz->len);
1086
1087         /* cmd_ring0 initialization */
1088         ring0->base = mz->addr;
1089         ring0->basePA = mz->phys_addr;
1090
1091         /* cmd_ring1 initialization */
1092         ring1->base = ring0->base + ring0->size;
1093         ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size;
1094
1095         /* comp_ring initialization */
1096         comp_ring->base = ring1->base + ring1->size;
1097         comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
1098                 ring1->size;
1099
1100         /* data_ring initialization */
1101         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) {
1102                 data_ring->base =
1103                         (uint8_t *)(comp_ring->base + comp_ring->size);
1104                 data_ring->basePA = comp_ring->basePA +
1105                         sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1106         }
1107
1108         /* cmd_ring0-cmd_ring1 buf_info allocation */
1109         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {
1110
1111                 ring = &rxq->cmd_ring[i];
1112                 ring->rid = i;
1113                 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i);
1114
1115                 ring->buf_info = rte_zmalloc(mem_name,
1116                                              ring->size * sizeof(vmxnet3_buf_info_t),
1117                                              RTE_CACHE_LINE_SIZE);
1118                 if (ring->buf_info == NULL) {
1119                         PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure");
1120                         return -ENOMEM;
1121                 }
1122         }
1123
1124         /* Update the data portion with rxq */
1125         dev->data->rx_queues[queue_idx] = rxq;
1126
1127         return 0;
1128 }
1129
1130 /*
1131  * Initializes Receive Unit
1132  * Load mbufs in rx queue in advance
1133  */
1134 int
1135 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
1136 {
1137         struct vmxnet3_hw *hw = dev->data->dev_private;
1138
1139         int i, ret;
1140         uint8_t j;
1141
1142         PMD_INIT_FUNC_TRACE();
1143
1144         for (i = 0; i < hw->num_rx_queues; i++) {
1145                 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
1146
1147                 for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) {
1148                         /* Passing 0 as alloc_num will allocate full ring */
1149                         ret = vmxnet3_post_rx_bufs(rxq, j);
1150                         if (ret <= 0) {
1151                                 PMD_INIT_LOG(ERR,
1152                                              "ERROR: Posting Rxq: %d buffers ring: %d",
1153                                              i, j);
1154                                 return -ret;
1155                         }
1156                         /*
1157                          * Updating device with the index:next2fill to fill the
1158                          * mbufs for coming packets.
1159                          */
1160                         if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1161                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1162                                                        rxq->cmd_ring[j].next2fill);
1163                         }
1164                 }
1165                 rxq->stopped = FALSE;
1166                 rxq->start_seg = NULL;
1167         }
1168
1169         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1170                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
1171
1172                 txq->stopped = FALSE;
1173         }
1174
1175         return 0;
1176 }
1177
1178 static uint8_t rss_intel_key[40] = {
1179         0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
1180         0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
1181         0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
1182         0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
1183         0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
1184 };
1185
1186 /*
1187  * Configure RSS feature
1188  */
1189 int
1190 vmxnet3_rss_configure(struct rte_eth_dev *dev)
1191 {
1192         struct vmxnet3_hw *hw = dev->data->dev_private;
1193         struct VMXNET3_RSSConf *dev_rss_conf;
1194         struct rte_eth_rss_conf *port_rss_conf;
1195         uint64_t rss_hf;
1196         uint8_t i, j;
1197
1198         PMD_INIT_FUNC_TRACE();
1199
1200         dev_rss_conf = hw->rss_conf;
1201         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1202
1203         /* loading hashFunc */
1204         dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
1205         /* loading hashKeySize */
1206         dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE;
1207         /* loading indTableSize: Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/
1208         dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4);
1209
1210         if (port_rss_conf->rss_key == NULL) {
1211                 /* Default hash key */
1212                 port_rss_conf->rss_key = rss_intel_key;
1213         }
1214
1215         /* loading hashKey */
1216         memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key,
1217                dev_rss_conf->hashKeySize);
1218
1219         /* loading indTable */
1220         for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) {
1221                 if (j == dev->data->nb_rx_queues)
1222                         j = 0;
1223                 dev_rss_conf->indTable[i] = j;
1224         }
1225
1226         /* loading hashType */
1227         dev_rss_conf->hashType = 0;
1228         rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL;
1229         if (rss_hf & ETH_RSS_IPV4)
1230                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4;
1231         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1232                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4;
1233         if (rss_hf & ETH_RSS_IPV6)
1234                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6;
1235         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1236                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
1237
1238         return VMXNET3_SUCCESS;
1239 }