drivers/net/vmxnet3/vmxnet3_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <errno.h>
  11 #include <stdint.h>
  12 #include <stdarg.h>
  13 #include <unistd.h>
  14 #include <inttypes.h>
  15
  16 #include <rte_byteorder.h>
  17 #include <rte_common.h>
  18 #include <rte_cycles.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_pci.h>
  23 #include <rte_memory.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_mempool.h>
  32 #include <rte_malloc.h>
  33 #include <rte_mbuf.h>
  34 #include <rte_ether.h>
  35 #include <rte_ethdev_driver.h>
  36 #include <rte_prefetch.h>
  37 #include <rte_ip.h>
  38 #include <rte_udp.h>
  39 #include <rte_tcp.h>
  40 #include <rte_sctp.h>
  41 #include <rte_string_fns.h>
  42 #include <rte_errno.h>
  43 #include <rte_net.h>
  44
  45 #include "base/vmxnet3_defs.h"
  46 #include "vmxnet3_ring.h"
  47
  48 #include "vmxnet3_logs.h"
  49 #include "vmxnet3_ethdev.h"
  50
  51 #define VMXNET3_TX_OFFLOAD_MASK ( \
  52                 PKT_TX_VLAN_PKT | \
  53                 PKT_TX_L4_MASK |  \
  54                 PKT_TX_TCP_SEG)
  55
  56 #define VMXNET3_TX_OFFLOAD_NOTSUP_MASK  \
  57         (PKT_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK)
  58
  59 static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
  60
  61 static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t);
  62 static void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
  63 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  64 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *);
  65 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *);
  66 #endif
  67
  68 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  69 static void
  70 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
  71 {
  72         uint32_t avail = 0;
  73
  74         if (rxq == NULL)
  75                 return;
  76
  77         PMD_RX_LOG(DEBUG,
  78                    "RXQ: cmd0 base : %p cmd1 base : %p comp ring base : %p.",
  79                    rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
  80         PMD_RX_LOG(DEBUG,
  81                    "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.",
  82                    (unsigned long)rxq->cmd_ring[0].basePA,
  83                    (unsigned long)rxq->cmd_ring[1].basePA,
  84                    (unsigned long)rxq->comp_ring.basePA);
  85
  86         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
  87         PMD_RX_LOG(DEBUG,
  88                    "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u",
  89                    (uint32_t)rxq->cmd_ring[0].size, avail,
  90                    rxq->comp_ring.next2proc,
  91                    rxq->cmd_ring[0].size - avail);
  92
  93         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
  94         PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u",
  95                    (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
  96                    rxq->cmd_ring[1].size - avail);
  97
  98 }
  99
 100 static void
 101 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq)
 102 {
 103         uint32_t avail = 0;
 104
 105         if (txq == NULL)
 106                 return;
 107
 108         PMD_TX_LOG(DEBUG, "TXQ: cmd base : %p comp ring base : %p data ring base : %p.",
 109                    txq->cmd_ring.base, txq->comp_ring.base, txq->data_ring.base);
 110         PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx data ring basePA : 0x%lx.",
 111                    (unsigned long)txq->cmd_ring.basePA,
 112                    (unsigned long)txq->comp_ring.basePA,
 113                    (unsigned long)txq->data_ring.basePA);
 114
 115         avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 116         PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u",
 117                    (uint32_t)txq->cmd_ring.size, avail,
 118                    txq->comp_ring.next2proc, txq->cmd_ring.size - avail);
 119 }
 120 #endif
 121
 122 static void
 123 vmxnet3_tx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 124 {
 125         while (ring->next2comp != ring->next2fill) {
 126                 /* No need to worry about desc ownership, device is quiesced by now. */
 127                 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp;
 128
 129                 if (buf_info->m) {
 130                         rte_pktmbuf_free(buf_info->m);
 131                         buf_info->m = NULL;
 132                         buf_info->bufPA = 0;
 133                         buf_info->len = 0;
 134                 }
 135                 vmxnet3_cmd_ring_adv_next2comp(ring);
 136         }
 137 }
 138
 139 static void
 140 vmxnet3_rx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 141 {
 142         uint32_t i;
 143
 144         for (i = 0; i < ring->size; i++) {
 145                 /* No need to worry about desc ownership, device is quiesced by now. */
 146                 vmxnet3_buf_info_t *buf_info = &ring->buf_info[i];
 147
 148                 if (buf_info->m) {
 149                         rte_pktmbuf_free_seg(buf_info->m);
 150                         buf_info->m = NULL;
 151                         buf_info->bufPA = 0;
 152                         buf_info->len = 0;
 153                 }
 154                 vmxnet3_cmd_ring_adv_next2comp(ring);
 155         }
 156 }
 157
 158 static void
 159 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring)
 160 {
 161         rte_free(ring->buf_info);
 162         ring->buf_info = NULL;
 163 }
 164
 165 void
 166 vmxnet3_dev_tx_queue_release(void *txq)
 167 {
 168         vmxnet3_tx_queue_t *tq = txq;
 169
 170         if (tq != NULL) {
 171                 /* Release mbufs */
 172                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 173                 /* Release the cmd_ring */
 174                 vmxnet3_cmd_ring_release(&tq->cmd_ring);
 175                 /* Release the memzone */
 176                 rte_memzone_free(tq->mz);
 177                 /* Release the queue */
 178                 rte_free(tq);
 179         }
 180 }
 181
 182 void
 183 vmxnet3_dev_rx_queue_release(void *rxq)
 184 {
 185         int i;
 186         vmxnet3_rx_queue_t *rq = rxq;
 187
 188         if (rq != NULL) {
 189                 /* Release mbufs */
 190                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 191                         vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 192
 193                 /* Release both the cmd_rings */
 194                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 195                         vmxnet3_cmd_ring_release(&rq->cmd_ring[i]);
 196
 197                 /* Release the memzone */
 198                 rte_memzone_free(rq->mz);
 199
 200                 /* Release the queue */
 201                 rte_free(rq);
 202         }
 203 }
 204
 205 static void
 206 vmxnet3_dev_tx_queue_reset(void *txq)
 207 {
 208         vmxnet3_tx_queue_t *tq = txq;
 209         struct vmxnet3_cmd_ring *ring = &tq->cmd_ring;
 210         struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
 211         struct vmxnet3_data_ring *data_ring = &tq->data_ring;
 212         int size;
 213
 214         if (tq != NULL) {
 215                 /* Release the cmd_ring mbufs */
 216                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 217         }
 218
 219         /* Tx vmxnet rings structure initialization*/
 220         ring->next2fill = 0;
 221         ring->next2comp = 0;
 222         ring->gen = VMXNET3_INIT_GEN;
 223         comp_ring->next2proc = 0;
 224         comp_ring->gen = VMXNET3_INIT_GEN;
 225
 226         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
 227         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
 228         size += tq->txdata_desc_size * data_ring->size;
 229
 230         memset(ring->base, 0, size);
 231 }
 232
 233 static void
 234 vmxnet3_dev_rx_queue_reset(void *rxq)
 235 {
 236         int i;
 237         vmxnet3_rx_queue_t *rq = rxq;
 238         struct vmxnet3_hw *hw = rq->hw;
 239         struct vmxnet3_cmd_ring *ring0, *ring1;
 240         struct vmxnet3_comp_ring *comp_ring;
 241         struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring;
 242         int size;
 243
 244         /* Release both the cmd_rings mbufs */
 245         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 246                 vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 247
 248         ring0 = &rq->cmd_ring[0];
 249         ring1 = &rq->cmd_ring[1];
 250         comp_ring = &rq->comp_ring;
 251
 252         /* Rx vmxnet rings structure initialization */
 253         ring0->next2fill = 0;
 254         ring1->next2fill = 0;
 255         ring0->next2comp = 0;
 256         ring1->next2comp = 0;
 257         ring0->gen = VMXNET3_INIT_GEN;
 258         ring1->gen = VMXNET3_INIT_GEN;
 259         comp_ring->next2proc = 0;
 260         comp_ring->gen = VMXNET3_INIT_GEN;
 261
 262         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
 263         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
 264         if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size)
 265                 size += rq->data_desc_size * data_ring->size;
 266
 267         memset(ring0->base, 0, size);
 268 }
 269
 270 void
 271 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev)
 272 {
 273         unsigned i;
 274
 275         PMD_INIT_FUNC_TRACE();
 276
 277         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 278                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
 279
 280                 if (txq != NULL) {
 281                         txq->stopped = TRUE;
 282                         vmxnet3_dev_tx_queue_reset(txq);
 283                 }
 284         }
 285
 286         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 287                 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
 288
 289                 if (rxq != NULL) {
 290                         rxq->stopped = TRUE;
 291                         vmxnet3_dev_rx_queue_reset(rxq);
 292                 }
 293         }
 294 }
 295
 296 static int
 297 vmxnet3_unmap_pkt(uint16_t eop_idx, vmxnet3_tx_queue_t *txq)
 298 {
 299         int completed = 0;
 300         struct rte_mbuf *mbuf;
 301
 302         /* Release cmd_ring descriptor and free mbuf */
 303         RTE_ASSERT(txq->cmd_ring.base[eop_idx].txd.eop == 1);
 304
 305         mbuf = txq->cmd_ring.buf_info[eop_idx].m;
 306         if (mbuf == NULL)
 307                 rte_panic("EOP desc does not point to a valid mbuf");
 308         rte_pktmbuf_free(mbuf);
 309
 310         txq->cmd_ring.buf_info[eop_idx].m = NULL;
 311
 312         while (txq->cmd_ring.next2comp != eop_idx) {
 313                 /* no out-of-order completion */
 314                 RTE_ASSERT(txq->cmd_ring.base[txq->cmd_ring.next2comp].txd.cq == 0);
 315                 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 316                 completed++;
 317         }
 318
 319         /* Mark the txd for which tcd was generated as completed */
 320         vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 321
 322         return completed + 1;
 323 }
 324
 325 static void
 326 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
 327 {
 328         int completed = 0;
 329         vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
 330         struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
 331                 (comp_ring->base + comp_ring->next2proc);
 332
 333         while (tcd->gen == comp_ring->gen) {
 334                 completed += vmxnet3_unmap_pkt(tcd->txdIdx, txq);
 335
 336                 vmxnet3_comp_ring_adv_next2proc(comp_ring);
 337                 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
 338                                                     comp_ring->next2proc);
 339         }
 340
 341         PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
 342 }
 343
 344 uint16_t
 345 vmxnet3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
 346         uint16_t nb_pkts)
 347 {
 348         int32_t ret;
 349         uint32_t i;
 350         uint64_t ol_flags;
 351         struct rte_mbuf *m;
 352
 353         for (i = 0; i != nb_pkts; i++) {
 354                 m = tx_pkts[i];
 355                 ol_flags = m->ol_flags;
 356
 357                 /* Non-TSO packet cannot occupy more than
 358                  * VMXNET3_MAX_TXD_PER_PKT TX descriptors.
 359                  */
 360                 if ((ol_flags & PKT_TX_TCP_SEG) == 0 &&
 361                                 m->nb_segs > VMXNET3_MAX_TXD_PER_PKT) {
 362                         rte_errno = -EINVAL;
 363                         return i;
 364                 }
 365
 366                 /* check that only supported TX offloads are requested. */
 367                 if ((ol_flags & VMXNET3_TX_OFFLOAD_NOTSUP_MASK) != 0 ||
 368                                 (ol_flags & PKT_TX_L4_MASK) ==
 369                                 PKT_TX_SCTP_CKSUM) {
 370                         rte_errno = -ENOTSUP;
 371                         return i;
 372                 }
 373
 374 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
 375                 ret = rte_validate_tx_offload(m);
 376                 if (ret != 0) {
 377                         rte_errno = ret;
 378                         return i;
 379                 }
 380 #endif
 381                 ret = rte_net_intel_cksum_prepare(m);
 382                 if (ret != 0) {
 383                         rte_errno = ret;
 384                         return i;
 385                 }
 386         }
 387
 388         return i;
 389 }
 390
 391 uint16_t
 392 vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 393                   uint16_t nb_pkts)
 394 {
 395         uint16_t nb_tx;
 396         vmxnet3_tx_queue_t *txq = tx_queue;
 397         struct vmxnet3_hw *hw = txq->hw;
 398         Vmxnet3_TxQueueCtrl *txq_ctrl = &txq->shared->ctrl;
 399         uint32_t deferred = rte_le_to_cpu_32(txq_ctrl->txNumDeferred);
 400
 401         if (unlikely(txq->stopped)) {
 402                 PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
 403                 return 0;
 404         }
 405
 406         /* Free up the comp_descriptors aggressively */
 407         vmxnet3_tq_tx_complete(txq);
 408
 409         nb_tx = 0;
 410         while (nb_tx < nb_pkts) {
 411                 Vmxnet3_GenericDesc *gdesc;
 412                 vmxnet3_buf_info_t *tbi;
 413                 uint32_t first2fill, avail, dw2;
 414                 struct rte_mbuf *txm = tx_pkts[nb_tx];
 415                 struct rte_mbuf *m_seg = txm;
 416                 int copy_size = 0;
 417                 bool tso = (txm->ol_flags & PKT_TX_TCP_SEG) != 0;
 418                 /* # of descriptors needed for a packet. */
 419                 unsigned count = txm->nb_segs;
 420
 421                 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 422                 if (count > avail) {
 423                         /* Is command ring full? */
 424                         if (unlikely(avail == 0)) {
 425                                 PMD_TX_LOG(DEBUG, "No free ring descriptors");
 426                                 txq->stats.tx_ring_full++;
 427                                 txq->stats.drop_total += (nb_pkts - nb_tx);
 428                                 break;
 429                         }
 430
 431                         /* Command ring is not full but cannot handle the
 432                          * multi-segmented packet. Let's try the next packet
 433                          * in this case.
 434                          */
 435                         PMD_TX_LOG(DEBUG, "Running out of ring descriptors "
 436                                    "(avail %d needed %d)", avail, count);
 437                         txq->stats.drop_total++;
 438                         if (tso)
 439                                 txq->stats.drop_tso++;
 440                         rte_pktmbuf_free(txm);
 441                         nb_tx++;
 442                         continue;
 443                 }
 444
 445                 /* Drop non-TSO packet that is excessively fragmented */
 446                 if (unlikely(!tso && count > VMXNET3_MAX_TXD_PER_PKT)) {
 447                         PMD_TX_LOG(ERR, "Non-TSO packet cannot occupy more than %d tx "
 448                                    "descriptors. Packet dropped.", VMXNET3_MAX_TXD_PER_PKT);
 449                         txq->stats.drop_too_many_segs++;
 450                         txq->stats.drop_total++;
 451                         rte_pktmbuf_free(txm);
 452                         nb_tx++;
 453                         continue;
 454                 }
 455
 456                 if (txm->nb_segs == 1 &&
 457                     rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) {
 458                         struct Vmxnet3_TxDataDesc *tdd;
 459
 460                         /* Skip empty packets */
 461                         if (unlikely(rte_pktmbuf_pkt_len(txm) == 0)) {
 462                                 txq->stats.drop_total++;
 463                                 rte_pktmbuf_free(txm);
 464                                 nb_tx++;
 465                                 continue;
 466                         }
 467
 468                         tdd = (struct Vmxnet3_TxDataDesc *)
 469                                 ((uint8 *)txq->data_ring.base +
 470                                  txq->cmd_ring.next2fill *
 471                                  txq->txdata_desc_size);
 472                         copy_size = rte_pktmbuf_pkt_len(txm);
 473                         rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size);
 474                 }
 475
 476                 /* use the previous gen bit for the SOP desc */
 477                 dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
 478                 first2fill = txq->cmd_ring.next2fill;
 479                 do {
 480                         /* Remember the transmit buffer for cleanup */
 481                         tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
 482
 483                         /* NB: the following assumes that VMXNET3 maximum
 484                          * transmit buffer size (16K) is greater than
 485                          * maximum size of mbuf segment size.
 486                          */
 487                         gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill;
 488
 489                         /* Skip empty segments */
 490                         if (unlikely(m_seg->data_len == 0))
 491                                 continue;
 492
 493                         if (copy_size) {
 494                                 uint64 offset =
 495                                         (uint64)txq->cmd_ring.next2fill *
 496                                                         txq->txdata_desc_size;
 497                                 gdesc->txd.addr =
 498                                         rte_cpu_to_le_64(txq->data_ring.basePA +
 499                                                          offset);
 500                         } else {
 501                                 gdesc->txd.addr = rte_mbuf_data_iova(m_seg);
 502                         }
 503
 504                         gdesc->dword[2] = dw2 | m_seg->data_len;
 505                         gdesc->dword[3] = 0;
 506
 507                         /* move to the next2fill descriptor */
 508                         vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
 509
 510                         /* use the right gen for non-SOP desc */
 511                         dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 512                 } while ((m_seg = m_seg->next) != NULL);
 513
 514                 /* set the last buf_info for the pkt */
 515                 tbi->m = txm;
 516                 /* Update the EOP descriptor */
 517                 gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ;
 518
 519                 /* Add VLAN tag if present */
 520                 gdesc = txq->cmd_ring.base + first2fill;
 521                 if (txm->ol_flags & PKT_TX_VLAN_PKT) {
 522                         gdesc->txd.ti = 1;
 523                         gdesc->txd.tci = txm->vlan_tci;
 524                 }
 525
 526                 if (tso) {
 527                         uint16_t mss = txm->tso_segsz;
 528
 529                         RTE_ASSERT(mss > 0);
 530
 531                         gdesc->txd.hlen = txm->l2_len + txm->l3_len + txm->l4_len;
 532                         gdesc->txd.om = VMXNET3_OM_TSO;
 533                         gdesc->txd.msscof = mss;
 534
 535                         deferred += (rte_pktmbuf_pkt_len(txm) - gdesc->txd.hlen + mss - 1) / mss;
 536                 } else if (txm->ol_flags & PKT_TX_L4_MASK) {
 537                         gdesc->txd.om = VMXNET3_OM_CSUM;
 538                         gdesc->txd.hlen = txm->l2_len + txm->l3_len;
 539
 540                         switch (txm->ol_flags & PKT_TX_L4_MASK) {
 541                         case PKT_TX_TCP_CKSUM:
 542                                 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct tcp_hdr, cksum);
 543                                 break;
 544                         case PKT_TX_UDP_CKSUM:
 545                                 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct udp_hdr, dgram_cksum);
 546                                 break;
 547                         default:
 548                                 PMD_TX_LOG(WARNING, "requested cksum offload not supported %#llx",
 549                                            txm->ol_flags & PKT_TX_L4_MASK);
 550                                 abort();
 551                         }
 552                         deferred++;
 553                 } else {
 554                         gdesc->txd.hlen = 0;
 555                         gdesc->txd.om = VMXNET3_OM_NONE;
 556                         gdesc->txd.msscof = 0;
 557                         deferred++;
 558                 }
 559
 560                 /* flip the GEN bit on the SOP */
 561                 rte_compiler_barrier();
 562                 gdesc->dword[2] ^= VMXNET3_TXD_GEN;
 563
 564                 txq_ctrl->txNumDeferred = rte_cpu_to_le_32(deferred);
 565                 nb_tx++;
 566         }
 567
 568         PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", rte_le_to_cpu_32(txq_ctrl->txThreshold));
 569
 570         if (deferred >= rte_le_to_cpu_32(txq_ctrl->txThreshold)) {
 571                 txq_ctrl->txNumDeferred = 0;
 572                 /* Notify vSwitch that packets are available. */
 573                 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
 574                                        txq->cmd_ring.next2fill);
 575         }
 576
 577         return nb_tx;
 578 }
 579
 580 static inline void
 581 vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id,
 582                    struct rte_mbuf *mbuf)
 583 {
 584         uint32_t val;
 585         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 586         struct Vmxnet3_RxDesc *rxd =
 587                 (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
 588         vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
 589
 590         if (ring_id == 0) {
 591                 /* Usually: One HEAD type buf per packet
 592                  * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
 593                  * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
 594                  */
 595
 596                 /* We use single packet buffer so all heads here */
 597                 val = VMXNET3_RXD_BTYPE_HEAD;
 598         } else {
 599                 /* All BODY type buffers for 2nd ring */
 600                 val = VMXNET3_RXD_BTYPE_BODY;
 601         }
 602
 603         /*
 604          * Load mbuf pointer into buf_info[ring_size]
 605          * buf_info structure is equivalent to cookie for virtio-virtqueue
 606          */
 607         buf_info->m = mbuf;
 608         buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM);
 609         buf_info->bufPA = rte_mbuf_data_iova_default(mbuf);
 610
 611         /* Load Rx Descriptor with the buffer's GPA */
 612         rxd->addr = buf_info->bufPA;
 613
 614         /* After this point rxd->addr MUST not be NULL */
 615         rxd->btype = val;
 616         rxd->len = buf_info->len;
 617         /* Flip gen bit at the end to change ownership */
 618         rxd->gen = ring->gen;
 619
 620         vmxnet3_cmd_ring_adv_next2fill(ring);
 621 }
 622 /*
 623  *  Allocates mbufs and clusters. Post rx descriptors with buffer details
 624  *  so that device can receive packets in those buffers.
 625  *  Ring layout:
 626  *      Among the two rings, 1st ring contains buffers of type 0 and type 1.
 627  *      bufs_per_pkt is set such that for non-LRO cases all the buffers required
 628  *      by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
 629  *      2nd ring contains buffers of type 1 alone. Second ring mostly be used
 630  *      only for LRO.
 631  */
 632 static int
 633 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id)
 634 {
 635         int err = 0;
 636         uint32_t i = 0;
 637         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 638
 639         while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
 640                 struct rte_mbuf *mbuf;
 641
 642                 /* Allocate blank mbuf for the current Rx Descriptor */
 643                 mbuf = rte_mbuf_raw_alloc(rxq->mp);
 644                 if (unlikely(mbuf == NULL)) {
 645                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 646                         rxq->stats.rx_buf_alloc_failure++;
 647                         err = ENOMEM;
 648                         break;
 649                 }
 650
 651                 vmxnet3_renew_desc(rxq, ring_id, mbuf);
 652                 i++;
 653         }
 654
 655         /* Return error only if no buffers are posted at present */
 656         if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1))
 657                 return -err;
 658         else
 659                 return i;
 660 }
 661
 662 /* MSS not provided by vmxnet3, guess one with available information */
 663 static uint16_t
 664 vmxnet3_guess_mss(struct vmxnet3_hw *hw, const Vmxnet3_RxCompDesc *rcd,
 665                 struct rte_mbuf *rxm)
 666 {
 667         uint32_t hlen, slen;
 668         struct ipv4_hdr *ipv4_hdr;
 669         struct ipv6_hdr *ipv6_hdr;
 670         struct tcp_hdr *tcp_hdr;
 671         char *ptr;
 672
 673         RTE_ASSERT(rcd->tcp);
 674
 675         ptr = rte_pktmbuf_mtod(rxm, char *);
 676         slen = rte_pktmbuf_data_len(rxm);
 677         hlen = sizeof(struct ether_hdr);
 678
 679         if (rcd->v4) {
 680                 if (unlikely(slen < hlen + sizeof(struct ipv4_hdr)))
 681                         return hw->mtu - sizeof(struct ipv4_hdr)
 682                                         - sizeof(struct tcp_hdr);
 683
 684                 ipv4_hdr = (struct ipv4_hdr *)(ptr + hlen);
 685                 hlen += (ipv4_hdr->version_ihl & IPV4_HDR_IHL_MASK) *
 686                                 IPV4_IHL_MULTIPLIER;
 687         } else if (rcd->v6) {
 688                 if (unlikely(slen < hlen + sizeof(struct ipv6_hdr)))
 689                         return hw->mtu - sizeof(struct ipv6_hdr) -
 690                                         sizeof(struct tcp_hdr);
 691
 692                 ipv6_hdr = (struct ipv6_hdr *)(ptr + hlen);
 693                 hlen += sizeof(struct ipv6_hdr);
 694                 if (unlikely(ipv6_hdr->proto != IPPROTO_TCP)) {
 695                         int frag;
 696
 697                         rte_net_skip_ip6_ext(ipv6_hdr->proto, rxm,
 698                                         &hlen, &frag);
 699                 }
 700         }
 701
 702         if (unlikely(slen < hlen + sizeof(struct tcp_hdr)))
 703                 return hw->mtu - hlen - sizeof(struct tcp_hdr) +
 704                                 sizeof(struct ether_hdr);
 705
 706         tcp_hdr = (struct tcp_hdr *)(ptr + hlen);
 707         hlen += (tcp_hdr->data_off & 0xf0) >> 2;
 708
 709         if (rxm->udata64 > 1)
 710                 return (rte_pktmbuf_pkt_len(rxm) - hlen +
 711                                 rxm->udata64 - 1) / rxm->udata64;
 712         else
 713                 return hw->mtu - hlen + sizeof(struct ether_hdr);
 714 }
 715
 716 /* Receive side checksum and other offloads */
 717 static inline void
 718 vmxnet3_rx_offload(struct vmxnet3_hw *hw, const Vmxnet3_RxCompDesc *rcd,
 719                 struct rte_mbuf *rxm, const uint8_t sop)
 720 {
 721         uint64_t ol_flags = rxm->ol_flags;
 722         uint32_t packet_type = rxm->packet_type;
 723
 724         /* Offloads set in sop */
 725         if (sop) {
 726                 /* Set packet type */
 727                 packet_type |= RTE_PTYPE_L2_ETHER;
 728
 729                 /* Check large packet receive */
 730                 if (VMXNET3_VERSION_GE_2(hw) &&
 731                     rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
 732                         const Vmxnet3_RxCompDescExt *rcde =
 733                                         (const Vmxnet3_RxCompDescExt *)rcd;
 734
 735                         rxm->tso_segsz = rcde->mss;
 736                         rxm->udata64 = rcde->segCnt;
 737                         ol_flags |= PKT_RX_LRO;
 738                 }
 739         } else { /* Offloads set in eop */
 740                 /* Check for RSS */
 741                 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE) {
 742                         ol_flags |= PKT_RX_RSS_HASH;
 743                         rxm->hash.rss = rcd->rssHash;
 744                 }
 745
 746                 /* Check for hardware stripped VLAN tag */
 747                 if (rcd->ts) {
 748                         ol_flags |= (PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED);
 749                         rxm->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci);
 750                 }
 751
 752                 /* Check packet type, checksum errors, etc. */
 753                 if (rcd->cnc) {
 754                         ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
 755                 } else {
 756                         if (rcd->v4) {
 757                                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
 758
 759                                 if (rcd->ipc)
 760                                         ol_flags |= PKT_RX_IP_CKSUM_GOOD;
 761                                 else
 762                                         ol_flags |= PKT_RX_IP_CKSUM_BAD;
 763
 764                                 if (rcd->tuc) {
 765                                         ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 766                                         if (rcd->tcp)
 767                                                 packet_type |= RTE_PTYPE_L4_TCP;
 768                                         else
 769                                                 packet_type |= RTE_PTYPE_L4_UDP;
 770                                 } else {
 771                                         if (rcd->tcp) {
 772                                                 packet_type |= RTE_PTYPE_L4_TCP;
 773                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 774                                         } else if (rcd->udp) {
 775                                                 packet_type |= RTE_PTYPE_L4_UDP;
 776                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 777                                         }
 778                                 }
 779                         } else if (rcd->v6) {
 780                                 packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
 781
 782                                 if (rcd->tuc) {
 783                                         ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 784                                         if (rcd->tcp)
 785                                                 packet_type |= RTE_PTYPE_L4_TCP;
 786                                         else
 787                                                 packet_type |= RTE_PTYPE_L4_UDP;
 788                                 } else {
 789                                         if (rcd->tcp) {
 790                                                 packet_type |= RTE_PTYPE_L4_TCP;
 791                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 792                                         } else if (rcd->udp) {
 793                                                 packet_type |= RTE_PTYPE_L4_UDP;
 794                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 795                                         }
 796                                 }
 797                         } else {
 798                                 packet_type |= RTE_PTYPE_UNKNOWN;
 799                         }
 800
 801                         /* Old variants of vmxnet3 do not provide MSS */
 802                         if ((ol_flags & PKT_RX_LRO) && rxm->tso_segsz == 0)
 803                                 rxm->tso_segsz = vmxnet3_guess_mss(hw,
 804                                                 rcd, rxm);
 805                 }
 806         }
 807
 808         rxm->ol_flags = ol_flags;
 809         rxm->packet_type = packet_type;
 810 }
 811
 812 /*
 813  * Process the Rx Completion Ring of given vmxnet3_rx_queue
 814  * for nb_pkts burst and return the number of packets received
 815  */
 816 uint16_t
 817 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 818 {
 819         uint16_t nb_rx;
 820         uint32_t nb_rxd, idx;
 821         uint8_t ring_idx;
 822         vmxnet3_rx_queue_t *rxq;
 823         Vmxnet3_RxCompDesc *rcd;
 824         vmxnet3_buf_info_t *rbi;
 825         Vmxnet3_RxDesc *rxd;
 826         struct rte_mbuf *rxm = NULL;
 827         struct vmxnet3_hw *hw;
 828
 829         nb_rx = 0;
 830         ring_idx = 0;
 831         nb_rxd = 0;
 832         idx = 0;
 833
 834         rxq = rx_queue;
 835         hw = rxq->hw;
 836
 837         rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 838
 839         if (unlikely(rxq->stopped)) {
 840                 PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
 841                 return 0;
 842         }
 843
 844         while (rcd->gen == rxq->comp_ring.gen) {
 845                 struct rte_mbuf *newm;
 846
 847                 if (nb_rx >= nb_pkts)
 848                         break;
 849
 850                 newm = rte_mbuf_raw_alloc(rxq->mp);
 851                 if (unlikely(newm == NULL)) {
 852                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 853                         rxq->stats.rx_buf_alloc_failure++;
 854                         break;
 855                 }
 856
 857                 idx = rcd->rxdIdx;
 858                 ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID);
 859                 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
 860                 RTE_SET_USED(rxd); /* used only for assert when enabled */
 861                 rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
 862
 863                 PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
 864
 865                 RTE_ASSERT(rcd->len <= rxd->len);
 866                 RTE_ASSERT(rbi->m);
 867
 868                 /* Get the packet buffer pointer from buf_info */
 869                 rxm = rbi->m;
 870
 871                 /* Clear descriptor associated buf_info to be reused */
 872                 rbi->m = NULL;
 873                 rbi->bufPA = 0;
 874
 875                 /* Update the index that we received a packet */
 876                 rxq->cmd_ring[ring_idx].next2comp = idx;
 877
 878                 /* For RCD with EOP set, check if there is frame error */
 879                 if (unlikely(rcd->eop && rcd->err)) {
 880                         rxq->stats.drop_total++;
 881                         rxq->stats.drop_err++;
 882
 883                         if (!rcd->fcs) {
 884                                 rxq->stats.drop_fcs++;
 885                                 PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
 886                         }
 887                         PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
 888                                    (int)(rcd - (struct Vmxnet3_RxCompDesc *)
 889                                          rxq->comp_ring.base), rcd->rxdIdx);
 890                         rte_pktmbuf_free_seg(rxm);
 891                         if (rxq->start_seg) {
 892                                 struct rte_mbuf *start = rxq->start_seg;
 893
 894                                 rxq->start_seg = NULL;
 895                                 rte_pktmbuf_free(start);
 896                         }
 897                         goto rcd_done;
 898                 }
 899
 900                 /* Initialize newly received packet buffer */
 901                 rxm->port = rxq->port_id;
 902                 rxm->nb_segs = 1;
 903                 rxm->next = NULL;
 904                 rxm->pkt_len = (uint16_t)rcd->len;
 905                 rxm->data_len = (uint16_t)rcd->len;
 906                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 907                 rxm->ol_flags = 0;
 908                 rxm->vlan_tci = 0;
 909                 rxm->packet_type = 0;
 910
 911                 /*
 912                  * If this is the first buffer of the received packet,
 913                  * set the pointer to the first mbuf of the packet
 914                  * Otherwise, update the total length and the number of segments
 915                  * of the current scattered packet, and update the pointer to
 916                  * the last mbuf of the current packet.
 917                  */
 918                 if (rcd->sop) {
 919                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
 920
 921                         if (unlikely(rcd->len == 0)) {
 922                                 RTE_ASSERT(rcd->eop);
 923
 924                                 PMD_RX_LOG(DEBUG,
 925                                            "Rx buf was skipped. rxring[%d][%d])",
 926                                            ring_idx, idx);
 927                                 rte_pktmbuf_free_seg(rxm);
 928                                 goto rcd_done;
 929                         }
 930
 931                         if (vmxnet3_rx_data_ring(hw, rcd->rqID)) {
 932                                 uint8_t *rdd = rxq->data_ring.base +
 933                                         idx * rxq->data_desc_size;
 934
 935                                 RTE_ASSERT(VMXNET3_VERSION_GE_3(hw));
 936                                 rte_memcpy(rte_pktmbuf_mtod(rxm, char *),
 937                                            rdd, rcd->len);
 938                         }
 939
 940                         rxq->start_seg = rxm;
 941                         rxq->last_seg = rxm;
 942                         vmxnet3_rx_offload(hw, rcd, rxm, 1);
 943                 } else {
 944                         struct rte_mbuf *start = rxq->start_seg;
 945
 946                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_BODY);
 947
 948                         if (rxm->data_len) {
 949                                 start->pkt_len += rxm->data_len;
 950                                 start->nb_segs++;
 951
 952                                 rxq->last_seg->next = rxm;
 953                                 rxq->last_seg = rxm;
 954                         } else {
 955                                 rte_pktmbuf_free_seg(rxm);
 956                         }
 957                 }
 958
 959                 if (rcd->eop) {
 960                         struct rte_mbuf *start = rxq->start_seg;
 961
 962                         vmxnet3_rx_offload(hw, rcd, start, 0);
 963                         rx_pkts[nb_rx++] = start;
 964                         rxq->start_seg = NULL;
 965                 }
 966
 967 rcd_done:
 968                 rxq->cmd_ring[ring_idx].next2comp = idx;
 969                 VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp,
 970                                           rxq->cmd_ring[ring_idx].size);
 971
 972                 /* It's time to renew descriptors */
 973                 vmxnet3_renew_desc(rxq, ring_idx, newm);
 974                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
 975                         VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
 976                                                rxq->cmd_ring[ring_idx].next2fill);
 977                 }
 978
 979                 /* Advance to the next descriptor in comp_ring */
 980                 vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
 981
 982                 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 983                 nb_rxd++;
 984                 if (nb_rxd > rxq->cmd_ring[0].size) {
 985                         PMD_RX_LOG(ERR, "Used up quota of receiving packets,"
 986                                    " relinquish control.");
 987                         break;
 988                 }
 989         }
 990
 991         if (unlikely(nb_rxd == 0)) {
 992                 uint32_t avail;
 993                 for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
 994                         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[ring_idx]);
 995                         if (unlikely(avail > 0)) {
 996                                 /* try to alloc new buf and renew descriptors */
 997                                 vmxnet3_post_rx_bufs(rxq, ring_idx);
 998                         }
 999                 }
1000                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1001                         for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
1002                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1003                                                        rxq->cmd_ring[ring_idx].next2fill);
1004                         }
1005                 }
1006         }
1007
1008         return nb_rx;
1009 }
1010
1011 int
1012 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev,
1013                            uint16_t queue_idx,
1014                            uint16_t nb_desc,
1015                            unsigned int socket_id,
1016                            const struct rte_eth_txconf *tx_conf __rte_unused)
1017 {
1018         struct vmxnet3_hw *hw = dev->data->dev_private;
1019         const struct rte_memzone *mz;
1020         struct vmxnet3_tx_queue *txq;
1021         struct vmxnet3_cmd_ring *ring;
1022         struct vmxnet3_comp_ring *comp_ring;
1023         struct vmxnet3_data_ring *data_ring;
1024         int size;
1025
1026         PMD_INIT_FUNC_TRACE();
1027
1028         txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue),
1029                           RTE_CACHE_LINE_SIZE);
1030         if (txq == NULL) {
1031                 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure");
1032                 return -ENOMEM;
1033         }
1034
1035         txq->queue_id = queue_idx;
1036         txq->port_id = dev->data->port_id;
1037         txq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
1038         txq->hw = hw;
1039         txq->qid = queue_idx;
1040         txq->stopped = TRUE;
1041         txq->txdata_desc_size = hw->txdata_desc_size;
1042
1043         ring = &txq->cmd_ring;
1044         comp_ring = &txq->comp_ring;
1045         data_ring = &txq->data_ring;
1046
1047         /* Tx vmxnet ring length should be between 512-4096 */
1048         if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) {
1049                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u",
1050                              VMXNET3_DEF_TX_RING_SIZE);
1051                 return -EINVAL;
1052         } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) {
1053                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u",
1054                              VMXNET3_TX_RING_MAX_SIZE);
1055                 return -EINVAL;
1056         } else {
1057                 ring->size = nb_desc;
1058                 ring->size &= ~VMXNET3_RING_SIZE_MASK;
1059         }
1060         comp_ring->size = data_ring->size = ring->size;
1061
1062         /* Tx vmxnet rings structure initialization*/
1063         ring->next2fill = 0;
1064         ring->next2comp = 0;
1065         ring->gen = VMXNET3_INIT_GEN;
1066         comp_ring->next2proc = 0;
1067         comp_ring->gen = VMXNET3_INIT_GEN;
1068
1069         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
1070         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
1071         size += txq->txdata_desc_size * data_ring->size;
1072
1073         mz = rte_eth_dma_zone_reserve(dev, "txdesc", queue_idx, size,
1074                                       VMXNET3_RING_BA_ALIGN, socket_id);
1075         if (mz == NULL) {
1076                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1077                 return -ENOMEM;
1078         }
1079         txq->mz = mz;
1080         memset(mz->addr, 0, mz->len);
1081
1082         /* cmd_ring initialization */
1083         ring->base = mz->addr;
1084         ring->basePA = mz->iova;
1085
1086         /* comp_ring initialization */
1087         comp_ring->base = ring->base + ring->size;
1088         comp_ring->basePA = ring->basePA +
1089                 (sizeof(struct Vmxnet3_TxDesc) * ring->size);
1090
1091         /* data_ring initialization */
1092         data_ring->base = (Vmxnet3_TxDataDesc *)(comp_ring->base + comp_ring->size);
1093         data_ring->basePA = comp_ring->basePA +
1094                         (sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size);
1095
1096         /* cmd_ring0 buf_info allocation */
1097         ring->buf_info = rte_zmalloc("tx_ring_buf_info",
1098                                      ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE);
1099         if (ring->buf_info == NULL) {
1100                 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure");
1101                 return -ENOMEM;
1102         }
1103
1104         /* Update the data portion with txq */
1105         dev->data->tx_queues[queue_idx] = txq;
1106
1107         return 0;
1108 }
1109
1110 int
1111 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
1112                            uint16_t queue_idx,
1113                            uint16_t nb_desc,
1114                            unsigned int socket_id,
1115                            __rte_unused const struct rte_eth_rxconf *rx_conf,
1116                            struct rte_mempool *mp)
1117 {
1118         const struct rte_memzone *mz;
1119         struct vmxnet3_rx_queue *rxq;
1120         struct vmxnet3_hw *hw = dev->data->dev_private;
1121         struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
1122         struct vmxnet3_comp_ring *comp_ring;
1123         struct vmxnet3_rx_data_ring *data_ring;
1124         int size;
1125         uint8_t i;
1126         char mem_name[32];
1127
1128         PMD_INIT_FUNC_TRACE();
1129
1130         rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue),
1131                           RTE_CACHE_LINE_SIZE);
1132         if (rxq == NULL) {
1133                 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure");
1134                 return -ENOMEM;
1135         }
1136
1137         rxq->mp = mp;
1138         rxq->queue_id = queue_idx;
1139         rxq->port_id = dev->data->port_id;
1140         rxq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
1141         rxq->hw = hw;
1142         rxq->qid1 = queue_idx;
1143         rxq->qid2 = queue_idx + hw->num_rx_queues;
1144         rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues;
1145         rxq->data_desc_size = hw->rxdata_desc_size;
1146         rxq->stopped = TRUE;
1147
1148         ring0 = &rxq->cmd_ring[0];
1149         ring1 = &rxq->cmd_ring[1];
1150         comp_ring = &rxq->comp_ring;
1151         data_ring = &rxq->data_ring;
1152
1153         /* Rx vmxnet rings length should be between 256-4096 */
1154         if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
1155                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256");
1156                 return -EINVAL;
1157         } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) {
1158                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096");
1159                 return -EINVAL;
1160         } else {
1161                 ring0->size = nb_desc;
1162                 ring0->size &= ~VMXNET3_RING_SIZE_MASK;
1163                 ring1->size = ring0->size;
1164         }
1165
1166         comp_ring->size = ring0->size + ring1->size;
1167         data_ring->size = ring0->size;
1168
1169         /* Rx vmxnet rings structure initialization */
1170         ring0->next2fill = 0;
1171         ring1->next2fill = 0;
1172         ring0->next2comp = 0;
1173         ring1->next2comp = 0;
1174         ring0->gen = VMXNET3_INIT_GEN;
1175         ring1->gen = VMXNET3_INIT_GEN;
1176         comp_ring->next2proc = 0;
1177         comp_ring->gen = VMXNET3_INIT_GEN;
1178
1179         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
1180         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1181         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size)
1182                 size += rxq->data_desc_size * data_ring->size;
1183
1184         mz = rte_eth_dma_zone_reserve(dev, "rxdesc", queue_idx, size,
1185                                       VMXNET3_RING_BA_ALIGN, socket_id);
1186         if (mz == NULL) {
1187                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1188                 return -ENOMEM;
1189         }
1190         rxq->mz = mz;
1191         memset(mz->addr, 0, mz->len);
1192
1193         /* cmd_ring0 initialization */
1194         ring0->base = mz->addr;
1195         ring0->basePA = mz->iova;
1196
1197         /* cmd_ring1 initialization */
1198         ring1->base = ring0->base + ring0->size;
1199         ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size;
1200
1201         /* comp_ring initialization */
1202         comp_ring->base = ring1->base + ring1->size;
1203         comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
1204                 ring1->size;
1205
1206         /* data_ring initialization */
1207         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) {
1208                 data_ring->base =
1209                         (uint8_t *)(comp_ring->base + comp_ring->size);
1210                 data_ring->basePA = comp_ring->basePA +
1211                         sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1212         }
1213
1214         /* cmd_ring0-cmd_ring1 buf_info allocation */
1215         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {
1216
1217                 ring = &rxq->cmd_ring[i];
1218                 ring->rid = i;
1219                 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i);
1220
1221                 ring->buf_info = rte_zmalloc(mem_name,
1222                                              ring->size * sizeof(vmxnet3_buf_info_t),
1223                                              RTE_CACHE_LINE_SIZE);
1224                 if (ring->buf_info == NULL) {
1225                         PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure");
1226                         return -ENOMEM;
1227                 }
1228         }
1229
1230         /* Update the data portion with rxq */
1231         dev->data->rx_queues[queue_idx] = rxq;
1232
1233         return 0;
1234 }
1235
1236 /*
1237  * Initializes Receive Unit
1238  * Load mbufs in rx queue in advance
1239  */
1240 int
1241 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
1242 {
1243         struct vmxnet3_hw *hw = dev->data->dev_private;
1244
1245         int i, ret;
1246         uint8_t j;
1247
1248         PMD_INIT_FUNC_TRACE();
1249
1250         for (i = 0; i < hw->num_rx_queues; i++) {
1251                 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
1252
1253                 for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) {
1254                         /* Passing 0 as alloc_num will allocate full ring */
1255                         ret = vmxnet3_post_rx_bufs(rxq, j);
1256                         if (ret <= 0) {
1257                                 PMD_INIT_LOG(ERR,
1258                                              "ERROR: Posting Rxq: %d buffers ring: %d",
1259                                              i, j);
1260                                 return -ret;
1261                         }
1262                         /*
1263                          * Updating device with the index:next2fill to fill the
1264                          * mbufs for coming packets.
1265                          */
1266                         if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1267                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1268                                                        rxq->cmd_ring[j].next2fill);
1269                         }
1270                 }
1271                 rxq->stopped = FALSE;
1272                 rxq->start_seg = NULL;
1273         }
1274
1275         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1276                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
1277
1278                 txq->stopped = FALSE;
1279         }
1280
1281         return 0;
1282 }
1283
1284 static uint8_t rss_intel_key[40] = {
1285         0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
1286         0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
1287         0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
1288         0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
1289         0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
1290 };
1291
1292 /*
1293  * Configure RSS feature
1294  */
1295 int
1296 vmxnet3_rss_configure(struct rte_eth_dev *dev)
1297 {
1298         struct vmxnet3_hw *hw = dev->data->dev_private;
1299         struct VMXNET3_RSSConf *dev_rss_conf;
1300         struct rte_eth_rss_conf *port_rss_conf;
1301         uint64_t rss_hf;
1302         uint8_t i, j;
1303
1304         PMD_INIT_FUNC_TRACE();
1305
1306         dev_rss_conf = hw->rss_conf;
1307         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1308
1309         /* loading hashFunc */
1310         dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
1311         /* loading hashKeySize */
1312         dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE;
1313         /* loading indTableSize: Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/
1314         dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4);
1315
1316         if (port_rss_conf->rss_key == NULL) {
1317                 /* Default hash key */
1318                 port_rss_conf->rss_key = rss_intel_key;
1319         }
1320
1321         /* loading hashKey */
1322         memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key,
1323                dev_rss_conf->hashKeySize);
1324
1325         /* loading indTable */
1326         for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) {
1327                 if (j == dev->data->nb_rx_queues)
1328                         j = 0;
1329                 dev_rss_conf->indTable[i] = j;
1330         }
1331
1332         /* loading hashType */
1333         dev_rss_conf->hashType = 0;
1334         rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL;
1335         if (rss_hf & ETH_RSS_IPV4)
1336                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4;
1337         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1338                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4;
1339         if (rss_hf & ETH_RSS_IPV6)
1340                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6;
1341         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1342                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
1343
1344         return VMXNET3_SUCCESS;
1345 }