drivers/net/bnx2x/bnx2x_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
   3  * Copyright (c) 2015-2018 Cavium Inc.
   4  * All rights reserved.
   5  * www.cavium.com
   6  */
   7
   8 #include "bnx2x.h"
   9 #include "bnx2x_rxtx.h"
  10
  11 static const struct rte_memzone *
  12 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
  13                       uint16_t queue_id, uint32_t ring_size, int socket_id)
  14 {
  15         char z_name[RTE_MEMZONE_NAMESIZE];
  16         const struct rte_memzone *mz;
  17
  18         snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
  19                         dev->device->driver->name, ring_name,
  20                         dev->data->port_id, queue_id);
  21
  22         mz = rte_memzone_lookup(z_name);
  23         if (mz)
  24                 return mz;
  25
  26         return rte_memzone_reserve_aligned(z_name, ring_size, socket_id,
  27                         RTE_MEMZONE_IOVA_CONTIG, BNX2X_PAGE_SIZE);
  28 }
  29
  30 static void
  31 bnx2x_rx_queue_release(struct bnx2x_rx_queue *rx_queue)
  32 {
  33         uint16_t i;
  34         struct rte_mbuf **sw_ring;
  35
  36         if (NULL != rx_queue) {
  37
  38                 sw_ring = rx_queue->sw_ring;
  39                 if (NULL != sw_ring) {
  40                         for (i = 0; i < rx_queue->nb_rx_desc; i++) {
  41                                 if (NULL != sw_ring[i])
  42                                         rte_pktmbuf_free(sw_ring[i]);
  43                         }
  44                         rte_free(sw_ring);
  45                 }
  46                 rte_free(rx_queue);
  47         }
  48 }
  49
  50 void
  51 bnx2x_dev_rx_queue_release(void *rxq)
  52 {
  53         bnx2x_rx_queue_release(rxq);
  54 }
  55
  56 int
  57 bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
  58                        uint16_t queue_idx,
  59                        uint16_t nb_desc,
  60                        unsigned int socket_id,
  61                        __rte_unused const struct rte_eth_rxconf *rx_conf,
  62                        struct rte_mempool *mp)
  63 {
  64         uint16_t j, idx;
  65         const struct rte_memzone *dma;
  66         struct bnx2x_rx_queue *rxq;
  67         uint32_t dma_size;
  68         struct rte_mbuf *mbuf;
  69         struct bnx2x_softc *sc = dev->data->dev_private;
  70         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
  71         struct eth_rx_cqe_next_page *nextpg;
  72         rte_iova_t *rx_bd;
  73         rte_iova_t busaddr;
  74
  75         /* First allocate the rx queue data structure */
  76         rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
  77                                  RTE_CACHE_LINE_SIZE, socket_id);
  78         if (NULL == rxq) {
  79                 PMD_INIT_LOG(ERR, "rte_zmalloc for rxq failed!");
  80                 return -ENOMEM;
  81         }
  82         rxq->sc = sc;
  83         rxq->mb_pool = mp;
  84         rxq->queue_id = queue_idx;
  85         rxq->port_id = dev->data->port_id;
  86
  87         rxq->nb_rx_pages = 1;
  88         while (USABLE_RX_BD(rxq) < nb_desc)
  89                 rxq->nb_rx_pages <<= 1;
  90
  91         rxq->nb_rx_desc  = TOTAL_RX_BD(rxq);
  92         sc->rx_ring_size = USABLE_RX_BD(rxq);
  93         rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
  94
  95         PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, usable_bd=%lu, "
  96                        "total_bd=%lu, rx_pages=%u, cq_pages=%u",
  97                        queue_idx, nb_desc, (unsigned long)USABLE_RX_BD(rxq),
  98                        (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
  99                        rxq->nb_cq_pages);
 100
 101         /* Allocate RX ring hardware descriptors */
 102         dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd);
 103         dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id);
 104         if (NULL == dma) {
 105                 PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!");
 106                 bnx2x_rx_queue_release(rxq);
 107                 return -ENOMEM;
 108         }
 109         fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->iova;
 110         rxq->rx_ring = (uint64_t*)dma->addr;
 111         memset((void *)rxq->rx_ring, 0, dma_size);
 112
 113         /* Link the RX chain pages. */
 114         for (j = 1; j <= rxq->nb_rx_pages; j++) {
 115                 rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2];
 116                 busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages);
 117                 *rx_bd = busaddr;
 118         }
 119
 120         /* Allocate software ring */
 121         dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry);
 122         rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size,
 123                                           RTE_CACHE_LINE_SIZE,
 124                                           socket_id);
 125         if (NULL == rxq->sw_ring) {
 126                 PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!");
 127                 bnx2x_rx_queue_release(rxq);
 128                 return -ENOMEM;
 129         }
 130
 131         /* Initialize software ring entries */
 132         for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) {
 133                 mbuf = rte_mbuf_raw_alloc(mp);
 134                 if (NULL == mbuf) {
 135                         PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d",
 136                                    (unsigned)rxq->queue_id, idx);
 137                         bnx2x_rx_queue_release(rxq);
 138                         return -ENOMEM;
 139                 }
 140                 rxq->sw_ring[idx] = mbuf;
 141                 rxq->rx_ring[idx] =
 142                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
 143         }
 144         rxq->pkt_first_seg = NULL;
 145         rxq->pkt_last_seg = NULL;
 146         rxq->rx_bd_head = 0;
 147         rxq->rx_bd_tail = rxq->nb_rx_desc;
 148
 149         /* Allocate CQ chain. */
 150         dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages;
 151         dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id);
 152         if (NULL == dma) {
 153                 PMD_RX_LOG(ERR, "RCQ  alloc failed");
 154                 return -ENOMEM;
 155         }
 156         fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->iova;
 157         rxq->cq_ring = (union eth_rx_cqe*)dma->addr;
 158
 159         /* Link the CQ chain pages. */
 160         for (j = 1; j <= rxq->nb_cq_pages; j++) {
 161                 nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe;
 162                 busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages);
 163                 nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
 164                 nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
 165         }
 166         rxq->rx_cq_head = 0;
 167         rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq);
 168
 169         dev->data->rx_queues[queue_idx] = rxq;
 170         if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues;
 171
 172         return 0;
 173 }
 174
 175 static void
 176 bnx2x_tx_queue_release(struct bnx2x_tx_queue *tx_queue)
 177 {
 178         uint16_t i;
 179         struct rte_mbuf **sw_ring;
 180
 181         if (NULL != tx_queue) {
 182
 183                 sw_ring = tx_queue->sw_ring;
 184                 if (NULL != sw_ring) {
 185                         for (i = 0; i < tx_queue->nb_tx_desc; i++) {
 186                                 if (NULL != sw_ring[i])
 187                                         rte_pktmbuf_free(sw_ring[i]);
 188                         }
 189                         rte_free(sw_ring);
 190                 }
 191                 rte_free(tx_queue);
 192         }
 193 }
 194
 195 void
 196 bnx2x_dev_tx_queue_release(void *txq)
 197 {
 198         bnx2x_tx_queue_release(txq);
 199 }
 200
 201 static uint16_t
 202 bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 203 {
 204         struct bnx2x_tx_queue *txq;
 205         struct bnx2x_softc *sc;
 206         struct bnx2x_fastpath *fp;
 207         uint16_t nb_tx_pkts;
 208         uint16_t nb_pkt_sent = 0;
 209         uint32_t ret;
 210
 211         txq = p_txq;
 212         sc = txq->sc;
 213         fp = &sc->fp[txq->queue_id];
 214
 215         if ((unlikely((txq->nb_tx_desc - txq->nb_tx_avail) >
 216                                 txq->tx_free_thresh)))
 217                 bnx2x_txeof(sc, fp);
 218
 219         nb_tx_pkts = RTE_MIN(nb_pkts, txq->nb_tx_avail / BDS_PER_TX_PKT);
 220         if (unlikely(nb_tx_pkts == 0))
 221                 return 0;
 222
 223         while (nb_tx_pkts--) {
 224                 struct rte_mbuf *m = *tx_pkts++;
 225                 assert(m != NULL);
 226                 ret = bnx2x_tx_encap(txq, m);
 227                 fp->tx_db.data.prod += ret;
 228                 nb_pkt_sent++;
 229         }
 230
 231         bnx2x_update_fp_sb_idx(fp);
 232         mb();
 233         DOORBELL(sc, txq->queue_id, fp->tx_db.raw);
 234         mb();
 235
 236         if ((txq->nb_tx_desc - txq->nb_tx_avail) >
 237                                 txq->tx_free_thresh)
 238                 bnx2x_txeof(sc, fp);
 239
 240         return nb_pkt_sent;
 241 }
 242
 243 int
 244 bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
 245                        uint16_t queue_idx,
 246                        uint16_t nb_desc,
 247                        unsigned int socket_id,
 248                        const struct rte_eth_txconf *tx_conf)
 249 {
 250         uint16_t i;
 251         unsigned int tsize;
 252         const struct rte_memzone *tz;
 253         struct bnx2x_tx_queue *txq;
 254         struct eth_tx_next_bd *tx_n_bd;
 255         uint64_t busaddr;
 256         struct bnx2x_softc *sc = dev->data->dev_private;
 257         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
 258
 259         /* First allocate the tx queue data structure */
 260         txq = rte_zmalloc("ethdev TX queue", sizeof(struct bnx2x_tx_queue),
 261                           RTE_CACHE_LINE_SIZE);
 262         if (txq == NULL)
 263                 return -ENOMEM;
 264         txq->sc = sc;
 265
 266         txq->nb_tx_pages = 1;
 267         while (USABLE_TX_BD(txq) < nb_desc)
 268                 txq->nb_tx_pages <<= 1;
 269
 270         txq->nb_tx_desc  = TOTAL_TX_BD(txq);
 271         sc->tx_ring_size = TOTAL_TX_BD(txq);
 272
 273         txq->tx_free_thresh = tx_conf->tx_free_thresh ?
 274                 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH;
 275         txq->tx_free_thresh = min(txq->tx_free_thresh,
 276                                   txq->nb_tx_desc - BDS_PER_TX_PKT);
 277
 278         PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
 279                      "total_bd=%lu, tx_pages=%u",
 280                      queue_idx, nb_desc, txq->tx_free_thresh,
 281                      (unsigned long)USABLE_TX_BD(txq),
 282                      (unsigned long)TOTAL_TX_BD(txq), txq->nb_tx_pages);
 283
 284         /* Allocate TX ring hardware descriptors */
 285         tsize = txq->nb_tx_desc * sizeof(union eth_tx_bd_types);
 286         tz = ring_dma_zone_reserve(dev, "tx_hw_ring", queue_idx, tsize, socket_id);
 287         if (tz == NULL) {
 288                 bnx2x_tx_queue_release(txq);
 289                 return -ENOMEM;
 290         }
 291         fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->iova;
 292         txq->tx_ring = (union eth_tx_bd_types *) tz->addr;
 293         memset(txq->tx_ring, 0, tsize);
 294
 295         /* Allocate software ring */
 296         tsize = txq->nb_tx_desc * sizeof(struct rte_mbuf *);
 297         txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize,
 298                                    RTE_CACHE_LINE_SIZE);
 299         if (txq->sw_ring == NULL) {
 300                 bnx2x_tx_queue_release(txq);
 301                 return -ENOMEM;
 302         }
 303
 304         /* PMD_DRV_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
 305            txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */
 306
 307         /* Link TX pages */
 308         for (i = 1; i <= txq->nb_tx_pages; i++) {
 309                 tx_n_bd = &txq->tx_ring[TOTAL_TX_BD_PER_PAGE * i - 1].next_bd;
 310                 busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages);
 311                 tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
 312                 tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
 313                 /* PMD_DRV_LOG(DEBUG, "link tx page %lu", (TOTAL_TX_BD_PER_PAGE * i - 1)); */
 314         }
 315
 316         txq->queue_id = queue_idx;
 317         txq->port_id = dev->data->port_id;
 318         txq->tx_pkt_tail = 0;
 319         txq->tx_pkt_head = 0;
 320         txq->tx_bd_tail = 0;
 321         txq->tx_bd_head = 0;
 322         txq->nb_tx_avail = txq->nb_tx_desc;
 323         dev->tx_pkt_burst = bnx2x_xmit_pkts;
 324         dev->data->tx_queues[queue_idx] = txq;
 325         if (!sc->tx_queues) sc->tx_queues = dev->data->tx_queues;
 326
 327         return 0;
 328 }
 329
 330 static inline void
 331 bnx2x_upd_rx_prod_fast(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
 332                 uint16_t rx_bd_prod, uint16_t rx_cq_prod)
 333 {
 334         union ustorm_eth_rx_producers rx_prods;
 335
 336         rx_prods.prod.bd_prod  = rx_bd_prod;
 337         rx_prods.prod.cqe_prod = rx_cq_prod;
 338
 339         REG_WR(sc, fp->ustorm_rx_prods_offset, rx_prods.raw_data[0]);
 340 }
 341
 342 static uint16_t
 343 bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 344 {
 345         struct bnx2x_rx_queue *rxq = p_rxq;
 346         struct bnx2x_softc *sc = rxq->sc;
 347         struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id];
 348         uint32_t nb_rx = 0;
 349         uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod;
 350         uint16_t bd_cons, bd_prod;
 351         struct rte_mbuf *new_mb;
 352         uint16_t rx_pref;
 353         struct eth_fast_path_rx_cqe *cqe_fp;
 354         uint16_t len, pad;
 355         struct rte_mbuf *rx_mb = NULL;
 356
 357         hw_cq_cons = le16toh(*fp->rx_cq_cons_sb);
 358         if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) ==
 359                         USABLE_RCQ_ENTRIES_PER_PAGE) {
 360                 ++hw_cq_cons;
 361         }
 362
 363         bd_cons = rxq->rx_bd_head;
 364         bd_prod = rxq->rx_bd_tail;
 365         sw_cq_cons = rxq->rx_cq_head;
 366         sw_cq_prod = rxq->rx_cq_tail;
 367
 368         if (sw_cq_cons == hw_cq_cons)
 369                 return 0;
 370
 371         while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) {
 372
 373                 bd_prod &= MAX_RX_BD(rxq);
 374                 bd_cons &= MAX_RX_BD(rxq);
 375
 376                 cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe;
 377
 378                 if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) {
 379                         PMD_RX_LOG(ERR, "slowpath event during traffic processing");
 380                         break;
 381                 }
 382
 383                 if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) {
 384                         PMD_RX_LOG(ERR, "flags 0x%x rx packet %u",
 385                                         cqe_fp->type_error_flags, sw_cq_cons);
 386                         goto next_rx;
 387                 }
 388
 389                 len = cqe_fp->pkt_len_or_gro_seg_len;
 390                 pad = cqe_fp->placement_offset;
 391
 392                 new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
 393                 if (unlikely(!new_mb)) {
 394                         PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index);
 395                         rte_eth_devices[rxq->port_id].data->
 396                                         rx_mbuf_alloc_failed++;
 397                         goto next_rx;
 398                 }
 399
 400                 rx_mb = rxq->sw_ring[bd_cons];
 401                 rxq->sw_ring[bd_cons] = new_mb;
 402                 rxq->rx_ring[bd_prod] =
 403                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(new_mb));
 404
 405                 rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq);
 406                 rte_prefetch0(rxq->sw_ring[rx_pref]);
 407                 if ((rx_pref & 0x3) == 0) {
 408                         rte_prefetch0(&rxq->rx_ring[rx_pref]);
 409                         rte_prefetch0(&rxq->sw_ring[rx_pref]);
 410                 }
 411
 412                 rx_mb->data_off = pad + RTE_PKTMBUF_HEADROOM;
 413                 rx_mb->nb_segs = 1;
 414                 rx_mb->next = NULL;
 415                 rx_mb->pkt_len = rx_mb->data_len = len;
 416                 rx_mb->port = rxq->port_id;
 417                 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
 418
 419                 /*
 420                  * If we received a packet with a vlan tag,
 421                  * attach that information to the packet.
 422                  */
 423                 if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) {
 424                         rx_mb->vlan_tci = cqe_fp->vlan_tag;
 425                         rx_mb->ol_flags |= PKT_RX_VLAN;
 426                 }
 427
 428                 rx_pkts[nb_rx] = rx_mb;
 429                 nb_rx++;
 430
 431                 /* limit spinning on the queue */
 432                 if (unlikely(nb_rx == sc->rx_budget)) {
 433                         PMD_RX_LOG(ERR, "Limit spinning on the queue");
 434                         break;
 435                 }
 436
 437 next_rx:
 438                 bd_cons    = NEXT_RX_BD(bd_cons);
 439                 bd_prod    = NEXT_RX_BD(bd_prod);
 440                 sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod);
 441                 sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons);
 442         }
 443         rxq->rx_bd_head = bd_cons;
 444         rxq->rx_bd_tail = bd_prod;
 445         rxq->rx_cq_head = sw_cq_cons;
 446         rxq->rx_cq_tail = sw_cq_prod;
 447
 448         bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod);
 449
 450         return nb_rx;
 451 }
 452
 453 int
 454 bnx2x_dev_rx_init(struct rte_eth_dev *dev)
 455 {
 456         dev->rx_pkt_burst = bnx2x_recv_pkts;
 457
 458         return 0;
 459 }
 460
 461 void
 462 bnx2x_dev_clear_queues(struct rte_eth_dev *dev)
 463 {
 464         uint8_t i;
 465
 466         PMD_INIT_FUNC_TRACE();
 467
 468         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 469                 struct bnx2x_tx_queue *txq = dev->data->tx_queues[i];
 470                 if (txq != NULL) {
 471                         bnx2x_tx_queue_release(txq);
 472                         dev->data->tx_queues[i] = NULL;
 473                 }
 474         }
 475
 476         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 477                 struct bnx2x_rx_queue *rxq = dev->data->rx_queues[i];
 478                 if (rxq != NULL) {
 479                         bnx2x_rx_queue_release(rxq);
 480                         dev->data->rx_queues[i] = NULL;
 481                 }
 482         }
 483 }