drivers/net/bnx2x/bnx2x_rxtx.c

   1 /*
   2  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
   3  *
   4  * Copyright (c) 2015 QLogic Corporation.
   5  * All rights reserved.
   6  * www.qlogic.com
   7  *
   8  * See LICENSE.bnx2x_pmd for copyright and licensing details.
   9  */
  10
  11 #include "bnx2x.h"
  12 #include "bnx2x_rxtx.h"
  13
  14 static const struct rte_memzone *
  15 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
  16                       uint16_t queue_id, uint32_t ring_size, int socket_id)
  17 {
  18         char z_name[RTE_MEMZONE_NAMESIZE];
  19         const struct rte_memzone *mz;
  20
  21         snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
  22                         dev->driver->pci_drv.name, ring_name, dev->data->port_id, queue_id);
  23
  24         mz = rte_memzone_lookup(z_name);
  25         if (mz)
  26                 return mz;
  27
  28         return rte_memzone_reserve_aligned(z_name, ring_size, socket_id, 0, BNX2X_PAGE_SIZE);
  29 }
  30
  31 static void
  32 bnx2x_rx_queue_release(struct bnx2x_rx_queue *rx_queue)
  33 {
  34         uint16_t i;
  35         struct rte_mbuf **sw_ring;
  36
  37         if (NULL != rx_queue) {
  38
  39                 sw_ring = rx_queue->sw_ring;
  40                 if (NULL != sw_ring) {
  41                         for (i = 0; i < rx_queue->nb_rx_desc; i++) {
  42                                 if (NULL != sw_ring[i])
  43                                         rte_pktmbuf_free(sw_ring[i]);
  44                         }
  45                         rte_free(sw_ring);
  46                 }
  47                 rte_free(rx_queue);
  48         }
  49 }
  50
  51 void
  52 bnx2x_dev_rx_queue_release(void *rxq)
  53 {
  54         bnx2x_rx_queue_release(rxq);
  55 }
  56
  57 int
  58 bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
  59                        uint16_t queue_idx,
  60                        uint16_t nb_desc,
  61                        unsigned int socket_id,
  62                        const struct rte_eth_rxconf *rx_conf,
  63                        struct rte_mempool *mp)
  64 {
  65         uint16_t j, idx;
  66         const struct rte_memzone *dma;
  67         struct bnx2x_rx_queue *rxq;
  68         uint32_t dma_size;
  69         struct rte_mbuf *mbuf;
  70         struct bnx2x_softc *sc = dev->data->dev_private;
  71         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
  72         struct eth_rx_cqe_next_page *nextpg;
  73         phys_addr_t *rx_bd;
  74         phys_addr_t busaddr;
  75
  76         /* First allocate the rx queue data structure */
  77         rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
  78                                  RTE_CACHE_LINE_SIZE, socket_id);
  79         if (NULL == rxq) {
  80                 PMD_INIT_LOG(ERR, "rte_zmalloc for rxq failed!");
  81                 return -ENOMEM;
  82         }
  83         rxq->sc = sc;
  84         rxq->mb_pool = mp;
  85         rxq->queue_id = queue_idx;
  86         rxq->port_id = dev->data->port_id;
  87         rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.hw_strip_crc) ? 0 : ETHER_CRC_LEN);
  88
  89         rxq->nb_rx_pages = 1;
  90         while (USABLE_RX_BD(rxq) < nb_desc)
  91                 rxq->nb_rx_pages <<= 1;
  92
  93         rxq->nb_rx_desc  = TOTAL_RX_BD(rxq);
  94         sc->rx_ring_size = USABLE_RX_BD(rxq);
  95         rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
  96
  97         rxq->rx_free_thresh = rx_conf->rx_free_thresh ?
  98                 rx_conf->rx_free_thresh : DEFAULT_RX_FREE_THRESH;
  99
 100         PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
 101                        "total_bd=%lu, rx_pages=%u, cq_pages=%u",
 102                        queue_idx, nb_desc, rxq->rx_free_thresh,
 103                        (unsigned long)USABLE_RX_BD(rxq),
 104                        (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
 105                        rxq->nb_cq_pages);
 106
 107         /* Allocate RX ring hardware descriptors */
 108         dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd);
 109         dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id);
 110         if (NULL == dma) {
 111                 PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!");
 112                 bnx2x_rx_queue_release(rxq);
 113                 return -ENOMEM;
 114         }
 115         fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->phys_addr;
 116         rxq->rx_ring = (uint64_t*)dma->addr;
 117         memset((void *)rxq->rx_ring, 0, dma_size);
 118
 119         /* Link the RX chain pages. */
 120         for (j = 1; j <= rxq->nb_rx_pages; j++) {
 121                 rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2];
 122                 busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages);
 123                 *rx_bd = busaddr;
 124         }
 125
 126         /* Allocate software ring */
 127         dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry);
 128         rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size,
 129                                           RTE_CACHE_LINE_SIZE,
 130                                           socket_id);
 131         if (NULL == rxq->sw_ring) {
 132                 PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!");
 133                 bnx2x_rx_queue_release(rxq);
 134                 return -ENOMEM;
 135         }
 136
 137         /* Initialize software ring entries */
 138         rxq->rx_mbuf_alloc = 0;
 139         for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) {
 140                 mbuf = rte_mbuf_raw_alloc(mp);
 141                 if (NULL == mbuf) {
 142                         PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d",
 143                                    (unsigned)rxq->queue_id, idx);
 144                         bnx2x_rx_queue_release(rxq);
 145                         return -ENOMEM;
 146                 }
 147                 rxq->sw_ring[idx] = mbuf;
 148                 rxq->rx_ring[idx] = mbuf->buf_physaddr;
 149                 rxq->rx_mbuf_alloc++;
 150         }
 151         rxq->pkt_first_seg = NULL;
 152         rxq->pkt_last_seg = NULL;
 153         rxq->rx_bd_head = 0;
 154         rxq->rx_bd_tail = rxq->nb_rx_desc;
 155
 156         /* Allocate CQ chain. */
 157         dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages;
 158         dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id);
 159         if (NULL == dma) {
 160                 PMD_RX_LOG(ERR, "RCQ  alloc failed");
 161                 return -ENOMEM;
 162         }
 163         fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->phys_addr;
 164         rxq->cq_ring = (union eth_rx_cqe*)dma->addr;
 165
 166         /* Link the CQ chain pages. */
 167         for (j = 1; j <= rxq->nb_cq_pages; j++) {
 168                 nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe;
 169                 busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages);
 170                 nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
 171                 nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
 172         }
 173         rxq->rx_cq_head = 0;
 174         rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq);
 175
 176         dev->data->rx_queues[queue_idx] = rxq;
 177         if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues;
 178
 179         return 0;
 180 }
 181
 182 static void
 183 bnx2x_tx_queue_release(struct bnx2x_tx_queue *tx_queue)
 184 {
 185         uint16_t i;
 186         struct rte_mbuf **sw_ring;
 187
 188         if (NULL != tx_queue) {
 189
 190                 sw_ring = tx_queue->sw_ring;
 191                 if (NULL != sw_ring) {
 192                         for (i = 0; i < tx_queue->nb_tx_desc; i++) {
 193                                 if (NULL != sw_ring[i])
 194                                         rte_pktmbuf_free(sw_ring[i]);
 195                         }
 196                         rte_free(sw_ring);
 197                 }
 198                 rte_free(tx_queue);
 199         }
 200 }
 201
 202 void
 203 bnx2x_dev_tx_queue_release(void *txq)
 204 {
 205         bnx2x_tx_queue_release(txq);
 206 }
 207
 208 static uint16_t
 209 bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 210 {
 211         struct bnx2x_tx_queue *txq;
 212         struct bnx2x_softc *sc;
 213         struct bnx2x_fastpath *fp;
 214         uint16_t nb_tx_pkts;
 215         uint16_t nb_pkt_sent = 0;
 216         uint32_t ret;
 217
 218         txq = p_txq;
 219         sc = txq->sc;
 220         fp = &sc->fp[txq->queue_id];
 221
 222         if ((unlikely((txq->nb_tx_desc - txq->nb_tx_avail) >
 223                                 txq->tx_free_thresh)))
 224                 bnx2x_txeof(sc, fp);
 225
 226         nb_tx_pkts = RTE_MIN(nb_pkts, txq->nb_tx_avail / BDS_PER_TX_PKT);
 227         if (unlikely(nb_tx_pkts == 0))
 228                 return 0;
 229
 230         while (nb_tx_pkts--) {
 231                 struct rte_mbuf *m = *tx_pkts++;
 232                 assert(m != NULL);
 233                 ret = bnx2x_tx_encap(txq, m);
 234                 fp->tx_db.data.prod += ret;
 235                 nb_pkt_sent++;
 236         }
 237
 238         bnx2x_update_fp_sb_idx(fp);
 239         mb();
 240         DOORBELL(sc, txq->queue_id, fp->tx_db.raw);
 241         mb();
 242
 243         if ((txq->nb_tx_desc - txq->nb_tx_avail) >
 244                                 txq->tx_free_thresh)
 245                 bnx2x_txeof(sc, fp);
 246
 247         return nb_pkt_sent;
 248 }
 249
 250 int
 251 bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
 252                        uint16_t queue_idx,
 253                        uint16_t nb_desc,
 254                        unsigned int socket_id,
 255                        const struct rte_eth_txconf *tx_conf)
 256 {
 257         uint16_t i;
 258         unsigned int tsize;
 259         const struct rte_memzone *tz;
 260         struct bnx2x_tx_queue *txq;
 261         struct eth_tx_next_bd *tx_n_bd;
 262         uint64_t busaddr;
 263         struct bnx2x_softc *sc = dev->data->dev_private;
 264         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
 265
 266         /* First allocate the tx queue data structure */
 267         txq = rte_zmalloc("ethdev TX queue", sizeof(struct bnx2x_tx_queue),
 268                           RTE_CACHE_LINE_SIZE);
 269         if (txq == NULL)
 270                 return -ENOMEM;
 271         txq->sc = sc;
 272
 273         txq->nb_tx_pages = 1;
 274         while (USABLE_TX_BD(txq) < nb_desc)
 275                 txq->nb_tx_pages <<= 1;
 276
 277         txq->nb_tx_desc  = TOTAL_TX_BD(txq);
 278         sc->tx_ring_size = TOTAL_TX_BD(txq);
 279
 280         txq->tx_free_thresh = tx_conf->tx_free_thresh ?
 281                 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH;
 282
 283         PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
 284                      "total_bd=%lu, tx_pages=%u",
 285                      queue_idx, nb_desc, txq->tx_free_thresh,
 286                      (unsigned long)USABLE_TX_BD(txq),
 287                      (unsigned long)TOTAL_TX_BD(txq), txq->nb_tx_pages);
 288
 289         /* Allocate TX ring hardware descriptors */
 290         tsize = txq->nb_tx_desc * sizeof(union eth_tx_bd_types);
 291         tz = ring_dma_zone_reserve(dev, "tx_hw_ring", queue_idx, tsize, socket_id);
 292         if (tz == NULL) {
 293                 bnx2x_tx_queue_release(txq);
 294                 return -ENOMEM;
 295         }
 296         fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->phys_addr;
 297         txq->tx_ring = (union eth_tx_bd_types *) tz->addr;
 298         memset(txq->tx_ring, 0, tsize);
 299
 300         /* Allocate software ring */
 301         tsize = txq->nb_tx_desc * sizeof(struct rte_mbuf *);
 302         txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize,
 303                                    RTE_CACHE_LINE_SIZE);
 304         if (txq->sw_ring == NULL) {
 305                 bnx2x_tx_queue_release(txq);
 306                 return -ENOMEM;
 307         }
 308
 309         /* PMD_DRV_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
 310            txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */
 311
 312         /* Link TX pages */
 313         for (i = 1; i <= txq->nb_tx_pages; i++) {
 314                 tx_n_bd = &txq->tx_ring[TOTAL_TX_BD_PER_PAGE * i - 1].next_bd;
 315                 busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages);
 316                 tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
 317                 tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
 318                 /* PMD_DRV_LOG(DEBUG, "link tx page %lu", (TOTAL_TX_BD_PER_PAGE * i - 1)); */
 319         }
 320
 321         txq->queue_id = queue_idx;
 322         txq->port_id = dev->data->port_id;
 323         txq->tx_pkt_tail = 0;
 324         txq->tx_pkt_head = 0;
 325         txq->tx_bd_tail = 0;
 326         txq->tx_bd_head = 0;
 327         txq->nb_tx_avail = txq->nb_tx_desc;
 328         dev->tx_pkt_burst = bnx2x_xmit_pkts;
 329         dev->data->tx_queues[queue_idx] = txq;
 330         if (!sc->tx_queues) sc->tx_queues = dev->data->tx_queues;
 331
 332         return 0;
 333 }
 334
 335 static inline void
 336 bnx2x_upd_rx_prod_fast(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
 337                 uint16_t rx_bd_prod, uint16_t rx_cq_prod)
 338 {
 339         union ustorm_eth_rx_producers rx_prods;
 340
 341         rx_prods.prod.bd_prod  = rx_bd_prod;
 342         rx_prods.prod.cqe_prod = rx_cq_prod;
 343
 344         REG_WR(sc, fp->ustorm_rx_prods_offset, rx_prods.raw_data[0]);
 345 }
 346
 347 static uint16_t
 348 bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 349 {
 350         struct bnx2x_rx_queue *rxq = p_rxq;
 351         struct bnx2x_softc *sc = rxq->sc;
 352         struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id];
 353         uint32_t nb_rx = 0;
 354         uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod;
 355         uint16_t bd_cons, bd_prod;
 356         struct rte_mbuf *new_mb;
 357         uint16_t rx_pref;
 358         struct eth_fast_path_rx_cqe *cqe_fp;
 359         uint16_t len, pad;
 360         struct rte_mbuf *rx_mb = NULL;
 361
 362         hw_cq_cons = le16toh(*fp->rx_cq_cons_sb);
 363         if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) ==
 364                         USABLE_RCQ_ENTRIES_PER_PAGE) {
 365                 ++hw_cq_cons;
 366         }
 367
 368         bd_cons = rxq->rx_bd_head;
 369         bd_prod = rxq->rx_bd_tail;
 370         sw_cq_cons = rxq->rx_cq_head;
 371         sw_cq_prod = rxq->rx_cq_tail;
 372
 373         if (sw_cq_cons == hw_cq_cons)
 374                 return 0;
 375
 376         while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) {
 377
 378                 bd_prod &= MAX_RX_BD(rxq);
 379                 bd_cons &= MAX_RX_BD(rxq);
 380
 381                 cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe;
 382
 383                 if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) {
 384                         PMD_RX_LOG(ERR, "slowpath event during traffic processing");
 385                         break;
 386                 }
 387
 388                 if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) {
 389                         PMD_RX_LOG(ERR, "flags 0x%x rx packet %u",
 390                                         cqe_fp->type_error_flags, sw_cq_cons);
 391                         goto next_rx;
 392                 }
 393
 394                 len = cqe_fp->pkt_len_or_gro_seg_len;
 395                 pad = cqe_fp->placement_offset;
 396
 397                 new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
 398                 if (unlikely(!new_mb)) {
 399                         PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index);
 400                         rte_eth_devices[rxq->port_id].data->
 401                                         rx_mbuf_alloc_failed++;
 402                         goto next_rx;
 403                 }
 404
 405                 rx_mb = rxq->sw_ring[bd_cons];
 406                 rxq->sw_ring[bd_cons] = new_mb;
 407                 rxq->rx_ring[bd_prod] = new_mb->buf_physaddr;
 408
 409                 rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq);
 410                 rte_prefetch0(rxq->sw_ring[rx_pref]);
 411                 if ((rx_pref & 0x3) == 0) {
 412                         rte_prefetch0(&rxq->rx_ring[rx_pref]);
 413                         rte_prefetch0(&rxq->sw_ring[rx_pref]);
 414                 }
 415
 416                 rx_mb->data_off = pad;
 417                 rx_mb->nb_segs = 1;
 418                 rx_mb->next = NULL;
 419                 rx_mb->pkt_len = rx_mb->data_len = len;
 420                 rx_mb->port = rxq->port_id;
 421                 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
 422
 423                 /*
 424                  * If we received a packet with a vlan tag,
 425                  * attach that information to the packet.
 426                  */
 427                 if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) {
 428                         rx_mb->vlan_tci = cqe_fp->vlan_tag;
 429                         rx_mb->ol_flags |= PKT_RX_VLAN_PKT;
 430                 }
 431
 432                 rx_pkts[nb_rx] = rx_mb;
 433                 nb_rx++;
 434
 435                 /* limit spinning on the queue */
 436                 if (unlikely(nb_rx == sc->rx_budget)) {
 437                         PMD_RX_LOG(ERR, "Limit spinning on the queue");
 438                         break;
 439                 }
 440
 441 next_rx:
 442                 bd_cons    = NEXT_RX_BD(bd_cons);
 443                 bd_prod    = NEXT_RX_BD(bd_prod);
 444                 sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod);
 445                 sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons);
 446         }
 447         rxq->rx_bd_head = bd_cons;
 448         rxq->rx_bd_tail = bd_prod;
 449         rxq->rx_cq_head = sw_cq_cons;
 450         rxq->rx_cq_tail = sw_cq_prod;
 451
 452         bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod);
 453
 454         return nb_rx;
 455 }
 456
 457 int
 458 bnx2x_dev_rx_init(struct rte_eth_dev *dev)
 459 {
 460         dev->rx_pkt_burst = bnx2x_recv_pkts;
 461
 462         return 0;
 463 }
 464
 465 void
 466 bnx2x_dev_clear_queues(struct rte_eth_dev *dev)
 467 {
 468         uint8_t i;
 469
 470         PMD_INIT_FUNC_TRACE();
 471
 472         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 473                 struct bnx2x_tx_queue *txq = dev->data->tx_queues[i];
 474                 if (txq != NULL) {
 475                         bnx2x_tx_queue_release(txq);
 476                         dev->data->tx_queues[i] = NULL;
 477                 }
 478         }
 479
 480         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 481                 struct bnx2x_rx_queue *rxq = dev->data->rx_queues[i];
 482                 if (rxq != NULL) {
 483                         bnx2x_rx_queue_release(rxq);
 484                         dev->data->rx_queues[i] = NULL;
 485                 }
 486         }
 487 }