drivers/net/bnxt/bnxt_rxr.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) Broadcom Limited.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Broadcom Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <inttypes.h>
  35 #include <stdbool.h>
  36
  37 #include <rte_byteorder.h>
  38 #include <rte_malloc.h>
  39 #include <rte_memory.h>
  40
  41 #include "bnxt.h"
  42 #include "bnxt_cpr.h"
  43 #include "bnxt_ring.h"
  44 #include "bnxt_rxr.h"
  45 #include "bnxt_rxq.h"
  46 #include "hsi_struct_def_dpdk.h"
  47
  48 /*
  49  * RX Ring handling
  50  */
  51
  52 static inline struct rte_mbuf *__bnxt_alloc_rx_data(struct rte_mempool *mb)
  53 {
  54         struct rte_mbuf *data;
  55
  56         data = rte_mbuf_raw_alloc(mb);
  57
  58         return data;
  59 }
  60
  61 static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq,
  62                                      struct bnxt_rx_ring_info *rxr,
  63                                      uint16_t prod)
  64 {
  65         struct rx_prod_pkt_bd *rxbd = &rxr->rx_desc_ring[prod];
  66         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
  67         struct rte_mbuf *data;
  68
  69         data = __bnxt_alloc_rx_data(rxq->mb_pool);
  70         if (!data)
  71                 return -ENOMEM;
  72
  73         rx_buf->mbuf = data;
  74         data->data_off = RTE_PKTMBUF_HEADROOM;
  75
  76         rxbd->addr = rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(data));
  77
  78         return 0;
  79 }
  80
  81 static void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr, uint16_t cons,
  82                                struct rte_mbuf *mbuf)
  83 {
  84         uint16_t prod = rxr->rx_prod;
  85         struct bnxt_sw_rx_bd *prod_rx_buf;
  86         struct rx_prod_pkt_bd *prod_bd, *cons_bd;
  87
  88         prod_rx_buf = &rxr->rx_buf_ring[prod];
  89
  90         prod_rx_buf->mbuf = mbuf;
  91
  92         prod_bd = &rxr->rx_desc_ring[prod];
  93         cons_bd = &rxr->rx_desc_ring[cons];
  94
  95         prod_bd->addr = cons_bd->addr;
  96 }
  97
  98 static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt,
  99                             struct bnxt_rx_queue *rxq, uint32_t *raw_cons)
 100 {
 101         struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
 102         struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
 103         struct rx_pkt_cmpl *rxcmp;
 104         struct rx_pkt_cmpl_hi *rxcmp1;
 105         uint32_t tmp_raw_cons = *raw_cons;
 106         uint16_t cons, prod, cp_cons =
 107             RING_CMP(cpr->cp_ring_struct, tmp_raw_cons);
 108         struct bnxt_sw_rx_bd *rx_buf;
 109         struct rte_mbuf *mbuf;
 110         int rc = 0;
 111
 112         rxcmp = (struct rx_pkt_cmpl *)
 113             &cpr->cp_desc_ring[cp_cons];
 114
 115         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
 116         cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons);
 117         rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cp_cons];
 118
 119         if (!CMP_VALID(rxcmp1, tmp_raw_cons, cpr->cp_ring_struct))
 120                 return -EBUSY;
 121
 122         prod = rxr->rx_prod;
 123
 124         /* EW - GRO deferred to phase 3 */
 125         cons = rxcmp->opaque;
 126         rx_buf = &rxr->rx_buf_ring[cons];
 127         mbuf = rx_buf->mbuf;
 128         rte_prefetch0(mbuf);
 129
 130         mbuf->data_off = RTE_PKTMBUF_HEADROOM;
 131         mbuf->nb_segs = 1;
 132         mbuf->next = NULL;
 133         mbuf->pkt_len = rxcmp->len;
 134         mbuf->data_len = mbuf->pkt_len;
 135         mbuf->port = rxq->port_id;
 136         mbuf->ol_flags = 0;
 137         if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) {
 138                 mbuf->hash.rss = rxcmp->rss_hash;
 139                 mbuf->ol_flags |= PKT_RX_RSS_HASH;
 140         } else {
 141                 mbuf->hash.fdir.id = rxcmp1->cfa_code;
 142                 mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 143         }
 144         if (rxcmp1->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) {
 145                 mbuf->vlan_tci = rxcmp1->metadata &
 146                         (RX_PKT_CMPL_METADATA_VID_MASK |
 147                         RX_PKT_CMPL_METADATA_DE |
 148                         RX_PKT_CMPL_METADATA_PRI_MASK);
 149                 mbuf->ol_flags |= PKT_RX_VLAN_PKT;
 150         }
 151
 152         rx_buf->mbuf = NULL;
 153
 154         if (likely(RX_CMP_IP_CS_OK(rxcmp1)))
 155                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
 156         else if (likely(RX_CMP_IP_CS_UNKNOWN(rxcmp1)))
 157                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
 158         else
 159                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 160
 161         if (likely(RX_CMP_L4_CS_OK(rxcmp1)))
 162                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 163         else if (likely(RX_CMP_L4_CS_UNKNOWN(rxcmp1)))
 164                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
 165         else
 166                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 167
 168         if (rxcmp1->errors_v2 & RX_CMP_L2_ERRORS) {
 169                 /* Re-install the mbuf back to the rx ring */
 170                 bnxt_reuse_rx_mbuf(rxr, cons, mbuf);
 171
 172                 rc = -EIO;
 173                 goto next_rx;
 174         }
 175         /*
 176          * TODO: Redesign this....
 177          * If the allocation fails, the packet does not get received.
 178          * Simply returning this will result in slowly falling behind
 179          * on the producer ring buffers.
 180          * Instead, "filling up" the producer just before ringing the
 181          * doorbell could be a better solution since it will let the
 182          * producer ring starve until memory is available again pushing
 183          * the drops into hardware and getting them out of the driver
 184          * allowing recovery to a full producer ring.
 185          *
 186          * This could also help with cache usage by preventing per-packet
 187          * calls in favour of a tight loop with the same function being called
 188          * in it.
 189          */
 190         if (bnxt_alloc_rx_data(rxq, rxr, prod)) {
 191                 RTE_LOG(ERR, PMD, "mbuf alloc failed with prod=0x%x\n", prod);
 192                 rc = -ENOMEM;
 193                 goto next_rx;
 194         }
 195
 196         /*
 197          * All MBUFs are allocated with the same size under DPDK,
 198          * no optimization for rx_copy_thresh
 199          */
 200
 201         /* AGG buf operation is deferred */
 202
 203         /* EW - VLAN reception.  Must compare against the ol_flags */
 204
 205         *rx_pkt = mbuf;
 206 next_rx:
 207         rxr->rx_prod = RING_NEXT(rxr->rx_ring_struct, prod);
 208
 209         *raw_cons = tmp_raw_cons;
 210
 211         return rc;
 212 }
 213
 214 uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 215                                uint16_t nb_pkts)
 216 {
 217         struct bnxt_rx_queue *rxq = rx_queue;
 218         struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
 219         struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
 220         uint32_t raw_cons = cpr->cp_raw_cons;
 221         uint32_t cons;
 222         int nb_rx_pkts = 0;
 223         bool rx_event = false;
 224         struct rx_pkt_cmpl *rxcmp;
 225
 226         /* Handle RX burst request */
 227         while (1) {
 228                 int rc;
 229
 230                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
 231                 rte_prefetch0(&cpr->cp_desc_ring[cons]);
 232                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
 233
 234                 if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct))
 235                         break;
 236
 237                 /* TODO: Avoid magic numbers... */
 238                 if ((CMP_TYPE(rxcmp) & 0x30) == 0x10) {
 239                         rc = bnxt_rx_pkt(&rx_pkts[nb_rx_pkts], rxq, &raw_cons);
 240                         if (likely(!rc))
 241                                 nb_rx_pkts++;
 242                         else if (rc == -EBUSY)  /* partial completion */
 243                                 break;
 244                         rx_event = true;
 245                 }
 246                 raw_cons = NEXT_RAW_CMP(raw_cons);
 247                 if (nb_rx_pkts == nb_pkts)
 248                         break;
 249         }
 250         if (raw_cons == cpr->cp_raw_cons) {
 251                 /*
 252                  * For PMD, there is no need to keep on pushing to REARM
 253                  * the doorbell if there are no new completions
 254                  */
 255                 return nb_rx_pkts;
 256         }
 257         cpr->cp_raw_cons = raw_cons;
 258
 259         B_CP_DIS_DB(cpr, cpr->cp_raw_cons);
 260         if (rx_event)
 261                 B_RX_DB(rxr->rx_doorbell, rxr->rx_prod);
 262         return nb_rx_pkts;
 263 }
 264
 265 void bnxt_free_rx_rings(struct bnxt *bp)
 266 {
 267         int i;
 268
 269         for (i = 0; i < (int)bp->rx_nr_rings; i++) {
 270                 struct bnxt_rx_queue *rxq = bp->rx_queues[i];
 271
 272                 if (!rxq)
 273                         continue;
 274
 275                 bnxt_free_ring(rxq->rx_ring->rx_ring_struct);
 276                 rte_free(rxq->rx_ring->rx_ring_struct);
 277                 rte_free(rxq->rx_ring);
 278
 279                 bnxt_free_ring(rxq->cp_ring->cp_ring_struct);
 280                 rte_free(rxq->cp_ring->cp_ring_struct);
 281                 rte_free(rxq->cp_ring);
 282
 283                 rte_free(rxq);
 284                 bp->rx_queues[i] = NULL;
 285         }
 286 }
 287
 288 int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id)
 289 {
 290         struct bnxt *bp = rxq->bp;
 291         struct bnxt_cp_ring_info *cpr;
 292         struct bnxt_rx_ring_info *rxr;
 293         struct bnxt_ring *ring;
 294
 295         rxq->rx_buf_use_size = bp->eth_dev->data->mtu +
 296                                ETHER_HDR_LEN + ETHER_CRC_LEN +
 297                                (2 * VLAN_TAG_SIZE);
 298         rxq->rx_buf_size = rxq->rx_buf_use_size + sizeof(struct rte_mbuf);
 299
 300         rxr = rte_zmalloc_socket("bnxt_rx_ring",
 301                                  sizeof(struct bnxt_rx_ring_info),
 302                                  RTE_CACHE_LINE_SIZE, socket_id);
 303         if (rxr == NULL)
 304                 return -ENOMEM;
 305         rxq->rx_ring = rxr;
 306
 307         ring = rte_zmalloc_socket("bnxt_rx_ring_struct",
 308                                    sizeof(struct bnxt_ring),
 309                                    RTE_CACHE_LINE_SIZE, socket_id);
 310         if (ring == NULL)
 311                 return -ENOMEM;
 312         rxr->rx_ring_struct = ring;
 313         ring->ring_size = rte_align32pow2(rxq->nb_rx_desc);
 314         ring->ring_mask = ring->ring_size - 1;
 315         ring->bd = (void *)rxr->rx_desc_ring;
 316         ring->bd_dma = rxr->rx_desc_mapping;
 317         ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd);
 318         ring->vmem = (void **)&rxr->rx_buf_ring;
 319
 320         cpr = rte_zmalloc_socket("bnxt_rx_ring",
 321                                  sizeof(struct bnxt_cp_ring_info),
 322                                  RTE_CACHE_LINE_SIZE, socket_id);
 323         if (cpr == NULL)
 324                 return -ENOMEM;
 325         rxq->cp_ring = cpr;
 326
 327         ring = rte_zmalloc_socket("bnxt_rx_ring_struct",
 328                                    sizeof(struct bnxt_ring),
 329                                    RTE_CACHE_LINE_SIZE, socket_id);
 330         if (ring == NULL)
 331                 return -ENOMEM;
 332         cpr->cp_ring_struct = ring;
 333         ring->ring_size = rxr->rx_ring_struct->ring_size * 2;
 334         ring->ring_mask = ring->ring_size - 1;
 335         ring->bd = (void *)cpr->cp_desc_ring;
 336         ring->bd_dma = cpr->cp_desc_mapping;
 337         ring->vmem_size = 0;
 338         ring->vmem = NULL;
 339
 340         return 0;
 341 }
 342
 343 static void bnxt_init_rxbds(struct bnxt_ring *ring, uint32_t type,
 344                             uint16_t len)
 345 {
 346         uint32_t j;
 347         struct rx_prod_pkt_bd *rx_bd_ring = (struct rx_prod_pkt_bd *)ring->bd;
 348
 349         if (!rx_bd_ring)
 350                 return;
 351         for (j = 0; j < ring->ring_size; j++) {
 352                 rx_bd_ring[j].flags_type = type;
 353                 rx_bd_ring[j].len = len;
 354                 rx_bd_ring[j].opaque = j;
 355         }
 356 }
 357
 358 int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq)
 359 {
 360         struct bnxt_rx_ring_info *rxr;
 361         struct bnxt_ring *ring;
 362         uint32_t prod, type;
 363         unsigned int i;
 364
 365         type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT | RX_PROD_PKT_BD_FLAGS_EOP_PAD;
 366
 367         rxr = rxq->rx_ring;
 368         ring = rxr->rx_ring_struct;
 369         bnxt_init_rxbds(ring, type, rxq->rx_buf_use_size);
 370
 371         prod = rxr->rx_prod;
 372         for (i = 0; i < ring->ring_size; i++) {
 373                 if (bnxt_alloc_rx_data(rxq, rxr, prod) != 0) {
 374                         RTE_LOG(WARNING, PMD,
 375                                 "init'ed rx ring %d with %d/%d mbufs only\n",
 376                                 rxq->queue_id, i, ring->ring_size);
 377                         break;
 378                 }
 379                 rxr->rx_prod = prod;
 380                 prod = RING_NEXT(rxr->rx_ring_struct, prod);
 381         }
 382
 383         return 0;
 384 }