drivers/net/sfc/sfc_ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2016-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  *
   6  * This software was jointly developed between OKTET Labs (under contract
   7  * for Solarflare) and Solarflare Communications, Inc.
   8  */
   9
  10 /* EF10 native datapath implementation */
  11
  12 #include <stdbool.h>
  13
  14 #include <rte_byteorder.h>
  15 #include <rte_mbuf_ptype.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_io.h>
  18
  19 #include "efx.h"
  20 #include "efx_types.h"
  21 #include "efx_regs.h"
  22 #include "efx_regs_ef10.h"
  23
  24 #include "sfc_tweak.h"
  25 #include "sfc_dp_rx.h"
  26 #include "sfc_kvargs.h"
  27 #include "sfc_ef10.h"
  28
  29 #define SFC_EF10_RX_EV_ENCAP_SUPPORT    1
  30 #include "sfc_ef10_rx_ev.h"
  31
  32 #define sfc_ef10_rx_err(dpq, ...) \
  33         SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__)
  34
  35 /**
  36  * Maximum number of descriptors/buffers in the Rx ring.
  37  * It should guarantee that corresponding event queue never overfill.
  38  * EF10 native datapath uses event queue of the same size as Rx queue.
  39  * Maximum number of events on datapath can be estimated as number of
  40  * Rx queue entries (one event per Rx buffer in the worst case) plus
  41  * Rx error and flush events.
  42  */
  43 #define SFC_EF10_RXQ_LIMIT(_ndesc) \
  44         ((_ndesc) - 1 /* head must not step on tail */ - \
  45          (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \
  46          1 /* Rx error */ - 1 /* flush */)
  47
  48 struct sfc_ef10_rx_sw_desc {
  49         struct rte_mbuf                 *mbuf;
  50 };
  51
  52 struct sfc_ef10_rxq {
  53         /* Used on data path */
  54         unsigned int                    flags;
  55 #define SFC_EF10_RXQ_STARTED            0x1
  56 #define SFC_EF10_RXQ_NOT_RUNNING        0x2
  57 #define SFC_EF10_RXQ_EXCEPTION          0x4
  58 #define SFC_EF10_RXQ_RSS_HASH           0x8
  59         unsigned int                    ptr_mask;
  60         unsigned int                    pending;
  61         unsigned int                    completed;
  62         unsigned int                    evq_read_ptr;
  63         efx_qword_t                     *evq_hw_ring;
  64         struct sfc_ef10_rx_sw_desc      *sw_ring;
  65         uint64_t                        rearm_data;
  66         struct rte_mbuf                 *scatter_pkt;
  67         uint16_t                        prefix_size;
  68
  69         /* Used on refill */
  70         uint16_t                        buf_size;
  71         unsigned int                    added;
  72         unsigned int                    max_fill_level;
  73         unsigned int                    refill_threshold;
  74         struct rte_mempool              *refill_mb_pool;
  75         efx_qword_t                     *rxq_hw_ring;
  76         volatile void                   *doorbell;
  77
  78         /* Datapath receive queue anchor */
  79         struct sfc_dp_rxq               dp;
  80 };
  81
  82 static inline struct sfc_ef10_rxq *
  83 sfc_ef10_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq)
  84 {
  85         return container_of(dp_rxq, struct sfc_ef10_rxq, dp);
  86 }
  87
  88 static void
  89 sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq)
  90 {
  91         const unsigned int ptr_mask = rxq->ptr_mask;
  92         const uint32_t buf_size = rxq->buf_size;
  93         unsigned int free_space;
  94         unsigned int bulks;
  95         void *objs[SFC_RX_REFILL_BULK];
  96         unsigned int added = rxq->added;
  97
  98         RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0);
  99
 100         free_space = rxq->max_fill_level - (added - rxq->completed);
 101
 102         if (free_space < rxq->refill_threshold)
 103                 return;
 104
 105         bulks = free_space / RTE_DIM(objs);
 106         /* refill_threshold guarantees that bulks is positive */
 107         SFC_ASSERT(bulks > 0);
 108
 109         do {
 110                 unsigned int id;
 111                 unsigned int i;
 112
 113                 if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs,
 114                                                   RTE_DIM(objs)) < 0)) {
 115                         struct rte_eth_dev_data *dev_data =
 116                                 rte_eth_devices[rxq->dp.dpq.port_id].data;
 117
 118                         /*
 119                          * It is hardly a safe way to increment counter
 120                          * from different contexts, but all PMDs do it.
 121                          */
 122                         dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs);
 123                         /* Return if we have posted nothing yet */
 124                         if (added == rxq->added)
 125                                 return;
 126                         /* Push posted */
 127                         break;
 128                 }
 129
 130                 for (i = 0, id = added & ptr_mask;
 131                      i < RTE_DIM(objs);
 132                      ++i, ++id) {
 133                         struct rte_mbuf *m = objs[i];
 134                         struct sfc_ef10_rx_sw_desc *rxd;
 135                         rte_iova_t phys_addr;
 136
 137                         MBUF_RAW_ALLOC_CHECK(m);
 138
 139                         SFC_ASSERT((id & ~ptr_mask) == 0);
 140                         rxd = &rxq->sw_ring[id];
 141                         rxd->mbuf = m;
 142
 143                         /*
 144                          * Avoid writing to mbuf. It is cheaper to do it
 145                          * when we receive packet and fill in nearby
 146                          * structure members.
 147                          */
 148
 149                         phys_addr = rte_mbuf_data_iova_default(m);
 150                         EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id],
 151                             ESF_DZ_RX_KER_BYTE_CNT, buf_size,
 152                             ESF_DZ_RX_KER_BUF_ADDR, phys_addr);
 153                 }
 154
 155                 added += RTE_DIM(objs);
 156         } while (--bulks > 0);
 157
 158         SFC_ASSERT(rxq->added != added);
 159         rxq->added = added;
 160         sfc_ef10_rx_qpush(rxq->doorbell, added, ptr_mask);
 161 }
 162
 163 static void
 164 sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id)
 165 {
 166         struct rte_mbuf *next_mbuf;
 167
 168         /* Prefetch next bunch of software descriptors */
 169         if ((next_id % (RTE_CACHE_LINE_SIZE / sizeof(rxq->sw_ring[0]))) == 0)
 170                 rte_prefetch0(&rxq->sw_ring[next_id]);
 171
 172         /*
 173          * It looks strange to prefetch depending on previous prefetch
 174          * data, but measurements show that it is really efficient and
 175          * increases packet rate.
 176          */
 177         next_mbuf = rxq->sw_ring[next_id].mbuf;
 178         if (likely(next_mbuf != NULL)) {
 179                 /* Prefetch the next mbuf structure */
 180                 rte_mbuf_prefetch_part1(next_mbuf);
 181
 182                 /* Prefetch pseudo header of the next packet */
 183                 /* data_off is not filled in yet */
 184                 /* Yes, data could be not ready yet, but we hope */
 185                 rte_prefetch0((uint8_t *)next_mbuf->buf_addr +
 186                               RTE_PKTMBUF_HEADROOM);
 187         }
 188 }
 189
 190 static struct rte_mbuf **
 191 sfc_ef10_rx_pending(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
 192                     uint16_t nb_pkts)
 193 {
 194         uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending - rxq->completed);
 195
 196         SFC_ASSERT(rxq->pending == rxq->completed || rxq->scatter_pkt == NULL);
 197
 198         if (n_rx_pkts != 0) {
 199                 unsigned int completed = rxq->completed;
 200
 201                 rxq->completed = completed + n_rx_pkts;
 202
 203                 do {
 204                         *rx_pkts++ =
 205                                 rxq->sw_ring[completed++ & rxq->ptr_mask].mbuf;
 206                 } while (completed != rxq->completed);
 207         }
 208
 209         return rx_pkts;
 210 }
 211
 212 static uint16_t
 213 sfc_ef10_rx_pseudo_hdr_get_len(const uint8_t *pseudo_hdr)
 214 {
 215         return rte_le_to_cpu_16(*(const uint16_t *)&pseudo_hdr[8]);
 216 }
 217
 218 static uint32_t
 219 sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr)
 220 {
 221         return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr);
 222 }
 223
 224 static struct rte_mbuf **
 225 sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 226                           struct rte_mbuf **rx_pkts,
 227                           struct rte_mbuf ** const rx_pkts_end)
 228 {
 229         const unsigned int ptr_mask = rxq->ptr_mask;
 230         unsigned int pending = rxq->pending;
 231         unsigned int ready;
 232         struct sfc_ef10_rx_sw_desc *rxd;
 233         struct rte_mbuf *m;
 234         struct rte_mbuf *m0;
 235         const uint8_t *pseudo_hdr;
 236         uint16_t seg_len;
 237
 238         ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) &
 239                 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
 240
 241         if (ready == 0) {
 242                 /* Rx abort - it was no enough descriptors for Rx packet */
 243                 rte_pktmbuf_free(rxq->scatter_pkt);
 244                 rxq->scatter_pkt = NULL;
 245                 return rx_pkts;
 246         }
 247
 248         rxq->pending = pending + ready;
 249
 250         if (rx_ev.eq_u64[0] &
 251             rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) |
 252                              (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) {
 253                 SFC_ASSERT(rxq->completed == pending);
 254                 do {
 255                         rxd = &rxq->sw_ring[pending++ & ptr_mask];
 256                         rte_mbuf_raw_free(rxd->mbuf);
 257                 } while (pending != rxq->pending);
 258                 rxq->completed = pending;
 259                 return rx_pkts;
 260         }
 261
 262         /* If scattered packet is in progress */
 263         if (rxq->scatter_pkt != NULL) {
 264                 /* Events for scattered packet frags are not merged */
 265                 SFC_ASSERT(ready == 1);
 266                 SFC_ASSERT(rxq->completed == pending);
 267
 268                 /* There is no pseudo-header in scatter segments. */
 269                 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES);
 270
 271                 rxd = &rxq->sw_ring[pending++ & ptr_mask];
 272                 m = rxd->mbuf;
 273
 274                 MBUF_RAW_ALLOC_CHECK(m);
 275
 276                 m->data_off = RTE_PKTMBUF_HEADROOM;
 277                 rte_pktmbuf_data_len(m) = seg_len;
 278                 rte_pktmbuf_pkt_len(m) = seg_len;
 279
 280                 rxq->scatter_pkt->nb_segs++;
 281                 rte_pktmbuf_pkt_len(rxq->scatter_pkt) += seg_len;
 282                 rte_pktmbuf_lastseg(rxq->scatter_pkt)->next = m;
 283
 284                 if (~rx_ev.eq_u64[0] &
 285                     rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) {
 286                         *rx_pkts++ = rxq->scatter_pkt;
 287                         rxq->scatter_pkt = NULL;
 288                 }
 289                 rxq->completed = pending;
 290                 return rx_pkts;
 291         }
 292
 293         rxd = &rxq->sw_ring[pending++ & ptr_mask];
 294
 295         sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask);
 296
 297         m = rxd->mbuf;
 298
 299         RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data));
 300         m->rearm_data[0] = rxq->rearm_data;
 301
 302         /* Classify packet based on Rx event */
 303         /* Mask RSS hash offload flag if RSS is not enabled */
 304         sfc_ef10_rx_ev_to_offloads(rx_ev, m,
 305                                    (rxq->flags & SFC_EF10_RXQ_RSS_HASH) ?
 306                                    ~0ull : ~PKT_RX_RSS_HASH);
 307
 308         /* data_off already moved past pseudo header */
 309         pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM;
 310
 311         /*
 312          * Always get RSS hash from pseudo header to avoid
 313          * condition/branching. If it is valid or not depends on
 314          * PKT_RX_RSS_HASH in m->ol_flags.
 315          */
 316         m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr);
 317
 318         if (ready == 1)
 319                 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) -
 320                         rxq->prefix_size;
 321         else
 322                 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
 323         SFC_ASSERT(seg_len > 0);
 324         rte_pktmbuf_data_len(m) = seg_len;
 325         rte_pktmbuf_pkt_len(m) = seg_len;
 326
 327         SFC_ASSERT(m->next == NULL);
 328
 329         if (~rx_ev.eq_u64[0] & rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) {
 330                 *rx_pkts++ = m;
 331                 rxq->completed = pending;
 332         } else {
 333                 /* Events with CONT bit are not merged */
 334                 SFC_ASSERT(ready == 1);
 335                 rxq->scatter_pkt = m;
 336                 rxq->completed = pending;
 337                 return rx_pkts;
 338         }
 339
 340         /* Remember mbuf to copy offload flags and packet type from */
 341         m0 = m;
 342         while (pending != rxq->pending) {
 343                 rxd = &rxq->sw_ring[pending++ & ptr_mask];
 344
 345                 sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask);
 346
 347                 m = rxd->mbuf;
 348
 349                 if (rx_pkts != rx_pkts_end) {
 350                         *rx_pkts++ = m;
 351                         rxq->completed = pending;
 352                 }
 353
 354                 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) !=
 355                                  sizeof(rxq->rearm_data));
 356                 m->rearm_data[0] = rxq->rearm_data;
 357
 358                 /* Event-dependent information is the same */
 359                 m->ol_flags = m0->ol_flags;
 360                 m->packet_type = m0->packet_type;
 361
 362                 /* data_off already moved past pseudo header */
 363                 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM;
 364
 365                 /*
 366                  * Always get RSS hash from pseudo header to avoid
 367                  * condition/branching. If it is valid or not depends on
 368                  * PKT_RX_RSS_HASH in m->ol_flags.
 369                  */
 370                 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr);
 371
 372                 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
 373                 SFC_ASSERT(seg_len > 0);
 374                 rte_pktmbuf_data_len(m) = seg_len;
 375                 rte_pktmbuf_pkt_len(m) = seg_len;
 376
 377                 SFC_ASSERT(m->next == NULL);
 378         }
 379
 380         return rx_pkts;
 381 }
 382
 383 static bool
 384 sfc_ef10_rx_get_event(struct sfc_ef10_rxq *rxq, efx_qword_t *rx_ev)
 385 {
 386         *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask];
 387
 388         if (!sfc_ef10_ev_present(*rx_ev))
 389                 return false;
 390
 391         if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) !=
 392                      FSE_AZ_EV_CODE_RX_EV)) {
 393                 /*
 394                  * Do not move read_ptr to keep the event for exception
 395                  * handling by the control path.
 396                  */
 397                 rxq->flags |= SFC_EF10_RXQ_EXCEPTION;
 398                 sfc_ef10_rx_err(&rxq->dp.dpq,
 399                                 "RxQ exception at EvQ read ptr %#x",
 400                                 rxq->evq_read_ptr);
 401                 return false;
 402         }
 403
 404         rxq->evq_read_ptr++;
 405         return true;
 406 }
 407
 408 static uint16_t
 409 sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 410 {
 411         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue);
 412         struct rte_mbuf ** const rx_pkts_end = &rx_pkts[nb_pkts];
 413         unsigned int evq_old_read_ptr;
 414         efx_qword_t rx_ev;
 415
 416         rx_pkts = sfc_ef10_rx_pending(rxq, rx_pkts, nb_pkts);
 417
 418         if (unlikely(rxq->flags &
 419                      (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION)))
 420                 goto done;
 421
 422         evq_old_read_ptr = rxq->evq_read_ptr;
 423         while (rx_pkts != rx_pkts_end && sfc_ef10_rx_get_event(rxq, &rx_ev)) {
 424                 /*
 425                  * DROP_EVENT is an internal to the NIC, software should
 426                  * never see it and, therefore, may ignore it.
 427                  */
 428
 429                 rx_pkts = sfc_ef10_rx_process_event(rxq, rx_ev,
 430                                                     rx_pkts, rx_pkts_end);
 431         }
 432
 433         sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr,
 434                            rxq->evq_read_ptr);
 435
 436         /* It is not a problem if we refill in the case of exception */
 437         sfc_ef10_rx_qrefill(rxq);
 438
 439 done:
 440         return nb_pkts - (rx_pkts_end - rx_pkts);
 441 }
 442
 443 const uint32_t *
 444 sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps)
 445 {
 446         static const uint32_t ef10_native_ptypes[] = {
 447                 RTE_PTYPE_L2_ETHER,
 448                 RTE_PTYPE_L2_ETHER_ARP,
 449                 RTE_PTYPE_L2_ETHER_VLAN,
 450                 RTE_PTYPE_L2_ETHER_QINQ,
 451                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 452                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 453                 RTE_PTYPE_L4_FRAG,
 454                 RTE_PTYPE_L4_TCP,
 455                 RTE_PTYPE_L4_UDP,
 456                 RTE_PTYPE_UNKNOWN
 457         };
 458         static const uint32_t ef10_overlay_ptypes[] = {
 459                 RTE_PTYPE_L2_ETHER,
 460                 RTE_PTYPE_L2_ETHER_ARP,
 461                 RTE_PTYPE_L2_ETHER_VLAN,
 462                 RTE_PTYPE_L2_ETHER_QINQ,
 463                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 464                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 465                 RTE_PTYPE_L4_FRAG,
 466                 RTE_PTYPE_L4_TCP,
 467                 RTE_PTYPE_L4_UDP,
 468                 RTE_PTYPE_TUNNEL_VXLAN,
 469                 RTE_PTYPE_TUNNEL_NVGRE,
 470                 RTE_PTYPE_INNER_L2_ETHER,
 471                 RTE_PTYPE_INNER_L2_ETHER_VLAN,
 472                 RTE_PTYPE_INNER_L2_ETHER_QINQ,
 473                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
 474                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
 475                 RTE_PTYPE_INNER_L4_FRAG,
 476                 RTE_PTYPE_INNER_L4_TCP,
 477                 RTE_PTYPE_INNER_L4_UDP,
 478                 RTE_PTYPE_UNKNOWN
 479         };
 480
 481         /*
 482          * The function returns static set of supported packet types,
 483          * so we can't build it dynamically based on supported tunnel
 484          * encapsulations and should limit to known sets.
 485          */
 486         switch (tunnel_encaps) {
 487         case (1u << EFX_TUNNEL_PROTOCOL_VXLAN |
 488               1u << EFX_TUNNEL_PROTOCOL_GENEVE |
 489               1u << EFX_TUNNEL_PROTOCOL_NVGRE):
 490                 return ef10_overlay_ptypes;
 491         default:
 492                 SFC_GENERIC_LOG(ERR,
 493                         "Unexpected set of supported tunnel encapsulations: %#x",
 494                         tunnel_encaps);
 495                 /* FALLTHROUGH */
 496         case 0:
 497                 return ef10_native_ptypes;
 498         }
 499 }
 500
 501 static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending;
 502 static unsigned int
 503 sfc_ef10_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq)
 504 {
 505         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 506         efx_qword_t rx_ev;
 507         const unsigned int evq_old_read_ptr = rxq->evq_read_ptr;
 508         unsigned int pending = rxq->pending;
 509         unsigned int ready;
 510
 511         if (unlikely(rxq->flags &
 512                      (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION)))
 513                 goto done;
 514
 515         while (sfc_ef10_rx_get_event(rxq, &rx_ev)) {
 516                 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) -
 517                          pending) &
 518                         EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
 519                 pending += ready;
 520         }
 521
 522         /*
 523          * The function does not process events, so return event queue read
 524          * pointer to the original position to allow the events that were
 525          * read to be processed later
 526          */
 527         rxq->evq_read_ptr = evq_old_read_ptr;
 528
 529 done:
 530         return pending - rxq->completed;
 531 }
 532
 533 static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status;
 534 static int
 535 sfc_ef10_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset)
 536 {
 537         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 538         unsigned int npending = sfc_ef10_rx_qdesc_npending(dp_rxq);
 539
 540         if (unlikely(offset > rxq->ptr_mask))
 541                 return -EINVAL;
 542
 543         if (offset < npending)
 544                 return RTE_ETH_RX_DESC_DONE;
 545
 546         if (offset < (rxq->added - rxq->completed))
 547                 return RTE_ETH_RX_DESC_AVAIL;
 548
 549         return RTE_ETH_RX_DESC_UNAVAIL;
 550 }
 551
 552
 553 static sfc_dp_rx_get_dev_info_t sfc_ef10_rx_get_dev_info;
 554 static void
 555 sfc_ef10_rx_get_dev_info(struct rte_eth_dev_info *dev_info)
 556 {
 557         /*
 558          * Number of descriptors just defines maximum number of pushed
 559          * descriptors (fill level).
 560          */
 561         dev_info->rx_desc_lim.nb_min = SFC_RX_REFILL_BULK;
 562         dev_info->rx_desc_lim.nb_align = SFC_RX_REFILL_BULK;
 563 }
 564
 565
 566 static sfc_dp_rx_qsize_up_rings_t sfc_ef10_rx_qsize_up_rings;
 567 static int
 568 sfc_ef10_rx_qsize_up_rings(uint16_t nb_rx_desc,
 569                            __rte_unused struct rte_mempool *mb_pool,
 570                            unsigned int *rxq_entries,
 571                            unsigned int *evq_entries,
 572                            unsigned int *rxq_max_fill_level)
 573 {
 574         /*
 575          * rte_ethdev API guarantees that the number meets min, max and
 576          * alignment requirements.
 577          */
 578         if (nb_rx_desc <= EFX_RXQ_MINNDESCS)
 579                 *rxq_entries = EFX_RXQ_MINNDESCS;
 580         else
 581                 *rxq_entries = rte_align32pow2(nb_rx_desc);
 582
 583         *evq_entries = *rxq_entries;
 584
 585         *rxq_max_fill_level = RTE_MIN(nb_rx_desc,
 586                                       SFC_EF10_RXQ_LIMIT(*evq_entries));
 587         return 0;
 588 }
 589
 590
 591 static uint64_t
 592 sfc_ef10_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size)
 593 {
 594         struct rte_mbuf m;
 595
 596         memset(&m, 0, sizeof(m));
 597
 598         rte_mbuf_refcnt_set(&m, 1);
 599         m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size;
 600         m.nb_segs = 1;
 601         m.port = port_id;
 602
 603         /* rearm_data covers structure members filled in above */
 604         rte_compiler_barrier();
 605         RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t));
 606         return m.rearm_data[0];
 607 }
 608
 609 static sfc_dp_rx_qcreate_t sfc_ef10_rx_qcreate;
 610 static int
 611 sfc_ef10_rx_qcreate(uint16_t port_id, uint16_t queue_id,
 612                     const struct rte_pci_addr *pci_addr, int socket_id,
 613                     const struct sfc_dp_rx_qcreate_info *info,
 614                     struct sfc_dp_rxq **dp_rxqp)
 615 {
 616         struct sfc_ef10_rxq *rxq;
 617         int rc;
 618
 619         rc = EINVAL;
 620         if (info->rxq_entries != info->evq_entries)
 621                 goto fail_rxq_args;
 622
 623         rc = ENOMEM;
 624         rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq),
 625                                  RTE_CACHE_LINE_SIZE, socket_id);
 626         if (rxq == NULL)
 627                 goto fail_rxq_alloc;
 628
 629         sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr);
 630
 631         rc = ENOMEM;
 632         rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring",
 633                                          info->rxq_entries,
 634                                          sizeof(*rxq->sw_ring),
 635                                          RTE_CACHE_LINE_SIZE, socket_id);
 636         if (rxq->sw_ring == NULL)
 637                 goto fail_desc_alloc;
 638
 639         rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING;
 640         if (info->flags & SFC_RXQ_FLAG_RSS_HASH)
 641                 rxq->flags |= SFC_EF10_RXQ_RSS_HASH;
 642         rxq->ptr_mask = info->rxq_entries - 1;
 643         rxq->evq_hw_ring = info->evq_hw_ring;
 644         rxq->max_fill_level = info->max_fill_level;
 645         rxq->refill_threshold = info->refill_threshold;
 646         rxq->rearm_data =
 647                 sfc_ef10_mk_mbuf_rearm_data(port_id, info->prefix_size);
 648         rxq->prefix_size = info->prefix_size;
 649         rxq->buf_size = info->buf_size;
 650         rxq->refill_mb_pool = info->refill_mb_pool;
 651         rxq->rxq_hw_ring = info->rxq_hw_ring;
 652         rxq->doorbell = (volatile uint8_t *)info->mem_bar +
 653                         ER_DZ_RX_DESC_UPD_REG_OFST +
 654                         (info->hw_index << info->vi_window_shift);
 655
 656         *dp_rxqp = &rxq->dp;
 657         return 0;
 658
 659 fail_desc_alloc:
 660         rte_free(rxq);
 661
 662 fail_rxq_alloc:
 663 fail_rxq_args:
 664         return rc;
 665 }
 666
 667 static sfc_dp_rx_qdestroy_t sfc_ef10_rx_qdestroy;
 668 static void
 669 sfc_ef10_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
 670 {
 671         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 672
 673         rte_free(rxq->sw_ring);
 674         rte_free(rxq);
 675 }
 676
 677 static sfc_dp_rx_qstart_t sfc_ef10_rx_qstart;
 678 static int
 679 sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr)
 680 {
 681         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 682
 683         SFC_ASSERT(rxq->completed == 0);
 684         SFC_ASSERT(rxq->pending == 0);
 685         SFC_ASSERT(rxq->added == 0);
 686
 687         sfc_ef10_rx_qrefill(rxq);
 688
 689         rxq->evq_read_ptr = evq_read_ptr;
 690
 691         rxq->flags |= SFC_EF10_RXQ_STARTED;
 692         rxq->flags &= ~(SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION);
 693
 694         return 0;
 695 }
 696
 697 static sfc_dp_rx_qstop_t sfc_ef10_rx_qstop;
 698 static void
 699 sfc_ef10_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr)
 700 {
 701         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 702
 703         rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING;
 704
 705         *evq_read_ptr = rxq->evq_read_ptr;
 706 }
 707
 708 static sfc_dp_rx_qrx_ev_t sfc_ef10_rx_qrx_ev;
 709 static bool
 710 sfc_ef10_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id)
 711 {
 712         __rte_unused struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 713
 714         SFC_ASSERT(rxq->flags & SFC_EF10_RXQ_NOT_RUNNING);
 715
 716         /*
 717          * It is safe to ignore Rx event since we free all mbufs on
 718          * queue purge anyway.
 719          */
 720
 721         return false;
 722 }
 723
 724 static sfc_dp_rx_qpurge_t sfc_ef10_rx_qpurge;
 725 static void
 726 sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 727 {
 728         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 729         unsigned int i;
 730         struct sfc_ef10_rx_sw_desc *rxd;
 731
 732         rte_pktmbuf_free(rxq->scatter_pkt);
 733         rxq->scatter_pkt = NULL;
 734
 735         for (i = rxq->completed; i != rxq->added; ++i) {
 736                 rxd = &rxq->sw_ring[i & rxq->ptr_mask];
 737                 rte_mbuf_raw_free(rxd->mbuf);
 738                 rxd->mbuf = NULL;
 739         }
 740
 741         rxq->completed = rxq->pending = rxq->added = 0;
 742
 743         rxq->flags &= ~SFC_EF10_RXQ_STARTED;
 744 }
 745
 746 struct sfc_dp_rx sfc_ef10_rx = {
 747         .dp = {
 748                 .name           = SFC_KVARG_DATAPATH_EF10,
 749                 .type           = SFC_DP_RX,
 750                 .hw_fw_caps     = SFC_DP_HW_FW_CAP_EF10,
 751         },
 752         .features               = SFC_DP_RX_FEAT_SCATTER |
 753                                   SFC_DP_RX_FEAT_MULTI_PROCESS |
 754                                   SFC_DP_RX_FEAT_TUNNELS |
 755                                   SFC_DP_RX_FEAT_CHECKSUM,
 756         .get_dev_info           = sfc_ef10_rx_get_dev_info,
 757         .qsize_up_rings         = sfc_ef10_rx_qsize_up_rings,
 758         .qcreate                = sfc_ef10_rx_qcreate,
 759         .qdestroy               = sfc_ef10_rx_qdestroy,
 760         .qstart                 = sfc_ef10_rx_qstart,
 761         .qstop                  = sfc_ef10_rx_qstop,
 762         .qrx_ev                 = sfc_ef10_rx_qrx_ev,
 763         .qpurge                 = sfc_ef10_rx_qpurge,
 764         .supported_ptypes_get   = sfc_ef10_supported_ptypes_get,
 765         .qdesc_npending         = sfc_ef10_rx_qdesc_npending,
 766         .qdesc_status           = sfc_ef10_rx_qdesc_status,
 767         .pkt_burst              = sfc_ef10_recv_pkts,
 768 };