--- /dev/null
+From 0963caf48db910ca6f93c5cbff8d5d24b2be64dd Mon Sep 17 00:00:00 2001
+From: John Lo <loj@cisco.com>
+Date: Mon, 7 Mar 2016 14:12:29 -0500
+Subject: [PATCH 2/2] Replacement of ENIC PMD receive path to improve
+ performance and code clarifty.
+
+---
+ drivers/net/enic/Makefile | 1 +
+ drivers/net/enic/base/vnic_rq.c | 99 ++---------
+ drivers/net/enic/base/vnic_rq.h | 147 +---------------
+ drivers/net/enic/enic.h | 16 +-
+ drivers/net/enic/enic_ethdev.c | 27 ++-
+ drivers/net/enic/enic_main.c | 321 ++++++++++------------------------
+ drivers/net/enic/enic_res.h | 16 +-
+ drivers/net/enic/enic_rx.c | 378 ++++++++++++++++++++++++++++++++++++++++
+ 8 files changed, 519 insertions(+), 486 deletions(-)
+ create mode 100644 drivers/net/enic/enic_rx.c
+
+diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile
+index f0ee093..f316274 100644
+--- a/drivers/net/enic/Makefile
++++ b/drivers/net/enic/Makefile
+@@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src
+ #
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c
++SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c
+diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c
+index 1441604..cb62c5e 100644
+--- a/drivers/net/enic/base/vnic_rq.c
++++ b/drivers/net/enic/base/vnic_rq.c
+@@ -35,77 +35,21 @@
+ #include "vnic_dev.h"
+ #include "vnic_rq.h"
+
+-static int vnic_rq_alloc_bufs(struct vnic_rq *rq)
+-{
+- struct vnic_rq_buf *buf;
+- unsigned int i, j, count = rq->ring.desc_count;
+- unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count);
+-
+- for (i = 0; i < blks; i++) {
+- rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC);
+- if (!rq->bufs[i])
+- return -ENOMEM;
+- }
+-
+- for (i = 0; i < blks; i++) {
+- buf = rq->bufs[i];
+- for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) {
+- buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j;
+- buf->desc = (u8 *)rq->ring.descs +
+- rq->ring.desc_size * buf->index;
+- if (buf->index + 1 == count) {
+- buf->next = rq->bufs[0];
+- break;
+- } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) {
+- buf->next = rq->bufs[i + 1];
+- } else {
+- buf->next = buf + 1;
+- buf++;
+- }
+- }
+- }
+-
+- rq->to_use = rq->to_clean = rq->bufs[0];
+-
+- return 0;
+-}
+-
+-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count,
+- unsigned int desc_size)
+-{
+- int mem_size = 0;
+-
+- mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size);
+-
+- mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) *
+- VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count);
+-
+- return mem_size;
+-}
+-
+ void vnic_rq_free(struct vnic_rq *rq)
+ {
+ struct vnic_dev *vdev;
+- unsigned int i;
+
+ vdev = rq->vdev;
+
+ vnic_dev_free_desc_ring(vdev, &rq->ring);
+
+- for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) {
+- if (rq->bufs[i]) {
+- kfree(rq->bufs[i]);
+- rq->bufs[i] = NULL;
+- }
+- }
+-
+ rq->ctrl = NULL;
+ }
+
+ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
+ unsigned int desc_count, unsigned int desc_size)
+ {
+- int err;
++ int rc;
+ char res_name[NAME_MAX];
+ static int instance;
+
+@@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
+ vnic_rq_disable(rq);
+
+ snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index);
+- err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size,
++ rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size,
+ rq->socket_id, res_name);
+- if (err)
+- return err;
+-
+- err = vnic_rq_alloc_bufs(rq);
+- if (err) {
+- vnic_rq_free(rq);
+- return err;
+- }
+-
+- return 0;
++ return rc;
+ }
+
+ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
+@@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
+ iowrite32(fetch_index, &rq->ctrl->fetch_index);
+ iowrite32(posted_index, &rq->ctrl->posted_index);
+
+- rq->to_use = rq->to_clean =
+- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)]
+- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)];
+ }
+
+ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
+@@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
+ fetch_index, fetch_index,
+ error_interrupt_enable,
+ error_interrupt_offset);
++ rq->rxst_idx = 0;
++ rq->tot_pkts = 0;
+ }
+
+ void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error)
+@@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq)
+ }
+
+ void vnic_rq_clean(struct vnic_rq *rq,
+- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf))
++ void (*buf_clean)(struct rte_mbuf **buf))
+ {
+- struct vnic_rq_buf *buf;
+- u32 fetch_index;
++ struct rte_mbuf **buf;
++ u32 fetch_index, i;
+ unsigned int count = rq->ring.desc_count;
+
+- buf = rq->to_clean;
+-
+- while (vnic_rq_desc_used(rq) > 0) {
++ buf = &rq->mbuf_ring[0];
+
+- (*buf_clean)(rq, buf);
+-
+- buf = rq->to_clean = buf->next;
+- rq->ring.desc_avail++;
++ for (i = 0; i < count; i++) {
++ (*buf_clean)(buf);
++ buf++;
+ }
++ rq->ring.desc_avail = count - 1;
++ rq->rx_nb_hold = 0;
+
+ /* Use current fetch_index as the ring starting point */
+ fetch_index = ioread32(&rq->ctrl->fetch_index);
+@@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq,
+ /* Hardware surprise removal: reset fetch_index */
+ fetch_index = 0;
+ }
+- rq->to_use = rq->to_clean =
+- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)]
+- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)];
++
+ iowrite32(fetch_index, &rq->ctrl->posted_index);
+
+ vnic_dev_clear_desc_ring(&rq->ring);
+diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h
+index 0f5c3c1..e083ccc 100644
+--- a/drivers/net/enic/base/vnic_rq.h
++++ b/drivers/net/enic/base/vnic_rq.h
+@@ -66,42 +66,22 @@ struct vnic_rq_ctrl {
+ u32 pad10;
+ };
+
+-/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */
+-#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32
+-#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64
+-#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \
+- ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \
+- VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES))
+-#define VNIC_RQ_BUF_BLK_SZ(entries) \
+- (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf))
+-#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \
+- DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries))
+-#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096)
+-
+-struct vnic_rq_buf {
+- struct vnic_rq_buf *next;
+- dma_addr_t dma_addr;
+- void *os_buf;
+- unsigned int os_buf_index;
+- unsigned int len;
+- unsigned int index;
+- void *desc;
+- uint64_t wr_id;
+-};
+-
+ struct vnic_rq {
+ unsigned int index;
++ unsigned int posted_index;
+ struct vnic_dev *vdev;
+- struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */
++ struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */
+ struct vnic_dev_ring ring;
+- struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX];
+- struct vnic_rq_buf *to_use;
+- struct vnic_rq_buf *to_clean;
++ struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */
++ unsigned int mbuf_next_idx; /* next mb to consume */
+ void *os_buf_head;
+ unsigned int pkts_outstanding;
+-
++ uint16_t rx_nb_hold;
++ uint16_t rx_free_thresh;
+ unsigned int socket_id;
+ struct rte_mempool *mp;
++ uint16_t rxst_idx;
++ uint32_t tot_pkts;
+ };
+
+ static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq)
+@@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq)
+ return rq->ring.desc_count - rq->ring.desc_avail - 1;
+ }
+
+-static inline void *vnic_rq_next_desc(struct vnic_rq *rq)
+-{
+- return rq->to_use->desc;
+-}
+-
+-static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq)
+-{
+- return rq->to_use->index;
+-}
+-
+-static inline void vnic_rq_post(struct vnic_rq *rq,
+- void *os_buf, unsigned int os_buf_index,
+- dma_addr_t dma_addr, unsigned int len,
+- uint64_t wrid)
+-{
+- struct vnic_rq_buf *buf = rq->to_use;
+-
+- buf->os_buf = os_buf;
+- buf->os_buf_index = os_buf_index;
+- buf->dma_addr = dma_addr;
+- buf->len = len;
+- buf->wr_id = wrid;
+-
+- buf = buf->next;
+- rq->to_use = buf;
+- rq->ring.desc_avail--;
+-
+- /* Move the posted_index every nth descriptor
+- */
+-
+-#ifndef VNIC_RQ_RETURN_RATE
+-#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */
+-#endif
+-
+- if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) {
+- /* Adding write memory barrier prevents compiler and/or CPU
+- * reordering, thus avoiding descriptor posting before
+- * descriptor is initialized. Otherwise, hardware can read
+- * stale descriptor fields.
+- */
+- wmb();
+- iowrite32(buf->index, &rq->ctrl->posted_index);
+- }
+-}
+-
+-static inline void vnic_rq_post_commit(struct vnic_rq *rq,
+- void *os_buf, unsigned int os_buf_index,
+- dma_addr_t dma_addr, unsigned int len)
+-{
+- struct vnic_rq_buf *buf = rq->to_use;
+-
+- buf->os_buf = os_buf;
+- buf->os_buf_index = os_buf_index;
+- buf->dma_addr = dma_addr;
+- buf->len = len;
+-
+- buf = buf->next;
+- rq->to_use = buf;
+- rq->ring.desc_avail--;
+-
+- /* Move the posted_index every descriptor
+- */
+-
+- /* Adding write memory barrier prevents compiler and/or CPU
+- * reordering, thus avoiding descriptor posting before
+- * descriptor is initialized. Otherwise, hardware can read
+- * stale descriptor fields.
+- */
+- wmb();
+- iowrite32(buf->index, &rq->ctrl->posted_index);
+-}
+
+-static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count)
+-{
+- rq->ring.desc_avail += count;
+-}
+
+ enum desc_return_options {
+ VNIC_RQ_RETURN_DESC,
+ VNIC_RQ_DEFER_RETURN_DESC,
+ };
+
+-static inline int vnic_rq_service(struct vnic_rq *rq,
+- struct cq_desc *cq_desc, u16 completed_index,
+- int desc_return, int (*buf_service)(struct vnic_rq *rq,
+- struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
+- int skipped, void *opaque), void *opaque)
+-{
+- struct vnic_rq_buf *buf;
+- int skipped;
+- int eop = 0;
+-
+- buf = rq->to_clean;
+- while (1) {
+-
+- skipped = (buf->index != completed_index);
+-
+- if ((*buf_service)(rq, cq_desc, buf, skipped, opaque))
+- eop++;
+-
+- if (desc_return == VNIC_RQ_RETURN_DESC)
+- rq->ring.desc_avail++;
+-
+- rq->to_clean = buf->next;
+-
+- if (!skipped)
+- break;
+-
+- buf = rq->to_clean;
+- }
+- return eop;
+-}
+-
+ static inline int vnic_rq_fill(struct vnic_rq *rq,
+ int (*buf_fill)(struct vnic_rq *rq))
+ {
+@@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq);
+ void vnic_rq_enable(struct vnic_rq *rq);
+ int vnic_rq_disable(struct vnic_rq *rq);
+ void vnic_rq_clean(struct vnic_rq *rq,
+- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf));
+-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count,
+- unsigned int desc_size);
+-
++ void (*buf_clean)(struct rte_mbuf **buf));
+ #endif /* _VNIC_RQ_H_ */
+diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
+index 9e78305..8c914f5 100644
+--- a/drivers/net/enic/enic.h
++++ b/drivers/net/enic/enic.h
+@@ -45,6 +45,7 @@
+ #include "vnic_nic.h"
+ #include "vnic_rss.h"
+ #include "enic_res.h"
++#include "cq_enet_desc.h"
+
+ #define DRV_NAME "enic_pmd"
+ #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver"
+@@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev)
+ return (struct enic *)eth_dev->data->dev_private;
+ }
+
++#define RTE_LIBRTE_ENIC_ASSERT_ENABLE
++#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE
++#define ASSERT(x) do { \
++ if (!(x)) \
++ rte_panic("ENIC: x"); \
++} while (0)
++#else
++#define ASSERT(x)
++#endif
++
+ extern void enic_fdir_stats_get(struct enic *enic,
+ struct rte_eth_fdir_stats *stats);
+ extern int enic_fdir_add_fltr(struct enic *enic,
+@@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
+ uint16_t ol_flags, uint16_t vlan_tag);
+
+ extern void enic_post_wq_index(struct vnic_wq *wq);
+-extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts,
+- unsigned int budget, unsigned int *work_done);
+ extern int enic_probe(struct enic *enic);
+ extern int enic_clsf_init(struct enic *enic);
+ extern void enic_clsf_destroy(struct enic *enic);
++uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
++ uint16_t nb_pkts);
++
+ #endif /* _ENIC_H_ */
+diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
+index 2a88043..6f2ada5 100644
+--- a/drivers/net/enic/enic_ethdev.c
++++ b/drivers/net/enic/enic_ethdev.c
+@@ -255,7 +255,7 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
+ uint16_t queue_idx,
+ uint16_t nb_desc,
+ unsigned int socket_id,
+- __rte_unused const struct rte_eth_rxconf *rx_conf,
++ const struct rte_eth_rxconf *rx_conf,
+ struct rte_mempool *mp)
+ {
+ int ret;
+@@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
+ return ret;
+ }
+
++ enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh;
++ dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
++ enic->rq[queue_idx].rx_free_thresh);
++
+ return enicpmd_dev_setup_intr(enic);
+ }
+
+@@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev,
+ DEV_TX_OFFLOAD_IPV4_CKSUM |
+ DEV_TX_OFFLOAD_UDP_CKSUM |
+ DEV_TX_OFFLOAD_TCP_CKSUM;
++ device_info->default_rxconf = (struct rte_eth_rxconf) {
++ .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH
++ };
+ }
+
+ static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
+@@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ return index;
+ }
+
+-static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+- uint16_t nb_pkts)
+-{
+- struct vnic_rq *rq = (struct vnic_rq *)rx_queue;
+- unsigned int work_done;
+-
+- if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done))
+- dev_err(enic, "error in enicpmd poll\n");
+-
+- return work_done;
+-}
+-
+ static const struct eth_dev_ops enicpmd_eth_dev_ops = {
+ .dev_configure = enicpmd_dev_configure,
+ .dev_start = enicpmd_dev_start,
+@@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev)
+ enic->port_id = eth_dev->data->port_id;
+ enic->rte_dev = eth_dev;
+ eth_dev->dev_ops = &enicpmd_eth_dev_ops;
+- eth_dev->rx_pkt_burst = &enicpmd_recv_pkts;
++ eth_dev->rx_pkt_burst = &enic_recv_pkts;
+ eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts;
+
+ pdev = eth_dev->pci_dev;
+@@ -635,8 +630,8 @@ static struct eth_driver rte_enic_pmd = {
+ * Register as the [Poll Mode] Driver of Cisco ENIC device.
+ */
+ static int
+-rte_enic_pmd_init(const char *name __rte_unused,
+- const char *params __rte_unused)
++rte_enic_pmd_init(__rte_unused const char *name,
++ __rte_unused const char *params)
+ {
+ ENICPMD_FUNC_TRACE();
+
+diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
+index 07a9810..d0c9bff 100644
+--- a/drivers/net/enic/enic_main.c
++++ b/drivers/net/enic/enic_main.c
+@@ -60,6 +60,17 @@
+ #include "vnic_nic.h"
+ #include "enic_vnic_wq.h"
+
++static inline struct rte_mbuf *
++rte_rxmbuf_alloc(struct rte_mempool *mp)
++{
++ struct rte_mbuf *m;
++
++ m = __rte_mbuf_raw_alloc(mp);
++ __rte_mbuf_sanity_check_raw(m, 0);
++ return m;
++}
++
++
+ static inline int enic_is_sriov_vf(struct enic *enic)
+ {
+ return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
+@@ -80,16 +91,25 @@ static int is_eth_addr_valid(uint8_t *addr)
+ return !is_mcast_addr(addr) && !is_zero_addr(addr);
+ }
+
+-static inline struct rte_mbuf *
+-enic_rxmbuf_alloc(struct rte_mempool *mp)
++static void
++enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq)
+ {
+- struct rte_mbuf *m;
++ uint16_t i;
+
+- m = __rte_mbuf_raw_alloc(mp);
+- __rte_mbuf_sanity_check_raw(m, 0);
+- return m;
++ if (!rq || !rq->mbuf_ring) {
++ dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
++ return;
++ }
++
++ for (i = 0; i < enic->config.rq_desc_count; i++) {
++ if (rq->mbuf_ring[i]) {
++ rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
++ rq->mbuf_ring[i] = NULL;
++ }
++ }
+ }
+
++
+ void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size)
+ {
+ vnic_set_hdr_split_size(enic->vdev, split_hdr_size);
+@@ -262,13 +282,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
+ }
+
+ static void
+-enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf)
++enic_free_rq_buf(struct rte_mbuf **mbuf)
+ {
+- if (!buf->os_buf)
++ if (*mbuf == NULL)
+ return;
+
+- rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf);
+- buf->os_buf = NULL;
++ rte_pktmbuf_free(*mbuf);
++ mbuf = NULL;
+ }
+
+ void enic_init_vnic_resources(struct enic *enic)
+@@ -314,221 +334,47 @@ void enic_init_vnic_resources(struct enic *enic)
+ }
+
+
+-static int enic_rq_alloc_buf(struct vnic_rq *rq)
++static int
++enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
+ {
+- struct enic *enic = vnic_dev_priv(rq->vdev);
++ struct rte_mbuf *mb;
++ struct rq_enet_desc *rqd = rq->ring.descs;
++ unsigned i;
+ dma_addr_t dma_addr;
+- struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
+- uint8_t type = RQ_ENET_TYPE_ONLY_SOP;
+- u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev);
+- struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp);
+- struct rte_mbuf *hdr_mbuf = NULL;
+-
+- if (!mbuf) {
+- dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n");
+- return -1;
+- }
+-
+- if (unlikely(split_hdr_size)) {
+- if (vnic_rq_desc_avail(rq) < 2) {
+- rte_mempool_put(mbuf->pool, mbuf);
+- return -1;
+- }
+- hdr_mbuf = enic_rxmbuf_alloc(rq->mp);
+- if (!hdr_mbuf) {
+- rte_mempool_put(mbuf->pool, mbuf);
+- dev_err(enic,
+- "hdr_mbuf alloc in enic_rq_alloc_buf failed\n");
+- return -1;
+- }
+-
+- hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+-
+- hdr_mbuf->nb_segs = 2;
+- hdr_mbuf->port = enic->port_id;
+- hdr_mbuf->next = mbuf;
+-
+- dma_addr = (dma_addr_t)
+- (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off);
+-
+- rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size);
+
+- vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr,
+- (unsigned int)split_hdr_size, 0 /*wrid*/);
++ dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index,
++ rq->ring.desc_count);
+
+- desc = vnic_rq_next_desc(rq);
+- type = RQ_ENET_TYPE_NOT_SOP;
+- } else {
+- mbuf->nb_segs = 1;
+- mbuf->port = enic->port_id;
+- }
+-
+- mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+- mbuf->next = NULL;
+-
+- dma_addr = (dma_addr_t)
+- (mbuf->buf_physaddr + mbuf->data_off);
+-
+- rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len);
+-
+- vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr,
+- (unsigned int)mbuf->buf_len, 0 /*wrid*/);
+-
+- return 0;
+-}
+-
+-static int enic_rq_indicate_buf(struct vnic_rq *rq,
+- struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
+- int skipped, void *opaque)
+-{
+- struct enic *enic = vnic_dev_priv(rq->vdev);
+- struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque;
+- struct rte_mbuf *rx_pkt = NULL;
+- struct rte_mbuf *hdr_rx_pkt = NULL;
+-
+- u8 type, color, eop, sop, ingress_port, vlan_stripped;
+- u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
+- u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok;
+- u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc;
+- u8 packet_error;
+- u16 q_number, completed_index, bytes_written, vlan_tci, checksum;
+- u32 rss_hash;
+-
+- cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc,
+- &type, &color, &q_number, &completed_index,
+- &ingress_port, &fcoe, &eop, &sop, &rss_type,
+- &csum_not_calc, &rss_hash, &bytes_written,
+- &packet_error, &vlan_stripped, &vlan_tci, &checksum,
+- &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error,
+- &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp,
+- &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment,
+- &fcs_ok);
+-
+- rx_pkt = (struct rte_mbuf *)buf->os_buf;
+- buf->os_buf = NULL;
+-
+- if (unlikely(packet_error)) {
+- dev_err(enic, "packet error\n");
+- rx_pkt->data_len = 0;
+- return 0;
+- }
+-
+- if (unlikely(skipped)) {
+- rx_pkt->data_len = 0;
+- return 0;
+- }
+-
+- if (likely(!vnic_get_hdr_split_size(enic->vdev))) {
+- /* No header split configured */
+- *rx_pkt_bucket = rx_pkt;
+- rx_pkt->pkt_len = bytes_written;
+-
+- if (ipv4) {
+- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
+- if (!csum_not_calc) {
+- if (unlikely(!ipv4_csum_ok))
+- rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD;
+-
+- if ((tcp || udp) && (!tcp_udp_csum_ok))
+- rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD;
+- }
+- } else if (ipv6)
+- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
+- } else {
+- /* Header split */
+- if (sop && !eop) {
+- /* This piece is header */
+- *rx_pkt_bucket = rx_pkt;
+- rx_pkt->pkt_len = bytes_written;
+- } else {
+- if (sop && eop) {
+- /* The packet is smaller than split_hdr_size */
+- *rx_pkt_bucket = rx_pkt;
+- rx_pkt->pkt_len = bytes_written;
+- if (ipv4) {
+- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
+- if (!csum_not_calc) {
+- if (unlikely(!ipv4_csum_ok))
+- rx_pkt->ol_flags |=
+- PKT_RX_IP_CKSUM_BAD;
+-
+- if ((tcp || udp) &&
+- (!tcp_udp_csum_ok))
+- rx_pkt->ol_flags |=
+- PKT_RX_L4_CKSUM_BAD;
+- }
+- } else if (ipv6)
+- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
+- } else {
+- /* Payload */
+- hdr_rx_pkt = *rx_pkt_bucket;
+- hdr_rx_pkt->pkt_len += bytes_written;
+- if (ipv4) {
+- hdr_rx_pkt->packet_type =
+- RTE_PTYPE_L3_IPV4;
+- if (!csum_not_calc) {
+- if (unlikely(!ipv4_csum_ok))
+- hdr_rx_pkt->ol_flags |=
+- PKT_RX_IP_CKSUM_BAD;
+-
+- if ((tcp || udp) &&
+- (!tcp_udp_csum_ok))
+- hdr_rx_pkt->ol_flags |=
+- PKT_RX_L4_CKSUM_BAD;
+- }
+- } else if (ipv6)
+- hdr_rx_pkt->packet_type =
+- RTE_PTYPE_L3_IPV6;
+- }
++ for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
++ mb = rte_rxmbuf_alloc(rq->mp);
++ if (mb == NULL) {
++ dev_err(enic, "RX mbuf alloc failed queue_id=%u",
++ (unsigned)rq->index);
++ return -ENOMEM;
+ }
+- }
+
+- rx_pkt->data_len = bytes_written;
++ dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off);
+
+- if (rss_hash) {
+- rx_pkt->ol_flags |= PKT_RX_RSS_HASH;
+- rx_pkt->hash.rss = rss_hash;
++ rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP,
++ mb->buf_len);
++ rq->mbuf_ring[i] = mb;
+ }
+
+- if (vlan_tci) {
+- rx_pkt->ol_flags |= PKT_RX_VLAN_PKT;
+- rx_pkt->vlan_tci = vlan_tci;
+- }
++ /* make sure all prior writes are complete before doing the PIO write */
++ rte_rmb();
+
+- return eop;
+-}
++ /* Post all but the last 2 cache lines' worth of descriptors */
++ rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE
++ / sizeof(struct rq_enet_desc));
++ rq->rx_nb_hold = 0;
+
+-static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
+- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque)
+-{
+- struct enic *enic = vnic_dev_priv(vdev);
+-
+- return vnic_rq_service(&enic->rq[q_number], cq_desc,
+- completed_index, VNIC_RQ_RETURN_DESC,
+- enic_rq_indicate_buf, opaque);
+-
+-}
++ dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
++ enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
++ iowrite32(rq->posted_index, &rq->ctrl->posted_index);
++ rte_rmb();
+
+-int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts,
+- unsigned int budget, unsigned int *work_done)
+-{
+- struct enic *enic = vnic_dev_priv(rq->vdev);
+- unsigned int cq = enic_cq_rq(enic, rq->index);
+- int err = 0;
+-
+- *work_done = vnic_cq_service(&enic->cq[cq],
+- budget, enic_rq_service, (void *)rx_pkts);
+-
+- if (*work_done) {
+- vnic_rq_fill(rq, enic_rq_alloc_buf);
++ return 0;
+
+- /* Need at least one buffer on ring to get going */
+- if (vnic_rq_desc_used(rq) == 0) {
+- dev_err(enic, "Unable to alloc receive buffers\n");
+- err = -1;
+- }
+- }
+- return err;
+ }
+
+ static void *
+@@ -576,6 +422,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle,
+ int enic_enable(struct enic *enic)
+ {
+ unsigned int index;
++ int err;
+ struct rte_eth_dev *eth_dev = enic->rte_dev;
+
+ eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
+@@ -586,15 +433,11 @@ int enic_enable(struct enic *enic)
+ dev_warning(enic, "Init of hash table for clsf failed."\
+ "Flow director feature will not work\n");
+
+- /* Fill RQ bufs */
+ for (index = 0; index < enic->rq_count; index++) {
+- vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf);
+-
+- /* Need at least one buffer on ring to get going
+- */
+- if (vnic_rq_desc_used(&enic->rq[index]) == 0) {
+- dev_err(enic, "Unable to alloc receive buffers\n");
+- return -1;
++ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]);
++ if (err) {
++ dev_err(enic, "Failed to alloc RX queue mbufs\n");
++ return err;
+ }
+ }
+
+@@ -636,6 +479,9 @@ void enic_free_rq(void *rxq)
+ struct vnic_rq *rq = (struct vnic_rq *)rxq;
+ struct enic *enic = vnic_dev_priv(rq->vdev);
+
++ enic_rxmbuf_queue_release(enic, rq);
++ rte_free(rq->mbuf_ring);
++ rq->mbuf_ring = NULL;
+ vnic_rq_free(rq);
+ vnic_cq_free(&enic->cq[rq->index]);
+ }
+@@ -664,7 +510,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
+ unsigned int socket_id, struct rte_mempool *mp,
+ uint16_t nb_desc)
+ {
+- int err;
++ int rc;
+ struct vnic_rq *rq = &enic->rq[queue_idx];
+
+ rq->socket_id = socket_id;
+@@ -687,23 +533,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
+ }
+
+ /* Allocate queue resources */
+- err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx,
+- enic->config.rq_desc_count,
+- sizeof(struct rq_enet_desc));
+- if (err) {
++ rc = vnic_rq_alloc(enic->vdev, rq, queue_idx,
++ enic->config.rq_desc_count, sizeof(struct rq_enet_desc));
++ if (rc) {
+ dev_err(enic, "error in allocation of rq\n");
+- return err;
++ goto err_exit;
+ }
+
+- err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
++ rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
+ socket_id, enic->config.rq_desc_count,
+ sizeof(struct cq_enet_rq_desc));
+- if (err) {
+- vnic_rq_free(rq);
++ if (rc) {
+ dev_err(enic, "error in allocation of cq for rq\n");
++ goto err_free_rq_exit;
+ }
+
+- return err;
++ /* Allocate the mbuf ring */
++ rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring",
++ sizeof(struct rte_mbuf *) * enic->config.rq_desc_count,
++ RTE_CACHE_LINE_SIZE, rq->socket_id);
++
++ if (rq->mbuf_ring != NULL)
++ return 0;
++
++ /* cleanup on error */
++ vnic_cq_free(&enic->cq[queue_idx]);
++err_free_rq_exit:
++ vnic_rq_free(rq);
++err_exit:
++ return -ENOMEM;
+ }
+
+ void enic_free_wq(void *txq)
+@@ -790,6 +648,7 @@ int enic_disable(struct enic *enic)
+
+ for (i = 0; i < enic->wq_count; i++)
+ vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
++
+ for (i = 0; i < enic->rq_count; i++)
+ vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
+ for (i = 0; i < enic->cq_count; i++)
+@@ -1074,7 +933,7 @@ int enic_probe(struct enic *enic)
+
+ /* Set ingress vlan rewrite mode before vnic initialization */
+ err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
+- IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN);
++ IG_VLAN_REWRITE_MODE_PASS_THRU);
+ if (err) {
+ dev_err(enic,
+ "Failed to set ingress vlan rewrite mode, aborting.\n");
+diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
+index 49f7e22..33f2e84 100644
+--- a/drivers/net/enic/enic_res.h
++++ b/drivers/net/enic/enic_res.h
+@@ -52,6 +52,7 @@
+ #define ENIC_UNICAST_PERFECT_FILTERS 32
+
+ #define ENIC_NON_TSO_MAX_DESC 16
++#define ENIC_DEFAULT_RX_FREE_THRESH 32
+
+ #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0)
+
+@@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq,
+ WQ_ENET_OFFLOAD_MODE_TSO,
+ eop, 1 /* SOP */, eop, loopback);
+ }
+-static inline void enic_queue_rq_desc(struct vnic_rq *rq,
+- void *os_buf, unsigned int os_buf_index,
+- dma_addr_t dma_addr, unsigned int len)
+-{
+- struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
+- u64 wrid = 0;
+- u8 type = os_buf_index ?
+- RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP;
+-
+- rq_enet_desc_enc(desc,
+- (u64)dma_addr | VNIC_PADDR_TARGET,
+- type, (u16)len);
+-
+- vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid);
+-}
+
+ struct enic;
+
+diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c
+new file mode 100644
+index 0000000..12b1f62
+--- /dev/null
++++ b/drivers/net/enic/enic_rx.c
+@@ -0,0 +1,378 @@
++/*
++ * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved.
++ * Copyright 2007 Nuova Systems, Inc. All rights reserved.
++ *
++ * Copyright (c) 2014, Cisco Systems, Inc.
++ * All rights reserved.
++ *
++ * Redistribution and use in source and binary forms, with or without
++ * modification, are permitted provided that the following conditions
++ * are met:
++ *
++ * 1. Redistributions of source code must retain the above copyright
++ * notice, this list of conditions and the following disclaimer.
++ *
++ * 2. Redistributions in binary form must reproduce the above copyright
++ * notice, this list of conditions and the following disclaimer in
++ * the documentation and/or other materials provided with the
++ * distribution.
++ *
++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
++ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
++ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
++ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
++ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
++ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
++ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
++ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
++ * POSSIBILITY OF SUCH DAMAGE.
++ *
++ */
++
++#include <rte_mbuf.h>
++#include <rte_ethdev.h>
++#include <rte_prefetch.h>
++
++#include "enic_compat.h"
++#include "rq_enet_desc.h"
++#include "enic.h"
++
++#define RTE_PMD_USE_PREFETCH
++
++#ifdef RTE_PMD_USE_PREFETCH
++/*
++ * Prefetch a cache line into all cache levels.
++ */
++#define rte_enic_prefetch(p) rte_prefetch0(p)
++#else
++#define rte_enic_prefetch(p) do {} while (0)
++#endif
++
++#ifdef RTE_PMD_PACKET_PREFETCH
++#define rte_packet_prefetch(p) rte_prefetch1(p)
++#else
++#define rte_packet_prefetch(p) do {} while (0)
++#endif
++
++static inline struct rte_mbuf *
++rte_rxmbuf_alloc(struct rte_mempool *mp)
++{
++ struct rte_mbuf *m;
++
++ m = __rte_mbuf_raw_alloc(mp);
++ __rte_mbuf_sanity_check_raw(m, 0);
++ return m;
++}
++
++static inline uint16_t
++enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
++{
++ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
++}
++
++static inline uint16_t
++enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
++{
++ return(le16_to_cpu(crd->bytes_written_flags) &
++ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_packet_error(uint16_t bwflags)
++{
++ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
++ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_eop(uint16_t ciflags)
++{
++ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
++ == CQ_ENET_RQ_DESC_FLAGS_EOP;
++}
++
++static inline uint8_t
++enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
++{
++ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
++{
++ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
++ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
++{
++ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
++ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
++{
++ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
++ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
++}
++
++static inline uint32_t
++enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
++{
++ return le32_to_cpu(cqrd->rss_hash);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd)
++{
++ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ==
++ CQ_ENET_RQ_DESC_FLAGS_FCS_OK);
++}
++
++static inline uint16_t
++enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
++{
++ return le16_to_cpu(cqrd->vlan);
++}
++
++static inline uint16_t
++enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
++{
++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++ return le16_to_cpu(cqrd->bytes_written_flags) &
++ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
++}
++
++static inline uint64_t
++enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd)
++{
++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++ uint16_t bwflags;
++ uint64_t pkt_err_flags = 0;
++
++ bwflags = enic_cq_rx_desc_bwflags(cqrd);
++
++ /* Check for packet error. Can't be more specific than MAC error */
++ if (enic_cq_rx_desc_packet_error(bwflags)) {
++#ifdef RTE_NEXT_ABI
++ pkt_err_flags |= PKT_RX_MAC_ERR;
++#else
++ pkt_err_flags |= PKT_EXT_RX_PKT_ERROR;
++#endif
++ }
++
++ /* Check for bad FCS. MAC error isn't quite, but no other choice */
++ if (!enic_cq_rx_desc_fcs_ok(cqrd)) {
++#ifdef RTE_NEXT_ABI
++ pkt_err_flags |= PKT_RX_MAC_ERR;
++#else
++ pkt_err_flags |= PKT_EXT_RX_BAD_FCS;
++#endif
++ }
++ return pkt_err_flags;
++}
++
++/*
++ * Lookup table to translate RX CQ flags to mbuf flags.
++ */
++static inline uint32_t
++enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
++{
++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++ uint8_t cqrd_flags = cqrd->flags;
++ static const uint32_t cq_type_table[128] __rte_cache_aligned = {
++ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
++ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
++ | RTE_PTYPE_L4_UDP,
++ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
++ | RTE_PTYPE_L4_TCP,
++ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
++ | RTE_PTYPE_L4_FRAG,
++ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
++ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
++ | RTE_PTYPE_L4_UDP,
++ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
++ | RTE_PTYPE_L4_TCP,
++ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
++ | RTE_PTYPE_L4_FRAG,
++ /* All others reserved */
++ };
++ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
++ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
++ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
++ return cq_type_table[cqrd_flags];
++}
++
++static inline void
++enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
++{
++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++ uint16_t ciflags, bwflags, pkt_flags = 0;
++ ciflags = enic_cq_rx_desc_ciflags(cqrd);
++ bwflags = enic_cq_rx_desc_bwflags(cqrd);
++
++ ASSERT(mbuf->ol_flags == 0);
++
++ /* flags are meaningless if !EOP */
++ if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
++ goto mbuf_flags_done;
++
++ /* VLAN stripping */
++ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
++ pkt_flags |= PKT_RX_VLAN_PKT;
++ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
++ } else {
++ mbuf->vlan_tci = 0;
++ }
++
++ /* RSS flag */
++ if (enic_cq_rx_desc_rss_type(cqrd)) {
++ pkt_flags |= PKT_RX_RSS_HASH;
++ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
++ }
++
++ /* checksum flags */
++ if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
++ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
++ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
++ pkt_flags |= PKT_RX_IP_CKSUM_BAD;
++ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
++ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
++ pkt_flags |= PKT_RX_L4_CKSUM_BAD;
++ }
++ }
++
++ mbuf_flags_done:
++ mbuf->ol_flags = pkt_flags;
++}
++
++static inline uint32_t
++enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
++{
++ uint32_t d = i0 + i1;
++ ASSERT(i0 < n_descriptors);
++ ASSERT(i1 < n_descriptors);
++ d -= (d >= n_descriptors) ? n_descriptors : 0;
++ return d;
++}
++
++
++uint16_t
++enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
++ uint16_t nb_pkts)
++{
++ struct vnic_rq *rq = rx_queue;
++ struct enic *enic = vnic_dev_priv(rq->vdev);
++ unsigned int rx_id;
++ struct rte_mbuf *nmb, *rxmb;
++ uint16_t nb_rx = 0;
++ uint16_t nb_hold;
++ struct vnic_cq *cq;
++ volatile struct cq_desc *cqd_ptr;
++ uint8_t color;
++
++ cq = &enic->cq[enic_cq_rq(enic, rq->index)];
++ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */
++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
++
++ nb_hold = rq->rx_nb_hold; /* mbufs held by software */
++
++ while (nb_rx < nb_pkts) {
++ uint16_t rx_pkt_len;
++ volatile struct rq_enet_desc *rqd_ptr;
++ dma_addr_t dma_addr;
++ struct cq_desc cqd;
++ uint64_t ol_err_flags;
++
++ /* Check for pkts available */
++ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
++ & CQ_DESC_COLOR_MASK;
++ if (color == cq->last_color)
++ break;
++
++ /* Get the cq descriptor and rq pointer */
++ cqd = *cqd_ptr;
++ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
++
++ /* allocate a new mbuf */
++ nmb = rte_rxmbuf_alloc(rq->mp);
++ if (nmb == NULL) {
++ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
++ enic->port_id, (unsigned)rq->index);
++ rte_eth_devices[enic->port_id].
++ data->rx_mbuf_alloc_failed++;
++ break;
++ }
++
++ /* Check for FCS or packet errors */
++ ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd);
++ if (ol_err_flags == 0)
++ rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
++ else
++ rx_pkt_len = 0;
++
++ /* Get the mbuf to return and replace with one just allocated */
++ rxmb = rq->mbuf_ring[rx_id];
++ rq->mbuf_ring[rx_id] = nmb;
++
++ /* Increment cqd, rqd, mbuf_table index */
++ rx_id++;
++ if (unlikely(rx_id == rq->ring.desc_count)) {
++ rx_id = 0;
++ cq->last_color = cq->last_color ? 0 : 1;
++ }
++
++ /* Prefetch next mbuf & desc while processing current one */
++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
++ rte_enic_prefetch(cqd_ptr);
++ rte_enic_prefetch(rq->mbuf_ring[rx_id]);
++ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
++ + rx_id);
++
++ /* Push descriptor for newly allocated mbuf */
++ dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off);
++ rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
++ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len);
++
++ /* Fill in the rest of the mbuf */
++ rxmb->data_off = RTE_PKTMBUF_HEADROOM;
++ rxmb->nb_segs = 1;
++ rxmb->next = NULL;
++ rxmb->pkt_len = rx_pkt_len;
++ rxmb->data_len = rx_pkt_len;
++ rxmb->port = enic->port_id;
++ rxmb->ol_flags = ol_err_flags;
++ if (!ol_err_flags)
++ enic_cq_rx_to_pkt_flags(&cqd, rxmb);
++ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
++
++ /* prefetch mbuf data for caller */
++ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
++ RTE_PKTMBUF_HEADROOM));
++
++ /* store the mbuf address into the next entry of the array */
++ rx_pkts[nb_rx++] = rxmb;
++ }
++
++ nb_hold += nb_rx;
++ cq->to_clean = rx_id;
++
++ if (nb_hold > rq->rx_free_thresh) {
++ rq->posted_index = enic_ring_add(rq->ring.desc_count,
++ rq->posted_index, nb_hold);
++ nb_hold = 0;
++ rte_mb();
++ iowrite32(rq->posted_index, &rq->ctrl->posted_index);
++ }
++
++ rq->rx_nb_hold = nb_hold;
++
++ return nb_rx;
++}
+--
+1.9.1
+
Code Review / vpp.git / commitdiff