New upstream version 18.11-rc4

[deb_dpdk.git] / drivers / net / mlx5 / mlx5_rxtx.h

diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 48ed2b2..f47d327 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -97,10 +97,11 @@ struct mlx5_rxq_data {
        volatile uint32_t *rq_db;
        volatile uint32_t *cq_db;
        uint16_t port_id;
-       uint16_t rq_ci;
+       uint32_t rq_ci;
        uint16_t consumed_strd; /* Number of consumed strides in WQE. */
-       uint16_t rq_pi;
-       uint16_t cq_ci;
+       uint32_t rq_pi;
+       uint32_t cq_ci;
+       uint16_t rq_repl_thresh; /* Threshold for buffer replenishment. */
        struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
        uint16_t mprq_max_memcpy_len; /* Maximum size of packet to memcpy. */
        volatile void *wqes;
@@ -345,6 +346,7 @@ uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts,
                          uint16_t pkts_n);
 int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset);
 int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
+uint32_t mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id);
 
 /* Vectorized version of mlx5_rxtx.c */
 int mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev);
@@ -362,7 +364,9 @@ uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
 
 void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
 uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
-uint32_t mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr);
+uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb);
+uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
+                              struct rte_mempool *mp);
 
 /**
  * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
@@ -376,17 +380,16 @@ uint32_t mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr);
  *   Address of the lock to use for that UAR access.
  */
 static __rte_always_inline void
-__mlx5_uar_write64_relaxed(uint64_t val, volatile void *addr,
+__mlx5_uar_write64_relaxed(uint64_t val, void *addr,
                           rte_spinlock_t *lock __rte_unused)
 {
 #ifdef RTE_ARCH_64
-       rte_write64_relaxed(val, addr);
+       *(uint64_t *)addr = val;
 #else /* !RTE_ARCH_64 */
        rte_spinlock_lock(lock);
-       rte_write32_relaxed(val, addr);
+       *(uint32_t *)addr = val;
        rte_io_wmb();
-       rte_write32_relaxed(val >> 32,
-                           (volatile void *)((volatile char *)addr + 4));
+       *((uint32_t *)addr + 1) = val >> 32;
        rte_spinlock_unlock(lock);
 #endif
 }
@@ -404,7 +407,7 @@ __mlx5_uar_write64_relaxed(uint64_t val, volatile void *addr,
  *   Address of the lock to use for that UAR access.
  */
 static __rte_always_inline void
-__mlx5_uar_write64(uint64_t val, volatile void *addr, rte_spinlock_t *lock)
+__mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
 {
        rte_io_wmb();
        __mlx5_uar_write64_relaxed(val, addr, lock);
@@ -606,6 +609,24 @@ mlx5_tx_complete(struct mlx5_txq_data *txq)
        *txq->cq_db = rte_cpu_to_be_32(cq_ci);
 }
 
+/**
+ * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the
+ * cloned mbuf is allocated is returned instead.
+ *
+ * @param buf
+ *   Pointer to mbuf.
+ *
+ * @return
+ *   Memory pool where data is located for given mbuf.
+ */
+static inline struct rte_mempool *
+mlx5_mb2mp(struct rte_mbuf *buf)
+{
+       if (unlikely(RTE_MBUF_INDIRECT(buf)))
+               return rte_mbuf_from_indirect(buf)->pool;
+       return buf->pool;
+}
+
 /**
  * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
  * as mempool is pre-configured and static.
@@ -647,9 +668,10 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
  *   Searched LKey on success, UINT32_MAX on no match.
  */
 static __rte_always_inline uint32_t
-mlx5_tx_addr2mr(struct mlx5_txq_data *txq, uintptr_t addr)
+mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
 {
        struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+       uintptr_t addr = (uintptr_t)mb->buf_addr;
        uint32_t lkey;
 
        /* Check generation bit to see if there's any change on existing MRs. */
@@ -660,12 +682,10 @@ mlx5_tx_addr2mr(struct mlx5_txq_data *txq, uintptr_t addr)
                                    MLX5_MR_CACHE_N, addr);
        if (likely(lkey != UINT32_MAX))
                return lkey;
-       /* Take slower bottom-half (binary search) on miss. */
-       return mlx5_tx_addr2mr_bh(txq, addr);
+       /* Take slower bottom-half on miss. */
+       return mlx5_tx_mb2mr_bh(txq, mb);
 }
 
-#define mlx5_tx_mb2mr(rxq, mb) mlx5_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
-
 /**
  * Ring TX queue doorbell and flush the update if requested.
  *