New upstream version 18.11.2

[deb_dpdk.git] / drivers / net / mlx5 / mlx5_rxtx_vec_neon.h

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
index 71a5eaf..38e915c 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
@@ -104,11 +104,11 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
                sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
        unsigned int n;
        volatile struct mlx5_wqe *wqe = NULL;
+       bool metadata_ol =
+               txq->offloads & DEV_TX_OFFLOAD_MATCH_METADATA ? true : false;
 
        assert(elts_n > pkts_n);
        mlx5_tx_complete(txq);
-       /* A CQE slot must always be available. */
-       assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
        if (unlikely(!pkts_n))
                return 0;
        for (n = 0; n < pkts_n; ++n) {
@@ -129,6 +129,9 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
                uint8x16_t *t_wqe;
                uint8_t *dseg;
                uint8x16_t ctrl;
+               rte_be32_t metadata =
+                       metadata_ol && (buf->ol_flags & PKT_TX_METADATA) ?
+                       buf->tx_metadata : 0;
 
                assert(segs_n);
                max_elts = elts_n - (elts_head - txq->elts_tail);
@@ -166,9 +169,10 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
                ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m);
                vst1q_u8((void *)t_wqe, ctrl);
                /* Fill ESEG in the header. */
-               vst1q_u16((void *)(t_wqe + 1),
-                         ((uint16x8_t) { 0, 0, cs_flags, rte_cpu_to_be_16(len),
-                                         0, 0, 0, 0 }));
+               vst1q_u32((void *)(t_wqe + 1),
+                         ((uint32x4_t){ 0,
+                                        cs_flags << 16 | rte_cpu_to_be_16(len),
+                                        metadata, 0 }));
                txq->wqe_ci = wqe_ci;
        }
        if (!n)
@@ -176,12 +180,11 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
        txq->elts_comp += (uint16_t)(elts_head - txq->elts_head);
        txq->elts_head = elts_head;
        if (txq->elts_comp >= MLX5_TX_COMP_THRESH) {
+               /* A CQE slot must always be available. */
+               assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
                wqe->ctrl[2] = rte_cpu_to_be_32(8);
                wqe->ctrl[3] = txq->elts_head;
                txq->elts_comp = 0;
-#ifndef NDEBUG
-               ++txq->cq_pi;
-#endif
        }
 #ifdef MLX5_PMD_SOFT_COUNTERS
        txq->stats.opackets += n;
@@ -204,13 +207,15 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
  *   Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST).
  * @param cs_flags
  *   Checksum offload flags to be written in the descriptor.
+ * @param metadata
+ *   Metadata value to be written in the descriptor.
  *
  * @return
  *   Number of packets successfully transmitted (<= pkts_n).
  */
 static inline uint16_t
 txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
-           uint8_t cs_flags)
+           uint8_t cs_flags, rte_be32_t metadata)
 {
        struct rte_mbuf **elts;
        uint16_t elts_head = txq->elts_head;
@@ -245,8 +250,6 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
        assert(elts_n > pkts_n);
        mlx5_tx_complete(txq);
        max_elts = (elts_n - (elts_head - txq->elts_tail));
-       /* A CQE slot must always be available. */
-       assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
        max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
        pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
        if (unlikely(!pkts_n))
@@ -282,11 +285,10 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
        if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) {
                txq->elts_comp += pkts_n;
        } else {
+               /* A CQE slot must always be available. */
+               assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
                /* Request a completion. */
                txq->elts_comp = 0;
-#ifndef NDEBUG
-               ++txq->cq_pi;
-#endif
                comp_req = 8;
        }
        /* Fill CTRL in the header. */
@@ -299,11 +301,8 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
        ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m);
        vst1q_u8((void *)t_wqe, ctrl);
        /* Fill ESEG in the header. */
-       vst1q_u8((void *)(t_wqe + 1),
-                ((uint8x16_t) { 0, 0, 0, 0,
-                                cs_flags, 0, 0, 0,
-                                0, 0, 0, 0,
-                                0, 0, 0, 0 }));
+       vst1q_u32((void *)(t_wqe + 1),
+                ((uint32x4_t) { 0, cs_flags, metadata, 0 }));
 #ifdef MLX5_PMD_SOFT_COUNTERS
        txq->stats.opackets += pkts_n;
 #endif
@@ -739,7 +738,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
         *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
         */
        repl_n = q_n - (rxq->rq_ci - rxq->rq_pi);
-       if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH)
+       if (repl_n >= rxq->rq_repl_thresh)
                mlx5_rx_replenish_bulk_mbuf(rxq, repl_n);
        /* See if there're unreturned mbufs from compressed CQE. */
        rcvd_pkt = rxq->cq_ci - rxq->rq_pi;