rdma: implement multiseg rx without striding rq

[vpp.git] / src / plugins / rdma / rdma.h

diff --git a/src/plugins/rdma/rdma.h b/src/plugins/rdma/rdma.h
index 82f32ec..a72765d 100644 (file)
--- a/src/plugins/rdma/rdma.h
+++ b/src/plugins/rdma/rdma.h
@@ -30,7 +30,8 @@
   _(1, ADMIN_UP, "admin-up") \
   _(2, LINK_UP, "link-up") \
   _(3, PROMISC, "promiscuous") \
-  _(4, MLX5DV, "mlx5dv")
+  _(4, MLX5DV, "mlx5dv") \
+  _(5, STRIDING_RQ, "striding-rq")
 
 enum
 {
@@ -39,10 +40,24 @@ enum
 #undef _
 };
 
+#ifndef MLX5_ETH_L2_INLINE_HEADER_SIZE
+#define MLX5_ETH_L2_INLINE_HEADER_SIZE  18
+#endif
+
 typedef struct
 {
   CLIB_ALIGN_MARK (align0, MLX5_SEND_WQE_BB);
-  struct mlx5_wqe_ctrl_seg ctrl;
+  union
+  {
+    struct mlx5_wqe_ctrl_seg ctrl;
+    struct
+    {
+      u8 opc_mod;
+      u8 wqe_index_hi;
+      u8 wqe_index_lo;
+      u8 opcode;
+    };
+  };
   struct mlx5_wqe_eth_seg eseg;
   struct mlx5_wqe_data_seg dseg;
 } rdma_mlx5_wqe_t;
@@ -67,12 +82,35 @@ typedef struct
   u16 n_mini_cqes_left;
   u16 last_cqe_flags;
   mlx5dv_cqe_t *cqes;
-  mlx5dv_rwq_t *wqes;
+  mlx5dv_wqe_ds_t *wqes;
+    CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
   volatile u32 *wq_db;
   volatile u32 *cq_db;
   u32 cqn;
   u32 wqe_cnt;
   u32 wq_stride;
+  u32 buf_sz;
+  union
+  {
+    struct
+    {
+      u32 striding_wqe_tail;   /* Striding RQ: number of released whole WQE */
+      u8 log_stride_per_wqe;   /* Striding RQ: number of strides in a single WQE */
+    };
+
+    struct
+    {
+      u8 *n_used_per_chain;    /* Legacy RQ: for each buffer chain, how many additional segments are needed */
+
+      u32 *second_bufs;                /* Legacy RQ: ring of second buffers of each chain */
+      u32 incomplete_tail;     /* Legacy RQ: tail index in bufs,
+                                  corresponds to buffer chains with recycled valid head buffer,
+                                  but whose other buffers are not yet recycled (due to pool exhaustion). */
+      u16 n_total_additional_segs;
+      u8 n_ds_per_wqe;         /* Legacy RQ: number of nonnull data segs per WQE */
+    };
+  };
+  u8 log_wqe_sz;               /* log-size of a single WQE (in data segments) */
 } rdma_rxq_t;
 
 typedef struct
@@ -132,7 +170,9 @@ STATIC_ASSERT_OFFSET_OF (rdma_txq_t, cacheline2, 128);
 
 #define RDMA_TXQ_USED_SZ(head, tail)            ((u16)((u16)(tail) - (u16)(head)))
 #define RDMA_TXQ_AVAIL_SZ(txq, head, tail)      ((u16)(RDMA_TXQ_BUF_SZ (txq) - RDMA_TXQ_USED_SZ (head, tail)))
-
+#define RDMA_RXQ_MAX_CHAIN_LOG_SZ 3    /* This should NOT be lower than 3! */
+#define RDMA_RXQ_MAX_CHAIN_SZ (1U << RDMA_RXQ_MAX_CHAIN_LOG_SZ)
+#define RDMA_RXQ_LEGACY_MODE_MAX_CHAIN_SZ 5
 typedef struct
 {
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
@@ -158,10 +198,13 @@ typedef struct
   struct ibv_context *ctx;
   struct ibv_pd *pd;
   struct ibv_mr *mr;
-  struct ibv_qp *rx_qp;
+  struct ibv_qp *rx_qp4;
+  struct ibv_qp *rx_qp6;
   struct ibv_rwq_ind_table *rx_rwq_ind_tbl;
-  struct ibv_flow *flow_ucast;
-  struct ibv_flow *flow_mcast;
+  struct ibv_flow *flow_ucast4;
+  struct ibv_flow *flow_mcast4;
+  struct ibv_flow *flow_ucast6;
+  struct ibv_flow *flow_mcast6;
 
   clib_error_t *error;
 } rdma_device_t;
@@ -175,6 +218,20 @@ typedef struct
     u16x8 cqe_flags8[VLIB_FRAME_SIZE / 8];
     u16x16 cqe_flags16[VLIB_FRAME_SIZE / 16];
   };
+  union
+  {
+    struct
+    {
+      u32 current_segs[VLIB_FRAME_SIZE];
+      u32 to_free_buffers[VLIB_FRAME_SIZE];
+    };                         /* Specific to STRIDING RQ mode */
+    struct
+    {
+      u32 tmp_bi[VLIB_FRAME_SIZE];
+      vlib_buffer_t *tmp_bufs[VLIB_FRAME_SIZE];
+    };                         /* Specific to LEGACY RQ mode */
+  };
+
   vlib_buffer_t buffer_template;
 } rdma_per_thread_data_t;
 
@@ -203,6 +260,9 @@ typedef struct
   u32 txq_size;
   u32 rxq_num;
   rdma_mode_t mode;
+  u8 no_multi_seg;
+  u8 disable_striding_rq;
+  u16 max_pktlen;
 
   /* return */
   int rv;