#include <infiniband/verbs.h>
#include <vlib/log.h>
+#include <vlib/pci/pci.h>
+#include <vnet/interface.h>
+#include <vnet/ethernet/mac_address.h>
+#include <rdma/rdma_mlx5dv.h>
#define foreach_rdma_device_flags \
- _(0, INITIALIZED, "initialized") \
- _(1, ERROR, "error") \
- _(2, ADMIN_UP, "admin-up") \
- _(3, VA_DMA, "vaddr-dma") \
- _(4, LINK_UP, "link-up") \
- _(5, SHARED_TXQ_LOCK, "shared-txq-lock") \
- _(6, ELOG, "elog") \
+ _(0, ERROR, "error") \
+ _(1, ADMIN_UP, "admin-up") \
+ _(2, LINK_UP, "link-up") \
+ _(3, PROMISC, "promiscuous") \
+ _(4, MLX5DV, "mlx5dv")
enum
{
#undef _
};
+#ifndef MLX5_ETH_L2_INLINE_HEADER_SIZE
+#define MLX5_ETH_L2_INLINE_HEADER_SIZE 18
+#endif
+
+typedef struct
+{
+ CLIB_ALIGN_MARK (align0, MLX5_SEND_WQE_BB);
+ union
+ {
+ struct mlx5_wqe_ctrl_seg ctrl;
+ struct
+ {
+ u8 opc_mod;
+ u8 wqe_index_hi;
+ u8 wqe_index_lo;
+ u8 opcode;
+ };
+ };
+ struct mlx5_wqe_eth_seg eseg;
+ struct mlx5_wqe_data_seg dseg;
+} rdma_mlx5_wqe_t;
+#define RDMA_MLX5_WQE_SZ sizeof(rdma_mlx5_wqe_t)
+#define RDMA_MLX5_WQE_DS (RDMA_MLX5_WQE_SZ/sizeof(struct mlx5_wqe_data_seg))
+STATIC_ASSERT (RDMA_MLX5_WQE_SZ == MLX5_SEND_WQE_BB &&
+ RDMA_MLX5_WQE_SZ % sizeof (struct mlx5_wqe_data_seg) == 0,
+ "bad size");
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- u32 size;
- u32 n_enq;
struct ibv_cq *cq;
- struct ibv_qp *qp;
+ struct ibv_wq *wq;
+ u32 *bufs;
+ u32 size;
+ u32 head;
+ u32 tail;
+ u32 cq_ci;
+ u16 log2_cq_size;
+ u16 n_mini_cqes;
+ u16 n_mini_cqes_left;
+ u16 last_cqe_flags;
+ mlx5dv_cqe_t *cqes;
+ mlx5dv_rwq_t *wqes;
+ volatile u32 *wq_db;
+ volatile u32 *cq_db;
+ u32 cqn;
+ u32 wqe_cnt;
+ u32 wq_stride;
} rdma_rxq_t;
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- u32 size;
- u32 n_enq;
+
+ /* following fields are accessed in datapath */
+ clib_spinlock_t lock;
+
+ union
+ {
+ struct
+ {
+ /* ibverb datapath. Cache of cq, sq below */
+ struct ibv_cq *ibv_cq;
+ struct ibv_qp *ibv_qp;
+ };
+ struct
+ {
+ /* direct verbs datapath */
+ rdma_mlx5_wqe_t *dv_sq_wqes;
+ volatile u32 *dv_sq_dbrec;
+ volatile u64 *dv_sq_db;
+ struct mlx5_cqe64 *dv_cq_cqes;
+ volatile u32 *dv_cq_dbrec;
+ };
+ };
+
+ u32 *bufs; /* vlib_buffer ring buffer */
+ u16 head;
+ u16 tail;
+ u16 dv_cq_idx; /* monotonic CQE index (valid only for direct verbs) */
+ u8 bufs_log2sz; /* log2 vlib_buffer entries */
+ u8 dv_sq_log2sz:4; /* log2 SQ WQE entries (valid only for direct verbs) */
+ u8 dv_cq_log2sz:4; /* log2 CQ CQE entries (valid only for direct verbs) */
+ STRUCT_MARK (cacheline1);
+
+ /* WQE template (valid only for direct verbs) */
+ u8 dv_wqe_tmpl[64];
+
+ /* end of 2nd 64-bytes cacheline (or 1st 128-bytes cacheline) */
+ STRUCT_MARK (cacheline2);
+
+ /* fields below are not accessed in datapath */
struct ibv_cq *cq;
struct ibv_qp *qp;
- clib_spinlock_t lock;
+
} rdma_txq_t;
+STATIC_ASSERT_OFFSET_OF (rdma_txq_t, cacheline1, 64);
+STATIC_ASSERT_OFFSET_OF (rdma_txq_t, cacheline2, 128);
+
+#define RDMA_TXQ_DV_INVALID_ID 0xffffffff
+
+#define RDMA_TXQ_BUF_SZ(txq) (1U << (txq)->bufs_log2sz)
+#define RDMA_TXQ_DV_SQ_SZ(txq) (1U << (txq)->dv_sq_log2sz)
+#define RDMA_TXQ_DV_CQ_SZ(txq) (1U << (txq)->dv_cq_log2sz)
+
+#define RDMA_TXQ_USED_SZ(head, tail) ((u16)((u16)(tail) - (u16)(head)))
+#define RDMA_TXQ_AVAIL_SZ(txq, head, tail) ((u16)(RDMA_TXQ_BUF_SZ (txq) - RDMA_TXQ_USED_SZ (head, tail)))
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+
+ /* following fields are accessed in datapath */
+ rdma_rxq_t *rxqs;
+ rdma_txq_t *txqs;
u32 flags;
u32 per_interface_next_index;
-
- u32 dev_instance;
u32 sw_if_index;
u32 hw_if_index;
+ u32 lkey; /* cache of mr->lkey */
+ u8 pool; /* buffer pool index */
- u32 async_event_clib_file_index;
-
- rdma_rxq_t *rxqs;
- rdma_txq_t *txqs;
-
- u8 hwaddr[6];
- vlib_pci_addr_t pci_addr;
+ /* fields below are not accessed in datapath */
+ vlib_pci_device_info_t *pci;
u8 *name;
+ u8 *linux_ifname;
+ mac_address_t hwaddr;
+ u32 async_event_clib_file_index;
+ u32 dev_instance;
struct ibv_context *ctx;
struct ibv_pd *pd;
struct ibv_mr *mr;
+ struct ibv_qp *rx_qp;
+ struct ibv_rwq_ind_table *rx_rwq_ind_tbl;
struct ibv_flow *flow_ucast;
struct ibv_flow *flow_mcast;
- /* error */
clib_error_t *error;
} rdma_device_t;
typedef struct
{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+ union
+ {
+ u16 cqe_flags[VLIB_FRAME_SIZE];
+ u16x8 cqe_flags8[VLIB_FRAME_SIZE / 8];
+ u16x16 cqe_flags16[VLIB_FRAME_SIZE / 16];
+ };
+ vlib_buffer_t buffer_template;
+} rdma_per_thread_data_t;
+
+typedef struct
+{
+ rdma_per_thread_data_t *per_thread_data;
rdma_device_t *devices;
vlib_log_class_t log_class;
+ u16 msg_id_base;
} rdma_main_t;
extern rdma_main_t rdma_main;
+typedef enum
+{
+ RDMA_MODE_AUTO = 0,
+ RDMA_MODE_IBV,
+ RDMA_MODE_DV,
+} rdma_mode_t;
+
typedef struct
{
u8 *ifname;
u8 *name;
+ u32 rxq_size;
+ u32 txq_size;
+ u32 rxq_num;
+ rdma_mode_t mode;
/* return */
int rv;
extern vlib_node_registration_t rdma_input_node;
extern vnet_device_class_t rdma_device_class;
-/* format.c */
format_function_t format_rdma_device;
format_function_t format_rdma_device_name;
format_function_t format_rdma_input_trace;
+format_function_t format_rdma_rxq;
+unformat_function_t unformat_rdma_create_if_args;
typedef struct
{
u32 next_index;
u32 hw_if_index;
+ u16 cqe_flags;
} rdma_input_trace_t;
-#define foreach_rdma_tx_func_error \
-_(NO_FREE_SLOTS, "no free tx slots")
+#define foreach_rdma_tx_func_error \
+_(SEGMENT_SIZE_EXCEEDED, "segment size exceeded") \
+_(NO_FREE_SLOTS, "no free tx slots") \
+_(SUBMISSION, "tx submission errors") \
+_(COMPLETION, "tx completion errors")
typedef enum
{
RDMA_TX_N_ERROR,
} rdma_tx_func_error_t;
-#endif /* AVF_H */
+#endif /* _RDMA_H_ */
/*
* fd.io coding-style-patch-verification: ON
Code Review / vpp.git / blobdiff