+ memset (&wqa, 0, sizeof (wqa));
+ wqa.attr_mask = IBV_WQ_ATTR_STATE;
+ wqa.wq_state = IBV_WQS_RDY;
+ if (ibv_modify_wq (rxq->wq, &wqa) != 0)
+ return clib_error_return_unix (0, "Modify WQ (RDY) Failed");
+
+ if (rd->flags & RDMA_DEVICE_F_MLX5DV)
+ {
+ struct mlx5dv_obj obj = { };
+ struct mlx5dv_cq dv_cq;
+ struct mlx5dv_rwq dv_rwq;
+ u64 qw0;
+
+ obj.cq.in = rxq->cq;
+ obj.cq.out = &dv_cq;
+ obj.rwq.in = rxq->wq;
+ obj.rwq.out = &dv_rwq;
+
+ if ((mlx5dv_init_obj (&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_RWQ)))
+ return clib_error_return_unix (0, "mlx5dv: failed to init rx obj");
+
+ if (dv_cq.cqe_size != sizeof (mlx5dv_cqe_t))
+ return clib_error_return_unix (0, "mlx5dv: incompatible rx CQE size");
+
+ rxq->log2_cq_size = max_log2 (dv_cq.cqe_cnt);
+ rxq->cqes = (mlx5dv_cqe_t *) dv_cq.buf;
+ rxq->cq_db = (volatile u32 *) dv_cq.dbrec;
+ rxq->cqn = dv_cq.cqn;
+
+ rxq->wqes = (mlx5dv_rwq_t *) dv_rwq.buf;
+ rxq->wq_db = (volatile u32 *) dv_rwq.dbrec;
+ rxq->wq_stride = dv_rwq.stride;
+ rxq->wqe_cnt = dv_rwq.wqe_cnt;
+
+ qw0 = clib_host_to_net_u32 (vlib_buffer_get_default_data_size (vm));
+ qw0 |= (u64) clib_host_to_net_u32 (rd->lkey) << 32;
+
+ for (int i = 0; i < rxq->size; i++)
+ rxq->wqes[i].dsz_and_lkey = qw0;
+
+ for (int i = 0; i < (1 << rxq->log2_cq_size); i++)
+ rxq->cqes[i].opcode_cqefmt_se_owner = 0xff;
+ }
+
+ return 0;
+}
+
+static clib_error_t *
+rdma_rxq_finalize (vlib_main_t * vm, rdma_device_t * rd)
+{
+ struct ibv_rwq_ind_table_init_attr rwqia;
+ struct ibv_qp_init_attr_ex qpia;
+ struct ibv_wq **ind_tbl;
+ u32 i;
+
+ ASSERT (is_pow2 (vec_len (rd->rxqs))
+ && "rxq number should be a power of 2");
+
+ ind_tbl = vec_new (struct ibv_wq *, vec_len (rd->rxqs));
+ vec_foreach_index (i, rd->rxqs)
+ ind_tbl[i] = vec_elt_at_index (rd->rxqs, i)->wq;
+ memset (&rwqia, 0, sizeof (rwqia));
+ rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl));
+ rwqia.ind_tbl = ind_tbl;
+ if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0)
+ return clib_error_return_unix (0, "RWQ indirection table create failed");
+ vec_free (ind_tbl);
+
+ memset (&qpia, 0, sizeof (qpia));
+ qpia.qp_type = IBV_QPT_RAW_PACKET;
+ qpia.comp_mask =
+ IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_IND_TABLE |
+ IBV_QP_INIT_ATTR_RX_HASH;
+ qpia.pd = rd->pd;
+ qpia.rwq_ind_tbl = rd->rx_rwq_ind_tbl;
+ STATIC_ASSERT_SIZEOF (rdma_rss_hash_key, 40);
+ qpia.rx_hash_conf.rx_hash_key_len = sizeof (rdma_rss_hash_key);
+ qpia.rx_hash_conf.rx_hash_key = rdma_rss_hash_key;
+ qpia.rx_hash_conf.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ;
+
+ qpia.rx_hash_conf.rx_hash_fields_mask =
+ IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4 | IBV_RX_HASH_SRC_PORT_TCP |
+ IBV_RX_HASH_DST_PORT_TCP;
+ if ((rd->rx_qp4 = ibv_create_qp_ex (rd->ctx, &qpia)) == 0)
+ return clib_error_return_unix (0, "IPv4 Queue Pair create failed");
+
+ qpia.rx_hash_conf.rx_hash_fields_mask =
+ IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6 | IBV_RX_HASH_SRC_PORT_TCP |
+ IBV_RX_HASH_DST_PORT_TCP;
+ if ((rd->rx_qp6 = ibv_create_qp_ex (rd->ctx, &qpia)) == 0)
+ return clib_error_return_unix (0, "IPv6 Queue Pair create failed");
+
+ if (rdma_dev_set_ucast (rd))
+ return clib_error_return_unix (0, "Set unicast mode failed");