X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;ds=sidebyside;f=src%2Fplugins%2Frdma%2Fdevice.c;h=c218bf93c7f50c68e466c9d16f15a0455c0bef3e;hb=3a533cd561eb75deab489d5ab39a7ec4e1bf7952;hp=62dff21b0609f48c4836ca1ae40c8de2814d70d0;hpb=0dcafcc505dccb9243c90c57a20295d4f0afce29;p=vpp.git diff --git a/src/plugins/rdma/device.c b/src/plugins/rdma/device.c index 62dff21b060..c218bf93c7f 100644 --- a/src/plugins/rdma/device.c +++ b/src/plugins/rdma/device.c @@ -26,6 +26,7 @@ #include #include #include +#include #include @@ -45,11 +46,14 @@ static u8 rdma_rss_hash_key[] = { rdma_main_t rdma_main; -#define rdma_log__(lvl, dev, f, ...) \ - do { \ - vlib_log((lvl), rdma_main.log_class, "%s: " f, \ - &(dev)->name, ##__VA_ARGS__); \ - } while (0) +/* (dev) is of type (rdma_device_t *) */ +#define rdma_log__(lvl, dev, f, ...) \ + do \ + { \ + vlib_log ((lvl), rdma_main.log_class, "%s: " f, (dev)->name, \ + ##__VA_ARGS__); \ + } \ + while (0) #define rdma_log(lvl, dev, f, ...) \ rdma_log__((lvl), (dev), "%s (%d): " f, strerror(errno), errno, ##__VA_ARGS__) @@ -57,7 +61,7 @@ rdma_main_t rdma_main; static struct ibv_flow * rdma_rxq_init_flow (const rdma_device_t * rd, struct ibv_qp *qp, const mac_address_t * mac, const mac_address_t * mask, - u32 flags) + u16 ether_type, u32 flags) { struct ibv_flow *flow; struct raw_eth_flow_attr @@ -76,6 +80,12 @@ rdma_rxq_init_flow (const rdma_device_t * rd, struct ibv_qp *qp, memcpy (fa.spec_eth.val.dst_mac, mac, sizeof (fa.spec_eth.val.dst_mac)); memcpy (fa.spec_eth.mask.dst_mac, mask, sizeof (fa.spec_eth.mask.dst_mac)); + if (ether_type) + { + fa.spec_eth.val.ether_type = ether_type; + fa.spec_eth.mask.ether_type = 0xffff; + } + flow = ibv_create_flow (qp, &fa.attr); if (!flow) rdma_log (VLIB_LOG_LEVEL_ERR, rd, "ibv_create_flow() failed"); @@ -104,16 +114,17 @@ rdma_dev_set_promisc (rdma_device_t * rd) const mac_address_t all = {.bytes = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0} }; int err; - err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast); - if (err) - return ~0; - - err = rdma_rxq_destroy_flow (rd, &rd->flow_ucast); + err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast6); + err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast6); + err |= rdma_rxq_destroy_flow (rd, &rd->flow_mcast4); + err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast4); if (err) return ~0; - rd->flow_ucast = rdma_rxq_init_flow (rd, rd->rx_qp, &all, &all, 0); - if (!rd->flow_ucast) + rd->flow_ucast6 = + rdma_rxq_init_flow (rd, rd->rx_qp6, &all, &all, ntohs (ETH_P_IPV6), 0); + rd->flow_ucast4 = rdma_rxq_init_flow (rd, rd->rx_qp4, &all, &all, 0, 0); + if (!rd->flow_ucast6 || !rd->flow_ucast4) return ~0; rd->flags |= RDMA_DEVICE_F_PROMISC; @@ -128,25 +139,30 @@ rdma_dev_set_ucast (rdma_device_t * rd) const mac_address_t mcast = {.bytes = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0} }; int err; - err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast); + err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast6); + err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast6); + err |= rdma_rxq_destroy_flow (rd, &rd->flow_mcast4); + err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast4); if (err) return ~0; - err = rdma_rxq_destroy_flow (rd, &rd->flow_ucast); - if (err) - return ~0; - - /* receive only packets with src = our MAC */ - rd->flow_ucast = rdma_rxq_init_flow (rd, rd->rx_qp, &rd->hwaddr, &ucast, 0); - if (!rd->flow_ucast) - return ~0; - - /* receive multicast packets */ - rd->flow_mcast = rdma_rxq_init_flow (rd, rd->rx_qp, &mcast, &mcast, - IBV_FLOW_ATTR_FLAGS_DONT_TRAP - /* let others receive mcast packet too (eg. Linux) */ + rd->flow_ucast6 = + rdma_rxq_init_flow (rd, rd->rx_qp6, &rd->hwaddr, &ucast, + ntohs (ETH_P_IPV6), 0); + rd->flow_mcast6 = + rdma_rxq_init_flow (rd, rd->rx_qp6, &mcast, &mcast, ntohs (ETH_P_IPV6), + IBV_FLOW_ATTR_FLAGS_DONT_TRAP + /* let others receive mcast packet too (eg. Linux) */ + ); + rd->flow_ucast4 = + rdma_rxq_init_flow (rd, rd->rx_qp4, &rd->hwaddr, &ucast, 0, 0); + rd->flow_mcast4 = + rdma_rxq_init_flow (rd, rd->rx_qp4, &mcast, &mcast, 0, + IBV_FLOW_ATTR_FLAGS_DONT_TRAP + /* let others receive mcast packet too (eg. Linux) */ ); - if (!rd->flow_mcast) + if (!rd->flow_ucast6 || !rd->flow_mcast6 || !rd->flow_ucast4 + || !rd->flow_mcast4) return ~0; rd->flags &= ~RDMA_DEVICE_F_PROMISC; @@ -182,7 +198,7 @@ rdma_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags) switch (flags) { - case 0: + case ETHERNET_INTERFACE_FLAG_DEFAULT_L3: return rdma_dev_set_ucast (rd); case ETHERNET_INTERFACE_FLAG_ACCEPT_ALL: return rdma_dev_set_promisc (rd); @@ -266,8 +282,7 @@ rdma_async_event_error_ready (clib_file_t * f) { rdma_main_t *rm = &rdma_main; rdma_device_t *rd = vec_elt_at_index (rm->devices, f->private_data); - return clib_error_return (0, "RDMA async event error for device %U", - format_vlib_pci_addr, &rd->pci_addr); + return clib_error_return (0, "RDMA: %s: async event error", rd->name); } static clib_error_t * @@ -293,8 +308,7 @@ rdma_async_event_read_ready (clib_file_t * f) case IBV_EVENT_DEVICE_FATAL: rd->flags &= ~RDMA_DEVICE_F_LINK_UP; vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0); - vlib_log_emerg (rm->log_class, "Fatal RDMA error for device %U", - format_vlib_pci_addr, &rd->pci_addr); + vlib_log_emerg (rm->log_class, "%s: fatal error", rd->name); break; default: rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "unhandeld RDMA async event %i", @@ -326,8 +340,7 @@ rdma_async_event_init (rdma_device_t * rd) t.file_descriptor = rd->ctx->async_fd; t.error_function = rdma_async_event_error_ready; t.private_data = rd->dev_instance; - t.description = - format (0, "RMDA %U async event", format_vlib_pci_addr, &rd->pci_addr); + t.description = format (0, "%v async event", rd->name); rd->async_event_clib_file_index = clib_file_add (&file_main, &t); return 0; @@ -342,16 +355,24 @@ rdma_async_event_cleanup (rdma_device_t * rd) static clib_error_t * rdma_register_interface (vnet_main_t * vnm, rdma_device_t * rd) { - return ethernet_register_interface (vnm, rdma_device_class.index, - rd->dev_instance, rd->hwaddr.bytes, - &rd->hw_if_index, rdma_flag_change); + clib_error_t *err = + ethernet_register_interface (vnm, rdma_device_class.index, + rd->dev_instance, rd->hwaddr.bytes, + &rd->hw_if_index, rdma_flag_change); + + /* Indicate ability to support L3 DMAC filtering and + * initialize interface to L3 non-promisc mode */ + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, rd->hw_if_index); + hi->caps |= VNET_HW_INTERFACE_CAP_SUPPORTS_MAC_FILTER; + ethernet_set_flags (vnm, rd->hw_if_index, + ETHERNET_INTERFACE_FLAG_DEFAULT_L3); + return err; } static void rdma_unregister_interface (vnet_main_t * vnm, rdma_device_t * rd) { vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0); - vnet_hw_interface_unassign_rx_thread (vnm, rd->hw_if_index, 0); ethernet_delete_interface (vnm, rd->hw_if_index); } @@ -369,8 +390,10 @@ rdma_dev_cleanup (rdma_device_t * rd) rdma_log (VLIB_LOG_LEVEL_DEBUG, rd, #fn "() failed (rv = %d)", rv); \ } - _(ibv_destroy_flow, rd->flow_mcast); - _(ibv_destroy_flow, rd->flow_ucast); + _(ibv_destroy_flow, rd->flow_mcast6); + _(ibv_destroy_flow, rd->flow_ucast6); + _(ibv_destroy_flow, rd->flow_mcast4); + _(ibv_destroy_flow, rd->flow_ucast4); _(ibv_dereg_mr, rd->mr); vec_foreach (txq, rd->txqs) { @@ -383,7 +406,8 @@ rdma_dev_cleanup (rdma_device_t * rd) _(ibv_destroy_cq, rxq->cq); } _(ibv_destroy_rwq_ind_table, rd->rx_rwq_ind_tbl); - _(ibv_destroy_qp, rd->rx_qp); + _(ibv_destroy_qp, rd->rx_qp6); + _(ibv_destroy_qp, rd->rx_qp4); _(ibv_dealloc_pd, rd->pd); _(ibv_close_device, rd->ctx); #undef _ @@ -393,22 +417,47 @@ rdma_dev_cleanup (rdma_device_t * rd) vec_free (rd->rxqs); vec_free (rd->txqs); vec_free (rd->name); + vlib_pci_free_device_info (rd->pci); pool_put (rm->devices, rd); } static clib_error_t * -rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc) +rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc, + u8 no_multi_seg, u16 max_pktlen) { rdma_rxq_t *rxq; struct ibv_wq_init_attr wqia; + struct ibv_cq_init_attr_ex cqa = { }; struct ibv_wq_attr wqa; + struct ibv_cq_ex *cqex; + struct mlx5dv_wq_init_attr dv_wqia = { }; + int is_mlx5dv = ! !(rd->flags & RDMA_DEVICE_F_MLX5DV); + int is_striding = ! !(rd->flags & RDMA_DEVICE_F_STRIDING_RQ); vec_validate_aligned (rd->rxqs, qid, CLIB_CACHE_LINE_BYTES); rxq = vec_elt_at_index (rd->rxqs, qid); rxq->size = n_desc; + rxq->log_wqe_sz = 0; + rxq->buf_sz = vlib_buffer_get_default_data_size (vm); + vec_validate_aligned (rxq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES); - if ((rxq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0) - return clib_error_return_unix (0, "Create CQ Failed"); + cqa.cqe = n_desc; + if (is_mlx5dv) + { + struct mlx5dv_cq_init_attr dvcq = { }; + dvcq.comp_mask = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE; + dvcq.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH; + + if ((cqex = mlx5dv_create_cq (rd->ctx, &cqa, &dvcq)) == 0) + return clib_error_return_unix (0, "Create mlx5dv rx CQ Failed"); + } + else + { + if ((cqex = ibv_create_cq_ex (rd->ctx, &cqa)) == 0) + return clib_error_return_unix (0, "Create CQ Failed"); + } + + rxq->cq = ibv_cq_ex_to_cq (cqex); memset (&wqia, 0, sizeof (wqia)); wqia.wq_type = IBV_WQT_RQ; @@ -416,7 +465,74 @@ rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc) wqia.max_sge = 1; wqia.pd = rd->pd; wqia.cq = rxq->cq; - if ((rxq->wq = ibv_create_wq (rd->ctx, &wqia)) == 0) + if (is_mlx5dv) + { + if (is_striding) + { + /* In STRIDING_RQ mode, map a descriptor to a stride, not a full WQE buffer */ + uword data_seg_log2_sz = + min_log2 (vlib_buffer_get_default_data_size (vm)); + rxq->buf_sz = 1 << data_seg_log2_sz; + /* The trick is also to map a descriptor to a data segment in the WQE SG list + The number of strides per WQE and the size of a WQE (in 16-bytes words) both + must be powers of two. + Moreover, in striding RQ mode, WQEs must include the SRQ header, which occupies + one 16-bytes word. That is why WQEs have 2*RDMA_RXQ_MAX_CHAIN_SZ 16-bytes words: + - One for the SRQ Header + - RDMA_RXQ_MAX_CHAIN_SZ for the different data segments (each mapped to + a stride, and a vlib_buffer) + - RDMA_RXQ_MAX_CHAIN_SZ-1 null data segments + */ + int max_chain_log_sz = + max_pktlen ? max_log2 ((max_pktlen / + (rxq->buf_sz)) + + 1) : RDMA_RXQ_MAX_CHAIN_LOG_SZ; + max_chain_log_sz = clib_max (max_chain_log_sz, 3); + wqia.max_sge = 1 << max_chain_log_sz; + dv_wqia.comp_mask = MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ; + dv_wqia.striding_rq_attrs.two_byte_shift_en = 0; + dv_wqia.striding_rq_attrs.single_wqe_log_num_of_strides = + max_chain_log_sz; + dv_wqia.striding_rq_attrs.single_stride_log_num_of_bytes = + data_seg_log2_sz; + wqia.max_wr >>= max_chain_log_sz; + rxq->log_wqe_sz = max_chain_log_sz + 1; + rxq->log_stride_per_wqe = max_chain_log_sz; + } + else + { + /* In non STRIDING_RQ mode and if multiseg is not disabled, each WQE is a SG list of data + segments, each pointing to a vlib_buffer. */ + if (no_multi_seg) + { + wqia.max_sge = 1; + rxq->log_wqe_sz = 0; + rxq->n_ds_per_wqe = 1; + } + else + { + int max_chain_sz = + max_pktlen ? (max_pktlen / + (rxq->buf_sz)) + + 1 : RDMA_RXQ_LEGACY_MODE_MAX_CHAIN_SZ; + int max_chain_log_sz = max_log2 (max_chain_sz); + wqia.max_sge = 1 << max_chain_log_sz; + rxq->log_wqe_sz = max_chain_log_sz; + rxq->n_ds_per_wqe = max_chain_sz; + } + + } + + if ((rxq->wq = mlx5dv_create_wq (rd->ctx, &wqia, &dv_wqia))) + { + rxq->wq->events_completed = 0; + pthread_mutex_init (&rxq->wq->mutex, NULL); + pthread_cond_init (&rxq->wq->cond, NULL); + } + else + return clib_error_return_unix (0, "Create WQ Failed"); + } + else if ((rxq->wq = ibv_create_wq (rd->ctx, &wqia)) == 0) return clib_error_return_unix (0, "Create WQ Failed"); memset (&wqa, 0, sizeof (wqa)); @@ -425,6 +541,73 @@ rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc) if (ibv_modify_wq (rxq->wq, &wqa) != 0) return clib_error_return_unix (0, "Modify WQ (RDY) Failed"); + if (is_mlx5dv) + { + struct mlx5dv_obj obj = { }; + struct mlx5dv_cq dv_cq; + struct mlx5dv_rwq dv_rwq; + u64 qw0; + u64 qw0_nullseg; + u32 wqe_sz_mask = (1 << rxq->log_wqe_sz) - 1; + + obj.cq.in = rxq->cq; + obj.cq.out = &dv_cq; + obj.rwq.in = rxq->wq; + obj.rwq.out = &dv_rwq; + + if ((mlx5dv_init_obj (&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_RWQ))) + return clib_error_return_unix (0, "mlx5dv: failed to init rx obj"); + + if (dv_cq.cqe_size != sizeof (mlx5dv_cqe_t)) + return clib_error_return_unix (0, "mlx5dv: incompatible rx CQE size"); + + rxq->log2_cq_size = max_log2 (dv_cq.cqe_cnt); + rxq->cqes = (mlx5dv_cqe_t *) dv_cq.buf; + rxq->cq_db = (volatile u32 *) dv_cq.dbrec; + rxq->cqn = dv_cq.cqn; + + rxq->wqes = (mlx5dv_wqe_ds_t *) dv_rwq.buf; + rxq->wq_db = (volatile u32 *) dv_rwq.dbrec; + rxq->wq_stride = dv_rwq.stride; + rxq->wqe_cnt = dv_rwq.wqe_cnt; + + qw0 = clib_host_to_net_u32 (rxq->buf_sz); + qw0_nullseg = 0; + qw0 |= (u64) clib_host_to_net_u32 (rd->lkey) << 32; + qw0_nullseg |= (u64) clib_host_to_net_u32 (rd->lkey) << 32; + +/* Prefill the different 16 bytes words of the WQ. + - If not in striding RQ mode, for each WQE, init with qw0 the first + RDMA_RXQ_LEGACY_MODE_MAX_CHAIN_SZ, and init the rest of the WQE + with null segments. + - If in striding RQ mode, for each WQE, the RDMA_RXQ_MAX_CHAIN_SZ + 1 + first 16-bytes words are initialised with qw0, the rest are null segments */ + + for (int i = 0; i < rxq->wqe_cnt << rxq->log_wqe_sz; i++) + if ((!is_striding + && ((i & wqe_sz_mask) < rxq->n_ds_per_wqe)) + || (is_striding + && ((i == 0) + || !(((i - 1) >> rxq->log_stride_per_wqe) & 0x1)))) + rxq->wqes[i].dsz_and_lkey = qw0; + else + rxq->wqes[i].dsz_and_lkey = qw0_nullseg; + + for (int i = 0; i < (1 << rxq->log2_cq_size); i++) + rxq->cqes[i].opcode_cqefmt_se_owner = 0xff; + + if (!is_striding) + { + vec_validate_aligned (rxq->second_bufs, n_desc - 1, + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (rxq->n_used_per_chain, n_desc - 1, + CLIB_CACHE_LINE_BYTES); + rxq->n_total_additional_segs = n_desc * (rxq->n_ds_per_wqe - 1); + for (int i = 0; i < n_desc; i++) + rxq->n_used_per_chain[i] = rxq->n_ds_per_wqe - 1; + } + } + return 0; } @@ -460,10 +643,18 @@ rdma_rxq_finalize (vlib_main_t * vm, rdma_device_t * rd) qpia.rx_hash_conf.rx_hash_key_len = sizeof (rdma_rss_hash_key); qpia.rx_hash_conf.rx_hash_key = rdma_rss_hash_key; qpia.rx_hash_conf.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ; + qpia.rx_hash_conf.rx_hash_fields_mask = - IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4; - if ((rd->rx_qp = ibv_create_qp_ex (rd->ctx, &qpia)) == 0) - return clib_error_return_unix (0, "Queue Pair create failed"); + IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4 | IBV_RX_HASH_SRC_PORT_TCP | + IBV_RX_HASH_DST_PORT_TCP; + if ((rd->rx_qp4 = ibv_create_qp_ex (rd->ctx, &qpia)) == 0) + return clib_error_return_unix (0, "IPv4 Queue Pair create failed"); + + qpia.rx_hash_conf.rx_hash_fields_mask = + IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6 | IBV_RX_HASH_SRC_PORT_TCP | + IBV_RX_HASH_DST_PORT_TCP; + if ((rd->rx_qp6 = ibv_create_qp_ex (rd->ctx, &qpia)) == 0) + return clib_error_return_unix (0, "IPv6 Queue Pair create failed"); if (rdma_dev_set_ucast (rd)) return clib_error_return_unix (0, "Set unicast mode failed"); @@ -481,7 +672,9 @@ rdma_txq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc) vec_validate_aligned (rd->txqs, qid, CLIB_CACHE_LINE_BYTES); txq = vec_elt_at_index (rd->txqs, qid); - txq->size = n_desc; + ASSERT (is_pow2 (n_desc)); + txq->bufs_log2sz = min_log2 (n_desc); + vec_validate_aligned (txq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES); if ((txq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0) return clib_error_return_unix (0, "Create CQ Failed"); @@ -492,7 +685,6 @@ rdma_txq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc) qpia.cap.max_send_wr = n_desc; qpia.cap.max_send_sge = 1; qpia.qp_type = IBV_QPT_RAW_PACKET; - qpia.sq_sig_all = 1; if ((txq->qp = ibv_create_qp (rd->pd, &qpia)) == 0) return clib_error_return_unix (0, "Queue Pair create failed"); @@ -515,16 +707,70 @@ rdma_txq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc) qpa.qp_state = IBV_QPS_RTS; if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0) return clib_error_return_unix (0, "Modify QP (send) Failed"); + + txq->ibv_cq = txq->cq; + txq->ibv_qp = txq->qp; + + if (rd->flags & RDMA_DEVICE_F_MLX5DV) + { + rdma_mlx5_wqe_t *tmpl = (void *) txq->dv_wqe_tmpl; + struct mlx5dv_cq dv_cq; + struct mlx5dv_qp dv_qp; + struct mlx5dv_obj obj = { }; + + obj.cq.in = txq->cq; + obj.cq.out = &dv_cq; + obj.qp.in = txq->qp; + obj.qp.out = &dv_qp; + + if (mlx5dv_init_obj (&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP)) + return clib_error_return_unix (0, "DV init obj failed"); + + if (RDMA_TXQ_BUF_SZ (txq) > dv_qp.sq.wqe_cnt + || !is_pow2 (dv_qp.sq.wqe_cnt) + || sizeof (rdma_mlx5_wqe_t) != dv_qp.sq.stride + || (uword) dv_qp.sq.buf % sizeof (rdma_mlx5_wqe_t)) + return clib_error_return (0, "Unsupported DV SQ parameters"); + + if (RDMA_TXQ_BUF_SZ (txq) > dv_cq.cqe_cnt + || !is_pow2 (dv_cq.cqe_cnt) + || sizeof (struct mlx5_cqe64) != dv_cq.cqe_size + || (uword) dv_cq.buf % sizeof (struct mlx5_cqe64)) + return clib_error_return (0, "Unsupported DV CQ parameters"); + + /* get SQ and doorbell addresses */ + txq->dv_sq_wqes = dv_qp.sq.buf; + txq->dv_sq_dbrec = dv_qp.dbrec; + txq->dv_sq_db = dv_qp.bf.reg; + txq->dv_sq_log2sz = min_log2 (dv_qp.sq.wqe_cnt); + + /* get CQ and doorbell addresses */ + txq->dv_cq_cqes = dv_cq.buf; + txq->dv_cq_dbrec = dv_cq.dbrec; + txq->dv_cq_log2sz = min_log2 (dv_cq.cqe_cnt); + + /* init tx desc template */ + STATIC_ASSERT_SIZEOF (txq->dv_wqe_tmpl, sizeof (*tmpl)); + mlx5dv_set_ctrl_seg (&tmpl->ctrl, 0, MLX5_OPCODE_SEND, 0, + txq->qp->qp_num, 0, RDMA_MLX5_WQE_DS, 0, + RDMA_TXQ_DV_INVALID_ID); + tmpl->eseg.inline_hdr_sz = htobe16 (MLX5_ETH_L2_INLINE_HEADER_SIZE); + mlx5dv_set_data_seg (&tmpl->dseg, 0, rd->lkey, 0); + } + return 0; } static clib_error_t * -rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd, u32 rxq_size, - u32 txq_size, u32 rxq_num) +rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd, + rdma_create_if_args_t * args) { clib_error_t *err; vlib_buffer_main_t *bm = vm->buffer_main; vlib_thread_main_t *tm = vlib_get_thread_main (); + u32 rxq_num = args->rxq_num; + u32 rxq_size = args->rxq_size; + u32 txq_size = args->txq_size; u32 i; if (rd->ctx == 0) @@ -533,22 +779,31 @@ rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd, u32 rxq_size, if ((rd->pd = ibv_alloc_pd (rd->ctx)) == 0) return clib_error_return_unix (0, "PD Alloc Failed"); - ethernet_mac_address_generate (rd->hwaddr.bytes); + if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start, + bm->buffer_mem_size, + IBV_ACCESS_LOCAL_WRITE)) == 0) + return clib_error_return_unix (0, "Register MR Failed"); - for (i = 0; i < rxq_num; i++) - if ((err = rdma_rxq_init (vm, rd, i, rxq_size))) - return err; - if ((err = rdma_rxq_finalize (vm, rd))) - return err; + rd->lkey = rd->mr->lkey; /* avoid indirection in datapath */ + ethernet_mac_address_generate (rd->hwaddr.bytes); + + /* + * /!\ WARNING /!\ creation order is important + * We *must* create TX queues *before* RX queues, otherwise we will receive + * the broacast packets we sent + */ for (i = 0; i < tm->n_vlib_mains; i++) if ((err = rdma_txq_init (vm, rd, i, txq_size))) return err; - if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start, - bm->buffer_mem_size, - IBV_ACCESS_LOCAL_WRITE)) == 0) - return clib_error_return_unix (0, "Register MR Failed"); + for (i = 0; i < rxq_num; i++) + if ((err = + rdma_rxq_init (vm, rd, i, rxq_size, + args->no_multi_seg, args->max_pktlen))) + return err; + if ((err = rdma_rxq_finalize (vm, rd))) + return err; return 0; } @@ -561,6 +816,9 @@ sysfs_path_to_pci_addr (char *path, vlib_pci_addr_t * addr) u8 *s; s = clib_sysfs_link_to_name (path); + if (!s) + return 0; + unformat_init_string (&in, (char *) s, strlen ((char *) s)); rv = unformat (&in, "%U", unformat_vlib_pci_addr, addr); unformat_free (&in); @@ -573,69 +831,84 @@ rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args) { vnet_main_t *vnm = vnet_get_main (); rdma_main_t *rm = &rdma_main; - rdma_device_t *rd = 0; - struct ibv_device **dev_list = 0; + rdma_device_t *rd; + vlib_pci_addr_t pci_addr; + struct ibv_device **dev_list; int n_devs; - u8 *s = 0, *s2 = 0; + u8 *s; u16 qid; + int i; - args->rxq_size = args->rxq_size ? args->rxq_size : 2 * VLIB_FRAME_SIZE; - args->txq_size = args->txq_size ? args->txq_size : 2 * VLIB_FRAME_SIZE; - args->rxq_num = args->rxq_num ? args->rxq_num : 1; + args->rxq_size = args->rxq_size ? args->rxq_size : 1024; + args->txq_size = args->txq_size ? args->txq_size : 1024; + args->rxq_num = args->rxq_num ? args->rxq_num : 2; if (!is_pow2 (args->rxq_num)) { args->rv = VNET_API_ERROR_INVALID_VALUE; args->error = clib_error_return (0, "rx queue number must be a power of two"); - return; + goto err0; } - if (!is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size)) + if (args->rxq_size < VLIB_FRAME_SIZE || args->txq_size < VLIB_FRAME_SIZE || + args->rxq_size > 65535 || args->txq_size > 65535 || + !is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size)) { args->rv = VNET_API_ERROR_INVALID_VALUE; - args->error = - clib_error_return (0, "queue size must be a power of two"); - return; + args->error = clib_error_return (0, "queue size must be a power of two " + "between %i and 65535", + VLIB_FRAME_SIZE); + goto err0; } - pool_get_zero (rm->devices, rd); - rd->dev_instance = rd - rm->devices; - rd->per_interface_next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; - rd->name = vec_dup (args->name); - - /* check if device exist and if it is bound to mlx5_core */ - s = format (s, "/sys/class/net/%s/device/driver/module%c", args->ifname, 0); - s2 = clib_sysfs_link_to_name ((char *) s); - - if (s2 == 0 || strncmp ((char *) s2, "mlx5_core", 9) != 0) + dev_list = ibv_get_device_list (&n_devs); + if (n_devs == 0) { args->error = - clib_error_return (0, - "invalid interface (only mlx5 supported for now)"); + clib_error_return_unix (0, + "no RDMA devices available. Is the ib_uverbs module loaded?"); goto err0; } - /* extract PCI address */ - vec_reset_length (s); - s = format (s, "/sys/class/net/%s/device%c", args->ifname, 0); - if (sysfs_path_to_pci_addr ((char *) s, &rd->pci_addr) == 0) + /* get PCI address */ + s = format (0, "/sys/class/net/%s/device%c", args->ifname, 0); + if (sysfs_path_to_pci_addr ((char *) s, &pci_addr) == 0) { - args->error = clib_error_return (0, "cannot find PCI address"); - goto err0; + args->error = + clib_error_return (0, "cannot find PCI address for device "); + goto err1; } - dev_list = ibv_get_device_list (&n_devs); - if (n_devs == 0) + pool_get_zero (rm->devices, rd); + rd->dev_instance = rd - rm->devices; + rd->per_interface_next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; + rd->linux_ifname = format (0, "%s", args->ifname); + + if (!args->name || 0 == args->name[0]) + rd->name = format (0, "%s/%d", args->ifname, rd->dev_instance); + else + rd->name = format (0, "%s", args->name); + + rd->pci = vlib_pci_get_device_info (vm, &pci_addr, &args->error); + if (!rd->pci) + goto err2; + + /* if we failed to parse NUMA node, default to 0 */ + if (-1 == rd->pci->numa_node) + rd->pci->numa_node = 0; + + rd->pool = vlib_buffer_pool_get_default_for_numa (vm, rd->pci->numa_node); + + if (strncmp ((char *) rd->pci->driver_name, "mlx5_core", 9)) { args->error = - clib_error_return_unix (0, - "no RDMA devices available, errno = %d. " - "Is the ib_uverbs module loaded?", errno); - goto err0; + clib_error_return (0, + "invalid interface (only mlx5 supported for now)"); + goto err2; } - for (int i = 0; i < n_devs; i++) + for (i = 0; i < n_devs; i++) { vlib_pci_addr_t addr; @@ -645,16 +918,52 @@ rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args) if (sysfs_path_to_pci_addr ((char *) s, &addr) == 0) continue; - if (addr.as_u32 != rd->pci_addr.as_u32) + if (addr.as_u32 != rd->pci->addr.as_u32) continue; if ((rd->ctx = ibv_open_device (dev_list[i]))) break; } - if ((args->error = - rdma_dev_init (vm, rd, args->rxq_size, args->txq_size, args->rxq_num))) - goto err1; + if (args->mode != RDMA_MODE_IBV) + { + struct mlx5dv_context mlx5dv_attrs = { }; + mlx5dv_attrs.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ; + + if (mlx5dv_query_device (rd->ctx, &mlx5dv_attrs) == 0) + { + uword data_seg_log2_sz = + min_log2 (vlib_buffer_get_default_data_size (vm)); + + if ((mlx5dv_attrs.flags & MLX5DV_CONTEXT_FLAGS_CQE_V1)) + rd->flags |= RDMA_DEVICE_F_MLX5DV; + +/* Enable striding RQ if neither multiseg nor striding rq +are explicitly disabled, and if the interface supports it.*/ + if (!args->no_multi_seg && !args->disable_striding_rq + && data_seg_log2_sz <= + mlx5dv_attrs.striding_rq_caps.max_single_stride_log_num_of_bytes + && data_seg_log2_sz >= + mlx5dv_attrs.striding_rq_caps.min_single_stride_log_num_of_bytes + && RDMA_RXQ_MAX_CHAIN_LOG_SZ >= + mlx5dv_attrs.striding_rq_caps.min_single_wqe_log_num_of_strides + && RDMA_RXQ_MAX_CHAIN_LOG_SZ <= + mlx5dv_attrs.striding_rq_caps.max_single_wqe_log_num_of_strides) + rd->flags |= RDMA_DEVICE_F_STRIDING_RQ; + } + else + { + if (args->mode == RDMA_MODE_DV) + { + args->error = clib_error_return (0, "Direct Verbs mode not " + "supported on this interface"); + goto err2; + } + } + } + + if ((args->error = rdma_dev_init (vm, rd, args))) + goto err2; if ((args->error = rdma_register_interface (vnm, rd))) goto err2; @@ -669,12 +978,18 @@ rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args) /* * FIXME: add support for interrupt mode * vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, rd->hw_if_index); - * hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE; + * hw->caps |= VNET_HW_INTERFACE_CAP_SUPPORTS_INT_MODE; */ - vnet_hw_interface_set_input_node (vnm, rd->hw_if_index, - rdma_input_node.index); + vnet_hw_if_set_input_node (vnm, rd->hw_if_index, rdma_input_node.index); + vec_foreach_index (qid, rd->rxqs) - vnet_hw_interface_assign_rx_thread (vnm, rd->hw_if_index, qid, ~0); + { + u32 queue_index = vnet_hw_if_register_rx_queue ( + vnm, rd->hw_if_index, qid, VNET_HW_IF_RXQ_THREAD_ANY); + rd->rxqs[qid].queue_index = queue_index; + } + vnet_hw_if_update_runtime_data (vnm, rd->hw_if_index); + vec_free (s); return; err3: @@ -683,10 +998,9 @@ err2: rdma_dev_cleanup (rd); err1: ibv_free_device_list (dev_list); -err0: - vec_free (s2); vec_free (s); args->rv = VNET_API_ERROR_INVALID_INTERFACE; +err0: vlib_log_err (rm->log_class, "%U", format_clib_error, args->error); } @@ -730,15 +1044,9 @@ rdma_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index, rdma_main_t *rm = &rdma_main; vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); rdma_device_t *rd = pool_elt_at_index (rm->devices, hw->dev_instance); - - /* Shut off redirection */ - if (node_index == ~0) - { - rd->per_interface_next_index = node_index; - return; - } - rd->per_interface_next_index = + ~0 == + node_index ? VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT : vlib_node_add_next (vlib_get_main (), rdma_input_node.index, node_index); } @@ -748,7 +1056,6 @@ static char *rdma_tx_func_error_strings[] = { #undef _ }; -/* *INDENT-OFF* */ VNET_DEVICE_CLASS (rdma_device_class) = { .name = "RDMA interface", @@ -760,19 +1067,35 @@ VNET_DEVICE_CLASS (rdma_device_class) = .tx_function_error_strings = rdma_tx_func_error_strings, .mac_addr_change_function = rdma_mac_change, }; -/* *INDENT-ON* */ clib_error_t * rdma_init (vlib_main_t * vm) { rdma_main_t *rm = &rdma_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); rm->log_class = vlib_log_register_class ("rdma", 0); + /* vlib_buffer_t template */ + vec_validate_aligned (rm->per_thread_data, tm->n_vlib_mains - 1, + CLIB_CACHE_LINE_BYTES); + + for (int i = 0; i < tm->n_vlib_mains; i++) + { + rdma_per_thread_data_t *ptd = vec_elt_at_index (rm->per_thread_data, i); + clib_memset (&ptd->buffer_template, 0, sizeof (vlib_buffer_t)); + ptd->buffer_template.flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; + ptd->buffer_template.ref_count = 1; + vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0; + } + return 0; } -VLIB_INIT_FUNCTION (rdma_init); +VLIB_INIT_FUNCTION (rdma_init) = +{ + .runs_after = VLIB_INITS ("pci_bus_init"), +}; /* * fd.io coding-style-patch-verification: ON