2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
21 #include <linux/if_link.h>
22 #include <linux/if_ether.h>
24 #include <vppinfra/linux/sysfs.h>
25 #include <vlib/vlib.h>
26 #include <vlib/unix/unix.h>
27 #include <vlib/pci/pci.h>
28 #include <vnet/ethernet/ethernet.h>
30 #include <rdma/rdma.h>
32 /* Default RSS hash key (from DPDK MLX driver) */
33 static u8 rdma_rss_hash_key[] = {
34 0x2c, 0xc6, 0x81, 0xd1,
35 0x5b, 0xdb, 0xf4, 0xf7,
36 0xfc, 0xa2, 0x83, 0x19,
37 0xdb, 0x1a, 0x3e, 0x94,
38 0x6b, 0x9e, 0x38, 0xd9,
39 0x2c, 0x9c, 0x03, 0xd1,
40 0xad, 0x99, 0x44, 0xa7,
41 0xd9, 0x56, 0x3d, 0x59,
42 0x06, 0x3c, 0x25, 0xf3,
43 0xfc, 0x1f, 0xdc, 0x2a,
46 rdma_main_t rdma_main;
48 #define rdma_log_debug(dev, f, ...) \
50 vlib_log(VLIB_LOG_LEVEL_DEBUG, rdma_main.log_class, "%U: " f, \
51 format_vlib_pci_addr, &rd->pci_addr, ##__VA_ARGS__); \
55 rdma_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
57 rdma_main_t *rm = &rdma_main;
58 vlib_log_warn (rm->log_class, "TODO");
63 rdma_update_state (vnet_main_t * vnm, rdma_device_t * rd, int port)
65 struct ibv_port_attr attr;
69 if (ibv_query_port (rd->ctx, port, &attr))
71 vnet_hw_interface_set_link_speed (vnm, rd->hw_if_index, 0);
72 vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0);
79 case IBV_PORT_ACTIVE: /* fallthrough */
80 case IBV_PORT_ACTIVE_DEFER:
81 rd->flags |= RDMA_DEVICE_F_LINK_UP;
82 vnet_hw_interface_set_flags (vnm, rd->hw_if_index,
83 VNET_HW_INTERFACE_FLAG_LINK_UP);
86 rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
87 vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0);
92 switch (attr.active_width)
107 switch (attr.active_speed)
115 case 4: /* fallthrough */
126 vnet_hw_interface_set_link_speed (vnm, rd->hw_if_index, width * speed);
129 static clib_error_t *
130 rdma_async_event_error_ready (clib_file_t * f)
132 rdma_main_t *rm = &rdma_main;
133 rdma_device_t *rd = vec_elt_at_index (rm->devices, f->private_data);
134 return clib_error_return (0, "RDMA async event error for device %U",
135 format_vlib_pci_addr, &rd->pci_addr);
138 static clib_error_t *
139 rdma_async_event_read_ready (clib_file_t * f)
141 vnet_main_t *vnm = vnet_get_main ();
142 rdma_main_t *rm = &rdma_main;
143 rdma_device_t *rd = vec_elt_at_index (rm->devices, f->private_data);
145 struct ibv_async_event event;
146 ret = ibv_get_async_event (rd->ctx, &event);
149 return clib_error_return_unix (0, "ibv_get_async_event() failed");
152 switch (event.event_type)
154 case IBV_EVENT_PORT_ACTIVE:
155 rdma_update_state (vnm, rd, event.element.port_num);
157 case IBV_EVENT_PORT_ERR:
158 rdma_update_state (vnm, rd, event.element.port_num);
160 case IBV_EVENT_DEVICE_FATAL:
161 rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
162 vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0);
163 vlib_log_emerg (rm->log_class, "Fatal RDMA error for device %U",
164 format_vlib_pci_addr, &rd->pci_addr);
167 vlib_log_warn (rm->log_class,
168 "Unhandeld RDMA async event %i for device %U",
169 event.event_type, format_vlib_pci_addr, &rd->pci_addr);
173 ibv_ack_async_event (&event);
177 static clib_error_t *
178 rdma_async_event_init (rdma_device_t * rd)
180 clib_file_t t = { 0 };
183 /* make RDMA async event fd non-blocking */
184 ret = fcntl (rd->ctx->async_fd, F_GETFL);
187 return clib_error_return_unix (0, "fcntl(F_GETFL) failed");
189 ret = fcntl (rd->ctx->async_fd, F_SETFL, ret | O_NONBLOCK);
192 return clib_error_return_unix (0, "fcntl(F_SETFL, O_NONBLOCK) failed");
195 /* register RDMA async event fd */
196 t.read_function = rdma_async_event_read_ready;
197 t.file_descriptor = rd->ctx->async_fd;
198 t.error_function = rdma_async_event_error_ready;
199 t.private_data = rd->dev_instance;
201 format (0, "RMDA %U async event", format_vlib_pci_addr, &rd->pci_addr);
203 rd->async_event_clib_file_index = clib_file_add (&file_main, &t);
209 rdma_async_event_cleanup (rdma_device_t * rd)
211 clib_file_del_by_index (&file_main, rd->async_event_clib_file_index);
214 static clib_error_t *
215 rdma_register_interface (vnet_main_t * vnm, rdma_device_t * rd)
217 return ethernet_register_interface (vnm, rdma_device_class.index,
218 rd->dev_instance, rd->hwaddr.bytes,
219 &rd->hw_if_index, rdma_flag_change);
223 rdma_unregister_interface (vnet_main_t * vnm, rdma_device_t * rd)
225 vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0);
226 vnet_hw_interface_unassign_rx_thread (vnm, rd->hw_if_index, 0);
227 ethernet_delete_interface (vnm, rd->hw_if_index);
231 rdma_dev_cleanup (rdma_device_t * rd)
233 rdma_main_t *rm = &rdma_main;
237 #define _(fn, arg) if (arg) \
240 if ((rv = fn (arg))) \
241 rdma_log_debug (rd, #fn "() failed (rv = %d)", rv); \
244 _(ibv_destroy_flow, rd->flow_mcast);
245 _(ibv_destroy_flow, rd->flow_ucast);
246 _(ibv_dereg_mr, rd->mr);
247 vec_foreach (txq, rd->txqs)
249 _(ibv_destroy_qp, txq->qp);
250 _(ibv_destroy_cq, txq->cq);
252 vec_foreach (rxq, rd->rxqs)
254 _(ibv_destroy_wq, rxq->wq);
255 _(ibv_destroy_cq, rxq->cq);
257 _(ibv_destroy_rwq_ind_table, rd->rx_rwq_ind_tbl);
258 _(ibv_destroy_qp, rd->rx_qp);
259 _(ibv_dealloc_pd, rd->pd);
260 _(ibv_close_device, rd->ctx);
263 clib_error_free (rd->error);
268 pool_put (rm->devices, rd);
271 static clib_error_t *
272 rdma_rxq_init_flow (struct ibv_flow **flow, struct ibv_qp *qp,
273 const mac_address_t * mac, const mac_address_t * mask,
276 struct raw_eth_flow_attr
278 struct ibv_flow_attr attr;
279 struct ibv_flow_spec_eth spec_eth;
280 } __attribute__ ((packed)) fa;
282 memset (&fa, 0, sizeof (fa));
283 fa.attr.num_of_specs = 1;
285 fa.attr.flags = flags;
286 fa.spec_eth.type = IBV_FLOW_SPEC_ETH;
287 fa.spec_eth.size = sizeof (struct ibv_flow_spec_eth);
289 memcpy (fa.spec_eth.val.dst_mac, mac, sizeof (fa.spec_eth.val.dst_mac));
290 memcpy (fa.spec_eth.mask.dst_mac, mask, sizeof (fa.spec_eth.mask.dst_mac));
292 if ((*flow = ibv_create_flow (qp, &fa.attr)) == 0)
293 return clib_error_return_unix (0, "create Flow Failed");
298 static clib_error_t *
299 rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc)
302 struct ibv_wq_init_attr wqia;
303 struct ibv_wq_attr wqa;
305 vec_validate_aligned (rd->rxqs, qid, CLIB_CACHE_LINE_BYTES);
306 rxq = vec_elt_at_index (rd->rxqs, qid);
309 if ((rxq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
310 return clib_error_return_unix (0, "Create CQ Failed");
312 memset (&wqia, 0, sizeof (wqia));
313 wqia.wq_type = IBV_WQT_RQ;
314 wqia.max_wr = n_desc;
318 if ((rxq->wq = ibv_create_wq (rd->ctx, &wqia)) == 0)
319 return clib_error_return_unix (0, "Create WQ Failed");
321 memset (&wqa, 0, sizeof (wqa));
322 wqa.attr_mask = IBV_WQ_ATTR_STATE;
323 wqa.wq_state = IBV_WQS_RDY;
324 if (ibv_modify_wq (rxq->wq, &wqa) != 0)
325 return clib_error_return_unix (0, "Modify WQ (RDY) Failed");
330 static clib_error_t *
331 rdma_rxq_finalize (vlib_main_t * vm, rdma_device_t * rd)
333 struct ibv_rwq_ind_table_init_attr rwqia;
334 struct ibv_qp_init_attr_ex qpia;
335 const mac_address_t ucast = {.bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
337 const mac_address_t mcast = {.bytes = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0} };
338 struct ibv_wq **ind_tbl;
342 ASSERT (is_pow2 (vec_len (rd->rxqs))
343 && "rxq number should be a power of 2");
345 ind_tbl = vec_new (struct ibv_wq *, vec_len (rd->rxqs));
346 vec_foreach_index (i, rd->rxqs)
347 ind_tbl[i] = vec_elt_at_index (rd->rxqs, i)->wq;
348 memset (&rwqia, 0, sizeof (rwqia));
349 rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl));
350 rwqia.ind_tbl = ind_tbl;
351 if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0)
352 return clib_error_return_unix (0, "RWQ indirection table create failed");
355 memset (&qpia, 0, sizeof (qpia));
356 qpia.qp_type = IBV_QPT_RAW_PACKET;
358 IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_IND_TABLE |
359 IBV_QP_INIT_ATTR_RX_HASH;
361 qpia.rwq_ind_tbl = rd->rx_rwq_ind_tbl;
362 STATIC_ASSERT_SIZEOF (rdma_rss_hash_key, 40);
363 qpia.rx_hash_conf.rx_hash_key_len = sizeof (rdma_rss_hash_key);
364 qpia.rx_hash_conf.rx_hash_key = rdma_rss_hash_key;
365 qpia.rx_hash_conf.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ;
366 qpia.rx_hash_conf.rx_hash_fields_mask =
367 IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4;
368 if ((rd->rx_qp = ibv_create_qp_ex (rd->ctx, &qpia)) == 0)
369 return clib_error_return_unix (0, "Queue Pair create failed");
371 /* receive only packets with src = our MAC */
373 rdma_rxq_init_flow (&rd->flow_ucast, rd->rx_qp, &rd->hwaddr, &ucast,
376 /* receive multicast packets */
377 return rdma_rxq_init_flow (&rd->flow_mcast, rd->rx_qp, &mcast, &mcast,
378 IBV_FLOW_ATTR_FLAGS_DONT_TRAP
379 /* let others receive mcast packet too (eg. Linux) */
383 static clib_error_t *
384 rdma_txq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc)
387 struct ibv_qp_init_attr qpia;
388 struct ibv_qp_attr qpa;
391 vec_validate_aligned (rd->txqs, qid, CLIB_CACHE_LINE_BYTES);
392 txq = vec_elt_at_index (rd->txqs, qid);
395 if ((txq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
396 return clib_error_return_unix (0, "Create CQ Failed");
398 memset (&qpia, 0, sizeof (qpia));
399 qpia.send_cq = txq->cq;
400 qpia.recv_cq = txq->cq;
401 qpia.cap.max_send_wr = n_desc;
402 qpia.cap.max_send_sge = 1;
403 qpia.qp_type = IBV_QPT_RAW_PACKET;
406 if ((txq->qp = ibv_create_qp (rd->pd, &qpia)) == 0)
407 return clib_error_return_unix (0, "Queue Pair create failed");
409 memset (&qpa, 0, sizeof (qpa));
410 qp_flags = IBV_QP_STATE | IBV_QP_PORT;
411 qpa.qp_state = IBV_QPS_INIT;
413 if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
414 return clib_error_return_unix (0, "Modify QP (init) Failed");
416 memset (&qpa, 0, sizeof (qpa));
417 qp_flags = IBV_QP_STATE;
418 qpa.qp_state = IBV_QPS_RTR;
419 if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
420 return clib_error_return_unix (0, "Modify QP (receive) Failed");
422 memset (&qpa, 0, sizeof (qpa));
423 qp_flags = IBV_QP_STATE;
424 qpa.qp_state = IBV_QPS_RTS;
425 if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
426 return clib_error_return_unix (0, "Modify QP (send) Failed");
430 static clib_error_t *
431 rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd, u32 rxq_size,
432 u32 txq_size, u32 rxq_num)
435 vlib_buffer_main_t *bm = vm->buffer_main;
436 vlib_thread_main_t *tm = vlib_get_thread_main ();
440 return clib_error_return_unix (0, "Device Open Failed");
442 if ((rd->pd = ibv_alloc_pd (rd->ctx)) == 0)
443 return clib_error_return_unix (0, "PD Alloc Failed");
445 ethernet_mac_address_generate (rd->hwaddr.bytes);
447 for (i = 0; i < rxq_num; i++)
448 if ((err = rdma_rxq_init (vm, rd, i, rxq_size)))
450 if ((err = rdma_rxq_finalize (vm, rd)))
453 for (i = 0; i < tm->n_vlib_mains; i++)
454 if ((err = rdma_txq_init (vm, rd, i, txq_size)))
457 if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start,
459 IBV_ACCESS_LOCAL_WRITE)) == 0)
460 return clib_error_return_unix (0, "Register MR Failed");
466 sysfs_path_to_pci_addr (char *path, vlib_pci_addr_t * addr)
472 s = clib_sysfs_link_to_name (path);
473 unformat_init_string (&in, (char *) s, strlen ((char *) s));
474 rv = unformat (&in, "%U", unformat_vlib_pci_addr, addr);
481 rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args)
483 vnet_main_t *vnm = vnet_get_main ();
484 rdma_main_t *rm = &rdma_main;
485 rdma_device_t *rd = 0;
486 struct ibv_device **dev_list = 0;
491 args->rxq_size = args->rxq_size ? args->rxq_size : 2 * VLIB_FRAME_SIZE;
492 args->txq_size = args->txq_size ? args->txq_size : 2 * VLIB_FRAME_SIZE;
493 args->rxq_num = args->rxq_num ? args->rxq_num : 1;
495 if (!is_pow2 (args->rxq_num))
497 args->rv = VNET_API_ERROR_INVALID_VALUE;
499 clib_error_return (0, "rx queue number must be a power of two");
503 if (!is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size))
505 args->rv = VNET_API_ERROR_INVALID_VALUE;
507 clib_error_return (0, "queue size must be a power of two");
511 pool_get_zero (rm->devices, rd);
512 rd->dev_instance = rd - rm->devices;
513 rd->per_interface_next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
514 rd->name = vec_dup (args->name);
516 /* check if device exist and if it is bound to mlx5_core */
517 s = format (s, "/sys/class/net/%s/device/driver/module%c", args->ifname, 0);
518 s2 = clib_sysfs_link_to_name ((char *) s);
520 if (s2 == 0 || strncmp ((char *) s2, "mlx5_core", 9) != 0)
523 clib_error_return (0,
524 "invalid interface (only mlx5 supported for now)");
528 /* extract PCI address */
529 vec_reset_length (s);
530 s = format (s, "/sys/class/net/%s/device%c", args->ifname, 0);
531 if (sysfs_path_to_pci_addr ((char *) s, &rd->pci_addr) == 0)
533 args->error = clib_error_return (0, "cannot find PCI address");
537 dev_list = ibv_get_device_list (&n_devs);
541 clib_error_return_unix (0,
542 "no RDMA devices available, errno = %d. "
543 "Is the ib_uverbs module loaded?", errno);
547 for (int i = 0; i < n_devs; i++)
549 vlib_pci_addr_t addr;
551 vec_reset_length (s);
552 s = format (s, "%s/device%c", dev_list[i]->dev_path, 0);
554 if (sysfs_path_to_pci_addr ((char *) s, &addr) == 0)
557 if (addr.as_u32 != rd->pci_addr.as_u32)
560 if ((rd->ctx = ibv_open_device (dev_list[i])))
565 rdma_dev_init (vm, rd, args->rxq_size, args->txq_size, args->rxq_num)))
568 if ((args->error = rdma_register_interface (vnm, rd)))
571 if ((args->error = rdma_async_event_init (rd)))
574 rdma_update_state (vnm, rd, 1);
576 vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, rd->hw_if_index);
577 args->sw_if_index = rd->sw_if_index = sw->sw_if_index;
579 * FIXME: add support for interrupt mode
580 * vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, rd->hw_if_index);
581 * hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
583 vnet_hw_interface_set_input_node (vnm, rd->hw_if_index,
584 rdma_input_node.index);
585 vec_foreach_index (qid, rd->rxqs)
586 vnet_hw_interface_assign_rx_thread (vnm, rd->hw_if_index, qid, ~0);
590 rdma_unregister_interface (vnm, rd);
592 rdma_dev_cleanup (rd);
594 ibv_free_device_list (dev_list);
598 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
599 vlib_log_err (rm->log_class, "%U", format_clib_error, args->error);
603 rdma_delete_if (vlib_main_t * vm, rdma_device_t * rd)
605 rdma_async_event_cleanup (rd);
606 rdma_unregister_interface (vnet_get_main (), rd);
607 rdma_dev_cleanup (rd);
610 static clib_error_t *
611 rdma_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
613 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
614 rdma_main_t *rm = &rdma_main;
615 rdma_device_t *rd = vec_elt_at_index (rm->devices, hi->dev_instance);
616 uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
618 if (rd->flags & RDMA_DEVICE_F_ERROR)
619 return clib_error_return (0, "device is in error state");
623 vnet_hw_interface_set_flags (vnm, rd->hw_if_index,
624 VNET_HW_INTERFACE_FLAG_LINK_UP);
625 rd->flags |= RDMA_DEVICE_F_ADMIN_UP;
629 vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0);
630 rd->flags &= ~RDMA_DEVICE_F_ADMIN_UP;
636 rdma_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
639 rdma_main_t *rm = &rdma_main;
640 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
641 rdma_device_t *rd = pool_elt_at_index (rm->devices, hw->dev_instance);
643 /* Shut off redirection */
644 if (node_index == ~0)
646 rd->per_interface_next_index = node_index;
650 rd->per_interface_next_index =
651 vlib_node_add_next (vlib_get_main (), rdma_input_node.index, node_index);
654 static char *rdma_tx_func_error_strings[] = {
656 foreach_rdma_tx_func_error
661 VNET_DEVICE_CLASS (rdma_device_class,) =
663 .name = "RDMA interface",
664 .format_device = format_rdma_device,
665 .format_device_name = format_rdma_device_name,
666 .admin_up_down_function = rdma_interface_admin_up_down,
667 .rx_redirect_to_node = rdma_set_interface_next_node,
668 .tx_function_n_errors = RDMA_TX_N_ERROR,
669 .tx_function_error_strings = rdma_tx_func_error_strings,
674 rdma_init (vlib_main_t * vm)
676 rdma_main_t *rm = &rdma_main;
678 rm->log_class = vlib_log_register_class ("rdma", 0);
683 VLIB_INIT_FUNCTION (rdma_init);
686 * fd.io coding-style-patch-verification: ON
689 * eval: (c-set-style "gnu")