2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vppinfra/ring.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/devices/devices.h>
25 #include <rdma/rdma.h>
27 static_always_inline u16
28 rdma_device_output_tx (vlib_main_t * vm, rdma_device_t * rd, rdma_txq_t * txq,
29 u32 * buffers, u16 n_left, u32 * n_tx_packets,
32 struct ibv_sge sg_entry;
33 struct ibv_send_wr wr, *bad_wr;
36 for (i = 0; i < n_left; i++)
38 vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]);
39 sg_entry.addr = vlib_buffer_get_current_va (b);
40 sg_entry.length = b->current_length;
41 sg_entry.lkey = rd->mr->lkey;
43 memset (&wr, 0, sizeof (wr));
45 wr.sg_list = &sg_entry;
46 wr.opcode = IBV_WR_SEND;
47 wr.send_flags = IBV_SEND_SIGNALED;
48 wr.wr_id = buffers[i];
50 if (ibv_post_send (txq->qp, &wr, &bad_wr) != 0)
53 *n_tx_bytes += b->current_length;
60 static_always_inline void
61 rdma_device_output_free (vlib_main_t * vm, rdma_txq_t * txq)
63 struct ibv_wc wc[VLIB_FRAME_SIZE];
64 u32 to_free[VLIB_FRAME_SIZE];
68 n_free = ibv_poll_cq (txq->cq, VLIB_FRAME_SIZE, wc);
72 for (i = 0; i < n_free; i++)
73 to_free[i] = wc[i].wr_id;
75 vlib_buffer_free (vm, to_free, n_free);
78 VNET_DEVICE_CLASS_TX_FN (rdma_device_class) (vlib_main_t * vm,
79 vlib_node_runtime_t * node,
82 vnet_main_t *vnm = vnet_get_main ();
83 rdma_main_t *rm = &rdma_main;
84 vnet_interface_output_runtime_t *ord = (void *) node->runtime_data;
85 rdma_device_t *rd = pool_elt_at_index (rm->devices, ord->dev_instance);
86 u32 thread_index = vm->thread_index;
87 u8 qid = thread_index;
88 rdma_txq_t *txq = vec_elt_at_index (rd->txqs, qid % vec_len (rd->txqs));
89 u32 *buffers = vlib_frame_vector_args (frame);
92 u32 n_tx_packets = 0, n_tx_bytes = 0;
94 clib_spinlock_lock_if_init (&txq->lock);
96 n_left = frame->n_vectors;
101 rdma_device_output_free (vm, txq);
103 rdma_device_output_tx (vm, rd, txq, buffers, n_left, &n_tx_packets,
108 if (n_left && n_retry--)
110 vlib_buffer_free (vm, buffers, n_left);
111 vlib_error_count (vm, node->node_index,
112 RDMA_TX_ERROR_NO_FREE_SLOTS, n_left);
117 clib_spinlock_unlock_if_init (&txq->lock);
119 vlib_increment_combined_counter
120 (vnm->interface_main.combined_sw_if_counters +
121 VNET_INTERFACE_COUNTER_TX, thread_index,
122 rd->hw_if_index, n_tx_packets, n_tx_bytes);
124 return frame->n_vectors - n_left;
128 * fd.io coding-style-patch-verification: ON
131 * eval: (c-set-style "gnu")