4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
37 #pragma GCC diagnostic ignored "-Wpedantic"
39 #include <infiniband/verbs.h>
41 #pragma GCC diagnostic error "-Wpedantic"
44 #include <rte_mempool.h>
45 #include <rte_malloc.h>
48 #include "mlx5_rxtx.h"
50 struct mlx5_check_mempool_data {
56 /* Called by mlx5_check_mempool() when iterating the memory chunks. */
58 mlx5_check_mempool_cb(struct rte_mempool *mp __rte_unused,
59 void *opaque, struct rte_mempool_memhdr *memhdr,
60 unsigned int mem_idx __rte_unused)
62 struct mlx5_check_mempool_data *data = opaque;
64 /* It already failed, skip the next chunks. */
67 /* It is the first chunk. */
68 if (data->start == NULL && data->end == NULL) {
69 data->start = memhdr->addr;
70 data->end = data->start + memhdr->len;
73 if (data->end == memhdr->addr) {
74 data->end += memhdr->len;
77 if (data->start == (char *)memhdr->addr + memhdr->len) {
78 data->start -= memhdr->len;
81 /* Error, mempool is not virtually contiguous. */
86 * Check if a mempool can be used: it must be virtually contiguous.
89 * Pointer to memory pool.
91 * Pointer to the start address of the mempool virtual memory area
93 * Pointer to the end address of the mempool virtual memory area
96 * 0 on success (mempool is virtually contiguous), -1 on error.
99 mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
102 struct mlx5_check_mempool_data data;
104 memset(&data, 0, sizeof(data));
105 rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data);
106 *start = (uintptr_t)data.start;
107 *end = (uintptr_t)data.end;
112 * Register a Memory Region (MR) <-> Memory Pool (MP) association in
113 * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
116 * Pointer to TX queue structure.
118 * Memory Pool for which a Memory Region lkey must be returned.
120 * Index of the next available entry.
123 * mr on success, NULL on failure and rte_errno is set.
126 mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp,
129 struct mlx5_txq_ctrl *txq_ctrl =
130 container_of(txq, struct mlx5_txq_ctrl, txq);
131 struct rte_eth_dev *dev;
134 rte_spinlock_lock(&txq_ctrl->priv->mr_lock);
135 /* Add a new entry, register MR first. */
136 DRV_LOG(DEBUG, "port %u discovered new memory pool \"%s\" (%p)",
137 PORT_ID(txq_ctrl->priv), mp->name, (void *)mp);
138 dev = ETH_DEV(txq_ctrl->priv);
139 mr = mlx5_mr_get(dev, mp);
141 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
143 "port %u using unregistered mempool 0x%p(%s)"
144 " in secondary process, please create mempool"
145 " before rte_eth_dev_start()",
146 PORT_ID(txq_ctrl->priv), (void *)mp, mp->name);
147 rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
151 mr = mlx5_mr_new(dev, mp);
153 if (unlikely(mr == NULL)) {
155 "port %u unable to configure memory region,"
156 " ibv_reg_mr() failed.",
157 PORT_ID(txq_ctrl->priv));
158 rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
161 if (unlikely(idx == RTE_DIM(txq->mp2mr))) {
162 /* Table is full, remove oldest entry. */
164 "port %u memory region <-> memory pool table full, "
165 " dropping oldest entry",
166 PORT_ID(txq_ctrl->priv));
168 mlx5_mr_release(txq->mp2mr[0]);
169 memmove(&txq->mp2mr[0], &txq->mp2mr[1],
170 (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
172 /* Store the new entry. */
173 txq_ctrl->txq.mp2mr[idx] = mr;
175 "port %u new memory region lkey for MP \"%s\" (%p): 0x%08"
177 PORT_ID(txq_ctrl->priv), mp->name, (void *)mp,
178 txq_ctrl->txq.mp2mr[idx]->lkey);
179 rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
183 struct mlx5_mp2mr_mbuf_check_data {
188 * Callback function for rte_mempool_obj_iter() to check whether a given
189 * mempool object looks like a mbuf.
192 * The mempool pointer
194 * Context data (struct txq_mp2mr_mbuf_check_data). Contains the
199 * Object index, unused.
202 txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
203 uint32_t index __rte_unused)
205 struct mlx5_mp2mr_mbuf_check_data *data = arg;
206 struct rte_mbuf *buf = obj;
209 * Check whether mbuf structure fits element size and whether mempool
212 if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
217 * Iterator function for rte_mempool_walk() to register existing mempools and
218 * fill the MP to MR cache of a TX queue.
221 * Memory Pool to register.
223 * Pointer to TX queue structure.
226 mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg)
228 struct priv *priv = (struct priv *)arg;
229 struct mlx5_mp2mr_mbuf_check_data data = {
234 /* Register mempool only if the first element looks like a mbuf. */
235 if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
238 mr = mlx5_mr_get(ETH_DEV(priv), mp);
243 mr = mlx5_mr_new(ETH_DEV(priv), mp);
245 DRV_LOG(ERR, "port %u cannot create memory region: %s",
246 PORT_ID(priv), strerror(rte_errno));
250 * Register a new memory region from the mempool and store it in the memory
254 * Pointer to Ethernet device.
256 * Pointer to the memory pool to register.
259 * The memory region on success, NULL on failure and rte_errno is set.
262 mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp)
264 struct priv *priv = dev->data->dev_private;
265 const struct rte_memseg *ms = rte_eal_get_physmem_layout();
271 mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id);
274 "port %u unable to configure memory region,"
275 " ibv_reg_mr() failed.",
280 if (mlx5_check_mempool(mp, &start, &end) != 0) {
281 DRV_LOG(ERR, "port %u mempool %p: not virtually contiguous",
282 dev->data->port_id, (void *)mp);
286 DRV_LOG(DEBUG, "port %u mempool %p area start=%p end=%p size=%zu",
287 dev->data->port_id, (void *)mp, (void *)start, (void *)end,
288 (size_t)(end - start));
289 /* Save original addresses for exact MR lookup. */
292 /* Round start and end to page boundary if found in memory segments. */
293 for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
294 uintptr_t addr = (uintptr_t)ms[i].addr;
295 size_t len = ms[i].len;
296 unsigned int align = ms[i].hugepage_sz;
298 if ((start > addr) && (start < addr + len))
299 start = RTE_ALIGN_FLOOR(start, align);
300 if ((end > addr) && (end < addr + len))
301 end = RTE_ALIGN_CEIL(end, align);
304 "port %u mempool %p using start=%p end=%p size=%zu for memory"
306 dev->data->port_id, (void *)mp, (void *)start, (void *)end,
307 (size_t)(end - start));
308 mr->mr = ibv_reg_mr(priv->pd, (void *)start, end - start,
309 IBV_ACCESS_LOCAL_WRITE);
315 mr->lkey = rte_cpu_to_be_32(mr->mr->lkey);
316 rte_atomic32_inc(&mr->refcnt);
317 DRV_LOG(DEBUG, "port %u new memory Region %p refcnt: %d",
318 dev->data->port_id, (void *)mr, rte_atomic32_read(&mr->refcnt));
319 LIST_INSERT_HEAD(&priv->mr, mr, next);
324 * Search the memory region object in the memory region list.
327 * Pointer to Ethernet device.
329 * Pointer to the memory pool to register.
332 * The memory region on success.
335 mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp)
337 struct priv *priv = dev->data->dev_private;
341 if (LIST_EMPTY(&priv->mr))
343 LIST_FOREACH(mr, &priv->mr, next) {
345 rte_atomic32_inc(&mr->refcnt);
346 DRV_LOG(DEBUG, "port %u memory region %p refcnt: %d",
347 dev->data->port_id, (void *)mr,
348 rte_atomic32_read(&mr->refcnt));
356 * Release the memory region object.
359 * Pointer to memory region to release.
362 * 1 while a reference on it exists, 0 when freed.
365 mlx5_mr_release(struct mlx5_mr *mr)
368 DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void *)mr,
369 rte_atomic32_read(&mr->refcnt));
370 if (rte_atomic32_dec_and_test(&mr->refcnt)) {
371 claim_zero(ibv_dereg_mr(mr->mr));
372 LIST_REMOVE(mr, next);
380 * Verify the flow list is empty
383 * Pointer to Ethernet device.
386 * The number of object not released.
389 mlx5_mr_verify(struct rte_eth_dev *dev)
391 struct priv *priv = dev->data->dev_private;
395 LIST_FOREACH(mr, &priv->mr, next) {
396 DRV_LOG(DEBUG, "port %u memory region %p still referenced",
397 dev->data->port_id, (void *)mr);