1 From c947fd2ec67e9bbacb8b106f320f6e6bae5a9731 Mon Sep 17 00:00:00 2001
2 From: Matthew Smith <mgsmith@netgate.com>
3 Date: Tue, 28 Aug 2018 13:21:04 -0500
4 Subject: [PATCH] mlx4: support externally allocated mempool
6 Port Mellanox mlx5 PMD patch to work for mlx4 PMD.
8 Signed-off-by: Matthew Smith <mgsmith@netgate.com>
10 drivers/net/mlx4/mlx4_mr.c | 150 +++++++++++++++++++++++++++++++++++++++++++
11 drivers/net/mlx4/mlx4_rxtx.h | 35 +++++++++-
12 2 files changed, 184 insertions(+), 1 deletion(-)
14 diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
15 index d23d3c613..55e5555ce 100644
16 --- a/drivers/net/mlx4/mlx4_mr.c
17 +++ b/drivers/net/mlx4/mlx4_mr.c
18 @@ -289,6 +289,23 @@ mr_find_next_chunk(struct mlx4_mr *mr, struct mlx4_mr_cache *entry,
22 + /* MR for external memory doesn't have memseg list. */
23 + if (mr->msl == NULL) {
24 + struct ibv_mr *ibv_mr = mr->ibv_mr;
26 + assert(mr->ms_bmp_n == 1);
27 + assert(mr->ms_n == 1);
28 + assert(base_idx == 0);
30 + * Can't search it from memseg list but get it directly from
31 + * verbs MR as there's only one chunk.
33 + entry->start = (uintptr_t)ibv_mr->addr;
34 + entry->end = (uintptr_t)ibv_mr->addr + mr->ibv_mr->length;
35 + entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
36 + /* Returning 1 ends iteration. */
39 for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
40 if (rte_bitmap_get(mr->ms_bmp, idx)) {
41 const struct rte_memseg_list *msl;
42 @@ -809,6 +826,7 @@ mlx4_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
43 mr = mr_lookup_dev_list(dev, &entry, start);
46 + assert(mr->msl); /* Can't be external memory. */
47 ms = rte_mem_virt2memseg((void *)start, msl);
49 assert(msl->page_sz == ms->hugepage_sz);
50 @@ -1055,6 +1073,134 @@ mlx4_mr_flush_local_cache(struct mlx4_mr_ctrl *mr_ctrl)
51 (void *)mr_ctrl, mr_ctrl->cur_gen);
55 + * Called during rte_mempool_mem_iter() by mlx4_mr_update_ext_mp().
57 + * Externally allocated chunk is registered and a MR is created for the chunk.
58 + * The MR object is added to the global list. If memseg list of a MR object
59 + * (mr->msl) is null, the MR object can be regarded as externally allocated
62 + * Once external memory is registered, it should be static. If the memory is
63 + * freed and the virtual address range has different physical memory mapped
64 + * again, it may cause crash on device due to the wrong translation entry. PMD
65 + * can't track the free event of the external memory for now.
68 +mlx4_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,
69 + struct rte_mempool_memhdr *memhdr,
70 + unsigned mem_idx __rte_unused)
72 + struct mr_update_mp_data *data = opaque;
73 + struct rte_eth_dev *dev = data->dev;
74 + struct priv *priv = dev->data->dev_private;
75 + struct mlx4_mr_ctrl *mr_ctrl = data->mr_ctrl;
76 + struct mlx4_mr *mr = NULL;
77 + uintptr_t addr = (uintptr_t)memhdr->addr;
78 + size_t len = memhdr->len;
79 + struct mlx4_mr_cache entry;
82 + /* If already registered, it should return. */
83 + rte_rwlock_read_lock(&priv->mr.rwlock);
84 + lkey = mr_lookup_dev(dev, &entry, addr);
85 + rte_rwlock_read_unlock(&priv->mr.rwlock);
86 + if (lkey != UINT32_MAX)
88 + mr = rte_zmalloc_socket(NULL,
89 + RTE_ALIGN_CEIL(sizeof(*mr),
90 + RTE_CACHE_LINE_SIZE),
91 + RTE_CACHE_LINE_SIZE, mp->socket_id);
93 + WARN("port %u unable to allocate memory for a new MR of"
95 + dev->data->port_id, mp->name);
99 + DEBUG("port %u register MR for chunk #%d of mempool (%s)",
100 + dev->data->port_id, mem_idx, mp->name);
101 + mr->ibv_mr = mlx4_glue->reg_mr(priv->pd, (void *)addr, len,
102 + IBV_ACCESS_LOCAL_WRITE);
103 + if (mr->ibv_mr == NULL) {
104 + WARN("port %u fail to create a verbs MR for address (%p)",
105 + dev->data->port_id, (void *)addr);
110 + mr->msl = NULL; /* Mark it is external memory. */
114 + rte_rwlock_write_lock(&priv->mr.rwlock);
115 + LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
116 + DEBUG("port %u MR CREATED (%p) for external memory %p:\n"
117 + " [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
118 + " lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
119 + dev->data->port_id, (void *)mr, (void *)addr,
120 + addr, addr + len, rte_cpu_to_be_32(mr->ibv_mr->lkey),
121 + mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
122 + /* Insert to the global cache table. */
123 + mr_insert_dev_cache(dev, mr);
124 + rte_rwlock_write_unlock(&priv->mr.rwlock);
125 + /* Insert to the local cache table */
126 + mlx4_mr_addr2mr_bh(dev, mr_ctrl, addr);
130 + * Register MR for entire memory chunks in a Mempool having externally allocated
131 + * memory and fill in local cache.
134 + * Pointer to Ethernet device.
136 + * Pointer to per-queue MR control structure.
138 + * Pointer to registering Mempool.
141 + * 0 on success, -1 on failure.
144 +mlx4_mr_update_ext_mp(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
145 + struct rte_mempool *mp)
147 + struct mr_update_mp_data data = {
149 + .mr_ctrl = mr_ctrl,
153 + rte_mempool_mem_iter(mp, mlx4_mr_update_ext_mp_cb, &data);
158 + * Register MR entire memory chunks in a Mempool having externally allocated
159 + * memory and search LKey of the address to return.
162 + * Pointer to Ethernet device.
166 + * Pointer to registering Mempool where addr belongs.
169 + * LKey for address on success, UINT32_MAX on failure.
172 +mlx4_tx_update_ext_mp(struct txq *txq, uintptr_t addr,
173 + struct rte_mempool *mp)
175 + struct mlx4_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
176 + struct priv *priv = txq->priv;
178 + mlx4_mr_update_ext_mp(priv->dev, mr_ctrl, mp);
179 + return mlx4_tx_addr2mr_bh(txq, addr);
182 /* Called during rte_mempool_mem_iter() by mlx4_mr_update_mp(). */
184 mlx4_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,
185 @@ -1098,6 +1244,10 @@ mlx4_mr_update_mp(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
188 rte_mempool_mem_iter(mp, mlx4_mr_update_mp_cb, &data);
189 + if (data.ret < 0 && rte_errno == ENXIO) {
190 + /* Mempool may have externally allocated memory. */
191 + return mlx4_mr_update_ext_mp(dev, mr_ctrl, mp);
196 diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
197 index ffa8abfca..1be060cda 100644
198 --- a/drivers/net/mlx4/mlx4_rxtx.h
199 +++ b/drivers/net/mlx4/mlx4_rxtx.h
200 @@ -163,6 +163,26 @@ void mlx4_tx_queue_release(void *dpdk_txq);
201 void mlx4_mr_flush_local_cache(struct mlx4_mr_ctrl *mr_ctrl);
202 uint32_t mlx4_rx_addr2mr_bh(struct rxq *rxq, uintptr_t addr);
203 uint32_t mlx4_tx_addr2mr_bh(struct txq *txq, uintptr_t addr);
204 +uint32_t mlx4_tx_update_ext_mp(struct txq *txq, uintptr_t addr,
205 + struct rte_mempool *mp);
208 + * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the
209 + * cloned mbuf is allocated is returned instead.
215 + * Memory pool where data is located for given mbuf.
217 +static struct rte_mempool *
218 +mlx4_mb2mp(struct rte_mbuf *buf)
220 + if (unlikely(RTE_MBUF_INDIRECT(buf)))
221 + return rte_mbuf_from_indirect(buf)->pool;
226 * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
227 @@ -222,6 +242,19 @@ mlx4_tx_addr2mr(struct txq *txq, uintptr_t addr)
228 return mlx4_tx_addr2mr_bh(txq, addr);
231 -#define mlx4_tx_mb2mr(rxq, mb) mlx4_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
232 +static __rte_always_inline uint32_t
233 +mlx4_tx_mb2mr(struct txq *txq, struct rte_mbuf *mb)
235 + uintptr_t addr = (uintptr_t)mb->buf_addr;
236 + uint32_t lkey = mlx4_tx_addr2mr(txq, addr);
238 + if (likely(lkey != UINT32_MAX))
240 + if (rte_errno == ENXIO) {
241 + /* Mempool may have externally allocated memory. */
242 + lkey = mlx4_tx_update_ext_mp(txq, addr, mlx4_mb2mp(mb));
247 #endif /* MLX4_RXTX_H_ */
249 2.15.2 (Apple Git-101.1)