2 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
19 #include <rte_config.h>
21 #include <rte_ethdev.h>
24 #include <vlib/vlib.h>
25 #include <dpdk/buffer.h>
27 STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
28 "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
34 struct rte_pktmbuf_pool_private mbp_priv;
36 } dpdk_mempool_private_t;
38 #ifndef CLIB_MARCH_VARIANT
39 struct rte_mempool **dpdk_mempool_by_buffer_pool_index = 0;
40 struct rte_mempool **dpdk_no_cache_mempool_by_buffer_pool_index = 0;
43 dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp)
45 uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
46 struct rte_mempool *mp, *nmp;
47 dpdk_mempool_private_t priv;
48 enum rte_iova_mode iova_mode;
53 sizeof (struct rte_mbuf) + sizeof (vlib_buffer_t) + bp->data_size;
55 /* create empty mempools */
56 vec_validate_aligned (dpdk_mempool_by_buffer_pool_index, bp->index,
57 CLIB_CACHE_LINE_BYTES);
58 vec_validate_aligned (dpdk_no_cache_mempool_by_buffer_pool_index, bp->index,
59 CLIB_CACHE_LINE_BYTES);
62 name = format (name, "vpp pool %u%c", bp->index, 0);
63 mp = rte_mempool_create_empty ((char *) name, vec_len (bp->buffers),
64 elt_size, 512, sizeof (priv),
66 vec_reset_length (name);
68 /* non-cached mempool */
69 name = format (name, "vpp pool %u (no cache)%c", bp->index, 0);
70 nmp = rte_mempool_create_empty ((char *) name, vec_len (bp->buffers),
71 elt_size, 0, sizeof (priv),
75 dpdk_mempool_by_buffer_pool_index[bp->index] = mp;
76 dpdk_no_cache_mempool_by_buffer_pool_index[bp->index] = nmp;
78 rte_mempool_set_ops_byname (mp, "vpp", NULL);
79 rte_mempool_set_ops_byname (nmp, "vpp-no-cache", NULL);
81 /* Call the mempool priv initializer */
82 priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE +
83 VLIB_BUFFER_DATA_SIZE;
84 priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE;
85 priv.buffer_pool_index = bp->index;
86 rte_pktmbuf_pool_init (mp, &priv);
87 rte_pktmbuf_pool_init (nmp, &priv);
89 iova_mode = rte_eal_iova_mode ();
91 /* populate mempool object buffer header */
93 vec_foreach (bi, bp->buffers)
95 struct rte_mempool_objhdr *hdr;
96 vlib_buffer_t *b = vlib_get_buffer (vm, *bi);
97 struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer (b);
98 hdr = (struct rte_mempool_objhdr *) RTE_PTR_SUB (mb, sizeof (*hdr));
100 hdr->iova = (iova_mode == RTE_IOVA_VA) ?
101 pointer_to_uword (mb) : vlib_physmem_get_pa (vm, mb);
102 STAILQ_INSERT_TAIL (&mp->elt_list, hdr, next);
103 STAILQ_INSERT_TAIL (&nmp->elt_list, hdr, next);
104 mp->populated_size++;
105 nmp->populated_size++;
109 /* call the object initializers */
110 rte_mempool_obj_iter (mp, rte_pktmbuf_init, 0);
113 vec_foreach (bi, bp->buffers)
116 b = vlib_buffer_ptr_from_index (buffer_mem_start, *bi, 0);
117 vlib_buffer_copy_template (b, &bp->buffer_template);
121 /* map DMA pages if at least one physical device exists */
122 if (rte_eth_dev_count_avail ())
126 vlib_physmem_map_t *pm;
128 pm = vlib_physmem_get_map (vm, bp->physmem_map_index);
129 page_sz = 1ULL << pm->log2_page_size;
131 for (i = 0; i < pm->n_pages; i++)
133 char *va = ((char *) pm->base) + i * page_sz;
134 uword pa = (iova_mode == RTE_IOVA_VA) ?
135 pointer_to_uword (va) : pm->page_table[i];
137 if (rte_vfio_dma_map (pointer_to_uword (va), pa, page_sz))
146 dpdk_ops_vpp_alloc (struct rte_mempool *mp)
153 dpdk_ops_vpp_free (struct rte_mempool *mp)
160 static_always_inline void
161 dpdk_ops_vpp_enqueue_one (vlib_buffer_t * bt, void *obj)
163 /* Only non-replicated packets (b->ref_count == 1) expected */
165 struct rte_mbuf *mb = obj;
166 vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb);
167 ASSERT (b->ref_count == 1);
168 ASSERT (b->buffer_pool_index == bt->buffer_pool_index);
169 vlib_buffer_copy_template (b, bt);
173 CLIB_MULTIARCH_FN (dpdk_ops_vpp_enqueue) (struct rte_mempool * mp,
174 void *const *obj_table, unsigned n)
176 const int batch_size = 32;
177 vlib_main_t *vm = vlib_get_main ();
179 dpdk_mempool_private_t *privp = rte_mempool_get_priv (mp);
180 u8 buffer_pool_index = privp->buffer_pool_index;
181 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
182 u32 bufs[batch_size];
184 void *const *obj = obj_table;
186 vlib_buffer_copy_template (&bt, &bp->buffer_template);
190 dpdk_ops_vpp_enqueue_one (&bt, obj[0]);
191 dpdk_ops_vpp_enqueue_one (&bt, obj[1]);
192 dpdk_ops_vpp_enqueue_one (&bt, obj[2]);
193 dpdk_ops_vpp_enqueue_one (&bt, obj[3]);
200 dpdk_ops_vpp_enqueue_one (&bt, obj[0]);
205 while (n >= batch_size)
207 vlib_get_buffer_indices_with_offset (vm, (void **) obj_table, bufs,
209 sizeof (struct rte_mbuf));
210 vlib_buffer_pool_put (vm, buffer_pool_index, bufs, batch_size);
212 obj_table += batch_size;
217 vlib_get_buffer_indices_with_offset (vm, (void **) obj_table, bufs,
218 n, sizeof (struct rte_mbuf));
219 vlib_buffer_pool_put (vm, buffer_pool_index, bufs, batch_size);
225 CLIB_MARCH_FN_REGISTRATION (dpdk_ops_vpp_enqueue);
227 static_always_inline void
228 dpdk_ops_vpp_enqueue_no_cache_one (vlib_main_t * vm, struct rte_mempool *old,
229 struct rte_mempool *new, void *obj,
232 struct rte_mbuf *mb = obj;
233 vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb);
235 if (clib_atomic_sub_fetch (&b->ref_count, 1) == 0)
237 u32 bi = vlib_get_buffer_index (vm, b);
239 vlib_buffer_copy_template (b, bt);
240 vlib_buffer_pool_put (vm, bt->buffer_pool_index, &bi, 1);
246 CLIB_MULTIARCH_FN (dpdk_ops_vpp_enqueue_no_cache) (struct rte_mempool * cmp,
247 void *const *obj_table,
250 vlib_main_t *vm = vlib_get_main ();
252 dpdk_mempool_private_t *privp = rte_mempool_get_priv (cmp);
253 struct rte_mempool *mp;
254 mp = dpdk_mempool_by_buffer_pool_index[privp->buffer_pool_index];
255 u8 buffer_pool_index = privp->buffer_pool_index;
256 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
257 vlib_buffer_copy_template (&bt, &bp->buffer_template);
261 dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[0], &bt);
262 dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[1], &bt);
263 dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[2], &bt);
264 dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[3], &bt);
271 dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[0], &bt);
279 CLIB_MARCH_FN_REGISTRATION (dpdk_ops_vpp_enqueue_no_cache);
282 CLIB_MULTIARCH_FN (dpdk_ops_vpp_dequeue) (struct rte_mempool * mp,
283 void **obj_table, unsigned n)
285 const int batch_size = 32;
286 vlib_main_t *vm = vlib_get_main ();
287 u32 bufs[batch_size], total = 0, n_alloc = 0;
288 dpdk_mempool_private_t *privp = rte_mempool_get_priv (mp);
289 u8 buffer_pool_index = privp->buffer_pool_index;
290 void **obj = obj_table;
292 while (n >= batch_size)
294 n_alloc = vlib_buffer_alloc_from_pool (vm, bufs, batch_size,
296 if (n_alloc != batch_size)
299 vlib_get_buffers_with_offset (vm, bufs, obj, batch_size,
300 -(i32) sizeof (struct rte_mbuf));
308 n_alloc = vlib_buffer_alloc_from_pool (vm, bufs, n, buffer_pool_index);
313 vlib_get_buffers_with_offset (vm, bufs, obj, n,
314 -(i32) sizeof (struct rte_mbuf));
320 /* dpdk doesn't support partial alloc, so we need to return what we
323 vlib_buffer_pool_put (vm, buffer_pool_index, bufs, n_alloc);
327 vlib_get_buffer_indices_with_offset (vm, obj, bufs, batch_size,
328 sizeof (struct rte_mbuf));
329 vlib_buffer_pool_put (vm, buffer_pool_index, bufs, batch_size);
337 CLIB_MARCH_FN_REGISTRATION (dpdk_ops_vpp_dequeue);
339 #ifndef CLIB_MARCH_VARIANT
342 dpdk_ops_vpp_dequeue_no_cache (struct rte_mempool *mp, void **obj_table,
350 dpdk_ops_vpp_get_count (const struct rte_mempool *mp)
357 dpdk_ops_vpp_get_count_no_cache (const struct rte_mempool *mp)
359 dpdk_mempool_private_t *privp;
360 struct rte_mempool *cmp;
361 privp = rte_mempool_get_priv ((struct rte_mempool *) mp);
362 cmp = dpdk_no_cache_mempool_by_buffer_pool_index[privp->buffer_pool_index];
363 return dpdk_ops_vpp_get_count (cmp);
367 dpdk_buffer_pools_create (vlib_main_t * vm)
370 vlib_buffer_pool_t *bp;
372 struct rte_mempool_ops ops = { };
374 strncpy (ops.name, "vpp", 4);
375 ops.alloc = dpdk_ops_vpp_alloc;
376 ops.free = dpdk_ops_vpp_free;
377 ops.get_count = dpdk_ops_vpp_get_count;
378 ops.enqueue = CLIB_MARCH_FN_POINTER (dpdk_ops_vpp_enqueue);
379 ops.dequeue = CLIB_MARCH_FN_POINTER (dpdk_ops_vpp_dequeue);
380 rte_mempool_register_ops (&ops);
382 strncpy (ops.name, "vpp-no-cache", 13);
383 ops.get_count = dpdk_ops_vpp_get_count_no_cache;
384 ops.enqueue = CLIB_MARCH_FN_POINTER (dpdk_ops_vpp_enqueue_no_cache);
385 ops.dequeue = dpdk_ops_vpp_dequeue_no_cache;
386 rte_mempool_register_ops (&ops);
389 vec_foreach (bp, vm->buffer_main->buffer_pools)
390 if (bp->start && (err = dpdk_buffer_pool_init (vm, bp)))
396 VLIB_BUFFER_SET_EXT_HDR_SIZE (sizeof (struct rte_mempool_objhdr) +
397 sizeof (struct rte_mbuf));
403 * fd.io coding-style-patch-verification: ON
406 * eval: (c-set-style "gnu")