STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
-static struct rte_mbuf ***mbuf_pending_free_list = 0;
+typedef struct
+{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+ struct rte_mbuf ***mbuf_pending_free_list;
+
+ /* cached last pool */
+ struct rte_mempool *last_pool;
+ u8 last_buffer_pool_index;
+} dpdk_buffer_per_thread_data;
+
+typedef struct
+{
+ dpdk_buffer_per_thread_data *ptd;
+} dpdk_buffer_main_t;
+
+dpdk_buffer_main_t dpdk_buffer_main;
static_always_inline void
-dpdk_rte_pktmbuf_free (vlib_main_t * vm, vlib_buffer_t * b)
+dpdk_rte_pktmbuf_free (vlib_main_t * vm, u32 thread_index, vlib_buffer_t * b)
{
vlib_buffer_t *hb = b;
+ dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
+ dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index);
struct rte_mbuf *mb;
u32 next, flags;
mb = rte_mbuf_from_vlib_buffer (hb);
- static struct rte_mempool *last_pool = 0;
- static u8 last_buffer_pool_index;
next:
flags = b->flags;
mb = rte_pktmbuf_prefree_seg (mb);
if (mb)
{
- if (mb->pool != last_pool)
+ if (mb->pool != d->last_pool)
{
- last_pool = mb->pool;
- dpdk_mempool_private_t *privp = rte_mempool_get_priv (last_pool);
- last_buffer_pool_index = privp->buffer_pool_index;
- vec_validate_aligned (mbuf_pending_free_list,
- last_buffer_pool_index,
+ d->last_pool = mb->pool;
+ dpdk_mempool_private_t *privp = rte_mempool_get_priv (d->last_pool);
+ d->last_buffer_pool_index = privp->buffer_pool_index;
+ vec_validate_aligned (d->mbuf_pending_free_list,
+ d->last_buffer_pool_index,
CLIB_CACHE_LINE_BYTES);
}
- vec_add1 (mbuf_pending_free_list[last_buffer_pool_index], mb);
+ vec_add1 (d->mbuf_pending_free_list[d->last_buffer_pool_index], mb);
}
if (flags & VLIB_BUFFER_NEXT_PRESENT)
{
u32 i;
vlib_buffer_t *b;
+ u32 thread_index = vlib_get_thread_index ();
for (i = 0; i < vec_len (f->buffers); i++)
{
b = vlib_get_buffer (vm, f->buffers[i]);
- dpdk_rte_pktmbuf_free (vm, b);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b);
}
vec_free (f->name);
#endif
/* Make sure free list has at least given number of free buffers. */
-static uword
-fill_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * fl, uword min_free_buffers)
+uword
+CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm,
+ vlib_buffer_free_list_t * fl,
+ uword min_free_buffers)
{
dpdk_main_t *dm = &dpdk_main;
vlib_buffer_t *b0, *b1, *b2, *b3;
return n;
}
-static u32
-alloc_from_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * free_list,
- u32 * alloc_buffers, u32 n_alloc_buffers)
-{
- u32 *dst, *src;
- uword len, n_filled;
-
- dst = alloc_buffers;
-
- n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
- if (n_filled == 0)
- return 0;
-
- len = vec_len (free_list->buffers);
- ASSERT (len >= n_alloc_buffers);
-
- src = free_list->buffers + len - n_alloc_buffers;
- clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
-
- _vec_len (free_list->buffers) -= n_alloc_buffers;
-
- return n_alloc_buffers;
-}
-
-/* Allocate a given number of buffers into given array.
- Returns number actually allocated which will be either zero or
- number requested. */
-u32
-CLIB_MULTIARCH_FN (dpdk_buffer_alloc) (vlib_main_t * vm, u32 * buffers,
- u32 n_buffers)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- return alloc_from_free_list
- (vm,
- pool_elt_at_index (bm->buffer_free_list_pool,
- VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
- buffers, n_buffers);
-}
-
-
-u32
-CLIB_MULTIARCH_FN (dpdk_buffer_alloc_from_free_list) (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers,
- u32 free_list_index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- vlib_buffer_free_list_t *f;
- f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
- return alloc_from_free_list (vm, f, buffers, n_buffers);
-}
-
static_always_inline void
dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi)
{
struct rte_mbuf *mb;
b = vlib_get_buffer (vm, bi);
mb = rte_mbuf_from_vlib_buffer (b);
- CLIB_PREFETCH (mb, CLIB_CACHE_LINE_BYTES, STORE);
+ CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, STORE);
CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
}
vlib_buffer_t * b)
{
vlib_buffer_free_list_t *fl;
+ u32 thread_index = vlib_get_thread_index ();
u32 fi;
fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
else
{
if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
- dpdk_rte_pktmbuf_free (vm, b);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b);
}
}
u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
{
vlib_buffer_main_t *bm = vm->buffer_main;
+ dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
vlib_buffer_t *b0, *b1, *b2, *b3;
+ u32 thread_index = vlib_get_thread_index ();
+ dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index);
int i = 0;
u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
u32 follow_buffer_next);
while (i < n_buffers)
{
b0 = vlib_get_buffer (vm, buffers[i]);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
recycle_or_free (vm, bm, buffers[i], b0);
i++;
}
_vec_len (bm->announce_list) = 0;
}
- vec_foreach_index (i, mbuf_pending_free_list)
+ vec_foreach_index (i, d->mbuf_pending_free_list)
{
- int len = vec_len (mbuf_pending_free_list[i]);
+ int len = vec_len (d->mbuf_pending_free_list[i]);
if (len)
{
- rte_mempool_put_bulk (mbuf_pending_free_list[i][len - 1]->pool,
- (void *) mbuf_pending_free_list[i], len);
- vec_reset_length (mbuf_pending_free_list[i]);
+ rte_mempool_put_bulk (d->mbuf_pending_free_list[i][len - 1]->pool,
+ (void *) d->mbuf_pending_free_list[i], len);
+ vec_reset_length (d->mbuf_pending_free_list[i]);
}
}
}
}
#endif
+static clib_error_t *
+dpdk_buffer_init (vlib_main_t * vm)
+{
+ dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
+ vlib_thread_main_t *tm = vlib_get_thread_main ();
+ vec_validate_aligned (dbm->ptd, tm->n_vlib_mains - 1,
+ CLIB_CACHE_LINE_BYTES);
+ return 0;
+}
+
+VLIB_INIT_FUNCTION (dpdk_buffer_init);
+
/* *INDENT-OFF* */
VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
- .vlib_buffer_alloc_cb = &dpdk_buffer_alloc,
- .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list,
+ .vlib_buffer_fill_free_list_cb = &dpdk_buffer_fill_free_list,
.vlib_buffer_free_cb = &dpdk_buffer_free,
.vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next,
.vlib_packet_template_init_cb = &dpdk_packet_template_init,
/* *INDENT-ON* */
#if __x86_64__
-vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx512;
-vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx2;
-vlib_buffer_alloc_from_free_list_cb_t __clib_weak
- dpdk_buffer_alloc_from_free_list_avx512;
-vlib_buffer_alloc_from_free_list_cb_t __clib_weak
- dpdk_buffer_alloc_from_free_list_avx2;
+vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512;
+vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2;
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512;
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2;
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512;
dpdk_input_multiarch_select (void)
{
vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks;
- if (dpdk_buffer_alloc_avx512 && clib_cpu_supports_avx512f ())
+ if (dpdk_buffer_fill_free_list_avx512 && clib_cpu_supports_avx512f ())
{
- cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx512;
- cb->vlib_buffer_alloc_from_free_list_cb =
- dpdk_buffer_alloc_from_free_list_avx512;
+ cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx512;
cb->vlib_buffer_free_cb = dpdk_buffer_free_avx512;
cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx512;
}
- else if (dpdk_buffer_alloc_avx2 && clib_cpu_supports_avx2 ())
+ else if (dpdk_buffer_fill_free_list_avx2 && clib_cpu_supports_avx2 ())
{
- cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx2;
- cb->vlib_buffer_alloc_from_free_list_cb =
- dpdk_buffer_alloc_from_free_list_avx2;
+ cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx2;
cb->vlib_buffer_free_cb = dpdk_buffer_free_avx2;
cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx2;
}
Code Review / vpp.git / blobdiff